class Gnip::GnipFullArchive::FullArchive
Attributes
counts_url[R]
search_url[R]
Public Class Methods
new(client)
click to toggle source
alias :total :total_entries
# File lib/gnip/gnip-full-archive/full_archive.rb, line 15 def initialize(client) @search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json" @counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json" @auth = { username: client.username, password: client.password } end
Public Instance Methods
search(options = {})
click to toggle source
Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page options query to twitter options default is 500 options as datetime options as datetime options cursor to the next page
# File lib/gnip/gnip-full-archive/full_archive.rb, line 27 def search(options = {}) search_options = {} search_options[:query] = options[:query] || '' search_options[:maxResults] = options[:per_page] || 500 search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from] search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to] search_options[:next] = options[:next_cursor] if options[:next_cursor] url = [search_url, search_options.to_query].join('?') begin gnip_call = self.class.get(url, basic_auth: @auth) response = gnip_call.response parsed_response = gnip_call.parsed_response parsed_response = (parsed_response || {}).with_indifferent_access raise response.message unless parsed_response.present? if parsed_response[:error].present? response = { results: [], next: nil, url: url, error: parsed_response[:error][:message], code: response.code.to_i } else response = { results: parsed_response[:results], url: url, next: parsed_response[:next], code: response.code.to_i } end rescue StandardError => e response = { results: [], url: url, next: nil, error: e.message, code: 500 } end response end
total(options = {})
click to toggle source
return total contents in a specific date interval with a passed query
# File lib/gnip/gnip-full-archive/full_archive.rb, line 112 def total(options = {}) extra = {} response = total_by_time_period(options) extra = { error: response[:error] } if response[:error].present? { query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra) end
total_by_time_period(options = {})
click to toggle source
full aarchive search endpoints return total contents by day, minute, hour paginated so to get totals across time period passed may need to run more than one call, the stop condition is cursor nil bucket: must be one of [minute, hour, day]
# File lib/gnip/gnip-full-archive/full_archive.rb, line 67 def total_by_time_period(options = {}) response = options[:response] || {} search_options = {} search_options[:query] = options[:query] || '' search_options[:bucket] = options[:bucket] || 'day' search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from] search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to] search_options[:next] = options[:next_cursor] if options[:next_cursor] url = [counts_url, search_options.to_query].join('?') call_done = 0 begin gnip_call = self.class.get(url, basic_auth: @auth) parsed_response = gnip_call.parsed_response parsed_response = (parsed_response || {}).with_indifferent_access raise gnip_call.response.message unless parsed_response.present? if parsed_response[:error].present? response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 } else call_done = 1 # we have received a valid response parsed_response[:results].each_with_index do |item, i| parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s) end response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 } end rescue StandardError => e response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done } end # If the next cursor is not present we fetched all the data # It happens that twitter returns the same cursor, in that case we stop return response if !parsed_response[:next].to_s.present? || (parsed_response[:next].to_s.present? && parsed_response[:next] == search_options[:next]) total_by_time_period(query: search_options[:query], date_from: search_options[:fromDate], date_to: search_options[:toDate], bucket: search_options[:bucket], next_cursor: parsed_response[:next], response: response) end