module GHTorrent::APIClient
Public Instance Methods
api_request(url)
click to toggle source
A normal request. Returns a hash or an array of hashes representing the parsed JSON result.
# File lib/ghtorrent/api_client.rb, line 58 def api_request(url) parse_request_result api_request_raw(ensure_max_per_page(url)) end
num_pages(url)
click to toggle source
Determine the number of pages contained in a multi-page API response
# File lib/ghtorrent/api_client.rb, line 63 def num_pages(url) url = ensure_max_per_page(url) data = api_request_raw(url) if data.nil? or data.meta.nil? or data.meta['link'].nil? return 1 end links = parse_links(data.meta['link']) if links.nil? or links['last'].nil? return 1 end params = CGI::parse(URI::parse(links['last']).query) params['page'][0].to_i end
paged_api_request(url, pages = config(:mirror_history_pages_back), last = nil)
click to toggle source
A paged request. Used when the result can expand to more than one result pages.
# File lib/ghtorrent/api_client.rb, line 26 def paged_api_request(url, pages = config(:mirror_history_pages_back), last = nil) url = ensure_max_per_page(url) data = api_request_raw(url) return [] if data.nil? unless data.meta['link'].nil? links = parse_links(data.meta['link']) last = links['last'] if last.nil? if pages > 0 pages = pages - 1 if pages == 0 return parse_request_result(data) end end if links['next'].nil? parse_request_result(data) else parse_request_result(data) | paged_api_request(links['next'], pages, last) end else parse_request_result(data) end end
Private Instance Methods
api_request_raw(url)
click to toggle source
Do the actual request and return the result object
# File lib/ghtorrent/api_client.rb, line 141 def api_request_raw(url) begin start_time = Time.now contents = do_request(url) total = Time.now.to_ms - start_time.to_ms info "Successful request. URL: #{url}, Remaining: #{@remaining}, Total: #{total} ms" contents rescue OpenURI::HTTPError => e @remaining = e.io.meta['x-ratelimit-remaining'].to_i @reset = e.io.meta['x-ratelimit-reset'].to_i case e.io.status[0].to_i # The following indicate valid Github return codes when 400, # Bad request 401, # Unauthorized 403, # Forbidden 404, # Not found 422 then # Unprocessable entity total = Time.now.to_ms - start_time.to_ms warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ') return nil else # Server error or HTTP conditions that Github does not report warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ') raise e end rescue StandardError => e warn error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ') raise e ensure # The exact limit is only enforced upon the first @reset if 5000 - @remaining >= @req_limit to_sleep = @reset - Time.now.to_i + 2 debug "Request limit reached, sleeping for #{to_sleep} secs" t = Thread.new do slept = 0 while true do debug "Sleeping for #{to_sleep - slept} seconds" sleep 1 slept += 1 end end sleep(to_sleep) t.exit end end end
attach_to(ip) { || ... }
click to toggle source
Attach to a specific IP address if the machine has multiple
# File lib/ghtorrent/api_client.rb, line 240 def attach_to(ip) TCPSocket.instance_eval do (class << self; self; end).instance_eval do alias_method :original_open, :open case RUBY_VERSION when /1.9/ define_method(:open) do |conn_address, conn_port| original_open(conn_address, conn_port, ip) end when /2.0/ define_method(:open) do |conn_address, conn_port, local_host, local_port| original_open(conn_address, conn_port, ip, local_port) end end end end result = begin yield rescue StandardError => e raise e ensure TCPSocket.instance_eval do (class << self; self; end).instance_eval do alias_method :open, :original_open remove_method :original_open end end end result end
auth_method(username, token)
click to toggle source
# File lib/ghtorrent/api_client.rb, line 191 def auth_method(username, token) if token.nil? or token.empty? if username.nil? or username.empty? :none else :username end else :token end end
do_request(url)
click to toggle source
# File lib/ghtorrent/api_client.rb, line 203 def do_request(url) @attach_ip ||= config(:attach_ip) @token ||= config(:github_token) @username ||= config(:github_username) @passwd ||= config(:github_passwd) @user_agent ||= config(:user_agent) @remaining ||= 5000 @reset ||= Time.now.to_i + 3600 @auth_type ||= auth_method(@username, @token) @req_limit ||= config(:req_limit) open_func ||= case @auth_type when :none lambda {|url| open(url, 'User-Agent' => @user_agent)} when :username lambda {|url| open(url, 'User-Agent' => @user_agent, :http_basic_authentication => [@username, @passwd])} when :token # As per: https://developer.github.com/v3/auth/#via-oauth-tokens lambda {|url| open(url, 'User-Agent' => @user_agent, 'Authorization' => "token #{@token}") } end result = if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0' open_func.call(url) else attach_to(@attach_ip) do open_func.call(url) end end @remaining = result.meta['x-ratelimit-remaining'].to_i @reset = result.meta['x-ratelimit-reset'].to_i result end
ensure_max_per_page(url)
click to toggle source
# File lib/ghtorrent/api_client.rb, line 83 def ensure_max_per_page(url) if url.include?('page') if not url.include?('per_page') if url.include?('?') url + '&per_page=100' else url + '?per_page=100' end else url end else url end end
error_msg(url, exception)
click to toggle source
# File lib/ghtorrent/api_client.rb, line 132 def error_msg(url, exception) <<-MSG Failed request. URL: #{url}, Exception: #{exception.message}, Access: #{if (@token.nil? or @token.empty?) then @username else @token end}, IP: #{@attach_ip}, Remaining: #{@remaining} MSG end
parse_links(links)
click to toggle source
Parse a Github link header
# File lib/ghtorrent/api_client.rb, line 100 def parse_links(links) links.split(/,/).reduce({}) do |acc, x| matches = x.strip.match(/<(.*)>; rel=\"(.*)\"/) acc[matches[2]] = matches[1] acc end end
parse_request_result(result)
click to toggle source
Parse the JSON result array
# File lib/ghtorrent/api_client.rb, line 109 def parse_request_result(result) if result.nil? [] else json = result.read if json.nil? [] else JSON.parse(json) end end end
request_error_msg(url, exception)
click to toggle source
# File lib/ghtorrent/api_client.rb, line 123 def request_error_msg(url, exception) <<-MSG Failed request. URL: #{url}, Status code: #{exception.io.status[0]}, Status: #{exception.io.status[1]}, Access: #{if (@token.nil? or @token.empty?) then @username else @token end}, IP: #{@attach_ip}, Remaining: #{@remaining} MSG end