module GHTorrent::APIClient

Public Instance Methods

api_request(url) click to toggle source

A normal request. Returns a hash or an array of hashes representing the parsed JSON result.

# File lib/ghtorrent/api_client.rb, line 58
def api_request(url)
  parse_request_result api_request_raw(ensure_max_per_page(url))
end
num_pages(url) click to toggle source

Determine the number of pages contained in a multi-page API response

# File lib/ghtorrent/api_client.rb, line 63
def num_pages(url)
  url = ensure_max_per_page(url)
  data = api_request_raw(url)

  if data.nil? or data.meta.nil? or data.meta['link'].nil?
    return 1
  end

  links = parse_links(data.meta['link'])

  if links.nil? or links['last'].nil?
    return 1
  end

  params = CGI::parse(URI::parse(links['last']).query)
  params['page'][0].to_i
end
paged_api_request(url, pages = config(:mirror_history_pages_back), last = nil) click to toggle source

A paged request. Used when the result can expand to more than one result pages.

# File lib/ghtorrent/api_client.rb, line 26
def paged_api_request(url, pages = config(:mirror_history_pages_back),
                      last = nil)

  url = ensure_max_per_page(url)
  data = api_request_raw(url)

  return [] if data.nil?

  unless data.meta['link'].nil?
    links = parse_links(data.meta['link'])
    last = links['last'] if last.nil?

    if pages > 0
      pages = pages - 1
      if pages == 0
        return parse_request_result(data)
      end
    end

    if links['next'].nil?
      parse_request_result(data)
    else
      parse_request_result(data) | paged_api_request(links['next'], pages, last)
    end
  else
    parse_request_result(data)
  end
end

Private Instance Methods

api_request_raw(url) click to toggle source

Do the actual request and return the result object

# File lib/ghtorrent/api_client.rb, line 141
def api_request_raw(url)

  begin
    start_time = Time.now

    contents = do_request(url)
    total = Time.now.to_ms - start_time.to_ms
    info "Successful request. URL: #{url}, Remaining: #{@remaining}, Total: #{total} ms"

    contents
  rescue OpenURI::HTTPError => e
      @remaining = e.io.meta['x-ratelimit-remaining'].to_i
      @reset = e.io.meta['x-ratelimit-reset'].to_i

      case e.io.status[0].to_i
      # The following indicate valid Github return codes
      when 400, # Bad request
          401, # Unauthorized
          403, # Forbidden
          404, # Not found
          422 then # Unprocessable entity
        total = Time.now.to_ms - start_time.to_ms
        warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
       return nil
      else # Server error or HTTP conditions that Github does not report
        warn request_error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
        raise e
    end
  rescue StandardError => e
    warn error_msg(url, e).strip.gsub(/\s+/,' ').gsub("\n", ' ')
    raise e
  ensure
    # The exact limit is only enforced upon the first @reset
    if 5000 - @remaining >= @req_limit
      to_sleep = @reset - Time.now.to_i + 2
      debug "Request limit reached, sleeping for #{to_sleep} secs"
      t = Thread.new do
        slept = 0
        while true do
          debug "Sleeping for #{to_sleep - slept} seconds"
          sleep 1
          slept += 1
        end
      end
      sleep(to_sleep)
      t.exit
    end
  end
end
attach_to(ip) { || ... } click to toggle source

Attach to a specific IP address if the machine has multiple

# File lib/ghtorrent/api_client.rb, line 240
def attach_to(ip)
  TCPSocket.instance_eval do
    (class << self; self; end).instance_eval do
      alias_method :original_open, :open

      case RUBY_VERSION
      when /1.9/
        define_method(:open) do |conn_address, conn_port|
          original_open(conn_address, conn_port, ip)
        end
      when /2.0/
        define_method(:open) do |conn_address, conn_port, local_host, local_port|
          original_open(conn_address, conn_port, ip, local_port)
        end
      end
    end
  end

  result = begin
    yield
  rescue StandardError => e
    raise e
  ensure
    TCPSocket.instance_eval do
      (class << self; self; end).instance_eval do
        alias_method :open, :original_open
        remove_method :original_open
      end
    end
  end

  result
end
auth_method(username, token) click to toggle source
# File lib/ghtorrent/api_client.rb, line 191
def auth_method(username, token)
  if token.nil? or token.empty?
    if username.nil? or username.empty?
      :none
    else
      :username
    end
  else
    :token
  end
end
do_request(url) click to toggle source
# File lib/ghtorrent/api_client.rb, line 203
def do_request(url)
  @attach_ip  ||= config(:attach_ip)
  @token      ||= config(:github_token)
  @username   ||= config(:github_username)
  @passwd     ||= config(:github_passwd)
  @user_agent ||= config(:user_agent)
  @remaining  ||= 5000
  @reset      ||= Time.now.to_i + 3600
  @auth_type  ||= auth_method(@username, @token)
  @req_limit  ||= config(:req_limit)

  open_func ||=
      case @auth_type
        when :none
          lambda {|url| open(url, 'User-Agent' => @user_agent)}
        when :username
          lambda {|url| open(url, 'User-Agent' => @user_agent,
                             :http_basic_authentication => [@username, @passwd])}
        when :token
          # As per: https://developer.github.com/v3/auth/#via-oauth-tokens
          lambda {|url| open(url, 'User-Agent' => @user_agent,
                             'Authorization' => "token #{@token}") }
      end

  result = if @attach_ip.nil? or @attach_ip.eql? '0.0.0.0'
      open_func.call(url)
    else
      attach_to(@attach_ip) do
        open_func.call(url)
      end
    end
  @remaining = result.meta['x-ratelimit-remaining'].to_i
  @reset = result.meta['x-ratelimit-reset'].to_i
  result
end
ensure_max_per_page(url) click to toggle source
# File lib/ghtorrent/api_client.rb, line 83
def ensure_max_per_page(url)
  if url.include?('page')
    if not url.include?('per_page')
      if url.include?('?')
        url + '&per_page=100'
      else
        url + '?per_page=100'
      end
    else
      url
    end
  else
    url
  end
end
error_msg(url, exception) click to toggle source
# File lib/ghtorrent/api_client.rb, line 132
    def error_msg(url, exception)
      <<-MSG
            Failed request. URL: #{url}, Exception: #{exception.message},
            Access: #{if (@token.nil? or @token.empty?) then @username else @token end},
            IP: #{@attach_ip}, Remaining: #{@remaining}
      MSG
    end
parse_request_result(result) click to toggle source

Parse the JSON result array

# File lib/ghtorrent/api_client.rb, line 109
def parse_request_result(result)
  if result.nil?
    []
  else
    json = result.read

    if json.nil?
      []
    else
      JSON.parse(json)
    end
  end
end
request_error_msg(url, exception) click to toggle source
# File lib/ghtorrent/api_client.rb, line 123
    def request_error_msg(url, exception)
      <<-MSG
            Failed request. URL: #{url}, Status code: #{exception.io.status[0]},
            Status: #{exception.io.status[1]},
            Access: #{if (@token.nil? or @token.empty?) then @username else @token end},
            IP: #{@attach_ip}, Remaining: #{@remaining}
      MSG
    end