class ProxyPool::Dealer

Pool

Attributes

pools[R]

Public Instance Methods

get() { |proxy| ... } click to toggle source

Get a random proxy

@return [Hash] Proxy

# File lib/proxy_pool/dealer.rb, line 51
def get
  update if _need_update?

  target_pools = if block_given?
                   @pools.select { |proxy| yield proxy }
                 else
                   @pools
                 end
  target_pools.sample
end
remove(proxy) click to toggle source

Remove this proxy from pool

@param proxy [Hash] Proxy @return [nil]

# File lib/proxy_pool/dealer.rb, line 66
def remove(proxy)
  @pools.delete(proxy)

  nil
end
update() click to toggle source

Update to latest proxy list from fate0/proxylist

# File lib/proxy_pool/dealer.rb, line 33
def update
  @pools = []

  res = HTTP.get 'https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list'
  raise HTTPError, "invalid http code #{res.code}" if res.code != 200

  @updated_at = Time.now

  res.body.to_s.split("\n").each do |line|
    _pool_parse(line)
  rescue ParseError
    next
  end
end

Private Instance Methods

_need_update?() click to toggle source

Return true if we need to update proxy list

@return [Boolean]

# File lib/proxy_pool/dealer.rb, line 77
def _need_update?
  return true if @updated_at.nil?

  # Re-update after 10 min
  delta = Time.now - @updated_at
  delta > 600
end
_pool_parse(line) click to toggle source

Parse proxy list

@param line [String]

# File lib/proxy_pool/dealer.rb, line 88
def _pool_parse(line)
  proxy = JSON.parse(line)
  if proxy['anonymity'].nil? ||
      proxy['type'].nil? ||
      proxy['country'].nil?
    raise ParseError
  end

  @pools << proxy
rescue JSON::ParserError
  raise ParseError, "JSON parser error when parsing #{line}"
end