class ProxyList
Public Class Methods
new()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 9 def initialize @proxylists = [] @urls = [] (1..2).each do |i| @urls << "http://proxy.com.ru/list_#{i}.html" end end
Public Instance Methods
fetch_list()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 17 def fetch_list @urls.each do |url| puts proxys_in_url(url).class @proxylists += proxys_in_url(url) end @proxylists.uniq! puts @proxylists puts "Fetched proxys #{@proxylists.size}" end
Also aliased as: start
get_proxylists()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 49 def get_proxylists return @proxylists end
get_urls()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 45 def get_urls return @urls end
proxys_in_url(url)
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 29 def proxys_in_url(url) proxys = [] agent = Mechanize.new page = agent.get(url) list = page.search("body font table tr td:last font table tr") list[1..-1].each do |tr| ip = tr.search("td")[1].text port = tr.search("td")[2].text proxy = { ip: ip, port: port } proxy = verify_proxy(proxy) # 比较费时间 proxys << proxy unless proxy == false puts "add #{proxy} to proxys array..." end return proxys end
save_all_proxylists()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 65 def save_all_proxylists pls = @proxylists.to_s puts pls if File.exist?("allproxylists.txt") File.rename("allproxylists.txt","allproxylists.txt.bak") end f = File.new("allproxylists.txt","w+") f.write(pls) f.close puts "Save all proxylists successfully." end
save_proxylists()
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 53 def save_proxylists pls = @proxylists.to_s puts pls if File.exist?("proxylists.txt") File.rename("proxylists.txt","proxylists.txt.bak") end f = File.new("proxylists.txt","w+") f.write(pls) f.close puts "Save proxylists successfully." end
verify_proxy(proxy)
click to toggle source
# File lib/sync_qiniu/getproxy.rb, line 77 def verify_proxy(proxy) ip, port = proxy[:ip], proxy[:port] testagent = Mechanize.new testagent.set_proxy ip, port testagent.read_timeout = 10 begin page = testagent.get("http://www.baidu.com") if page.title == "百度一下,你就知道" return proxy puts 'That a good proxy' end rescue => e puts e.message puts 'That a bad proxy' proxy = nil return end end