module Dumper
Constants
- USER_AGENT
Public Class Methods
get(path, url, options = {})
click to toggle source
# File lib/dumper/dumper.rb, line 72 def get(path, url, options = {}) url = url.to_s errors = 0 begin if url.start_with? 'data:image/' filename = File.join path, options[:filename] || rand(1000).to_s + '.' + url.split('data:image/')[1].split(?;)[0] filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR url.gsub /data:image\/png;base64,/, '' if File.exists? filename notify_observers error: "File #{filename} already exists." else notify_observers status: "Downloading base64 image as #{filename}..." File.open(filename, 'wb') { |f| f.write Base64.decode64(url) } end else filename = File.join path, options[:filename] || File.basename(url) filename.gsub! File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR if File.exists? filename notify_observers error: "File #{filename} already exists." else filename = File.join(path, rand(1000).to_s + '.jpg') unless filename[-4] == ?. || filename[-5] == ?. http_options = { 'User-Agent' => options[:user_agent] || USER_AGENT, 'Referer' => options[:referer ] || url } url.prepend('http:') if url.start_with?('//') uri = URI(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' http.verify_mode = OpenSSL::SSL::VERIFY_NONE http.start do |h| notify_observers status: "Downloading #{url} as #{filename}..." response = h.request Net::HTTP::Get.new(uri.request_uri, http_options) File.open(filename, 'wb') do |f| f.write response.body end end end end rescue Exception => e if errors <= 3 errors += 1 sleep 3 retry else notify_observers critical_error: "Error downloading #{url}.", critical_error_dump: e return false end end true end
get_generic(url, path, xpath)
click to toggle source
# File lib/dumper/dumper.rb, line 134 def get_generic(url, path, xpath) uri = nil Nokogiri::HTML(open(url)).xpath(xpath).each do |p| if p.to_s.start_with? ?/ uri = URI(url) if uri.nil? p = "#{uri.scheme}://#{uri.host}#{p}" end get path, p end end
list()
click to toggle source
# File lib/dumper/dumper.rb, line 66 def list Dir.glob(File.expand_path('../profiles/*.rb', __FILE__)).sort { |a, b| b <=> a }.map do |f| f = File.basename(f).split(?.)[0] end end
pool_size=(min, max = nil)
click to toggle source
# File lib/dumper/dumper.rb, line 43 def pool_size=(min, max = nil) @min = min @max = max end
shut_up!()
click to toggle source
# File lib/dumper/dumper.rb, line 52 def shut_up! @verbose == false end
Also aliased as: mute!
verbose!()
click to toggle source
# File lib/dumper/dumper.rb, line 57 def verbose! @verbose == true end
Also aliased as: unmute!
verbose=(verbose)
click to toggle source
# File lib/dumper/dumper.rb, line 48 def verbose=(verbose) @verbose = verbose end
verbose?()
click to toggle source
# File lib/dumper/dumper.rb, line 62 def verbose? @verbose == nil || @verbose == true end
version()
click to toggle source
# File lib/dumper/version.rb, line 21 def self.version '0.8.5' end
Public Instance Methods
method_missing(method, *args, &block)
click to toggle source
# File lib/dumper/dumper.rb, line 147 def method_missing(method, *args, &block) "'#{method.split('get_')[1]}' profile not found." end
pool_size()
click to toggle source
# File lib/dumper/dumper.rb, line 23 def pool_size { min: @min || 4, max: @max } end
verbose?()
click to toggle source
# File lib/dumper/dumper.rb, line 30 def verbose? @verbose == nil || @verbose == true end
Also aliased as: mute?