module SilentFetcher
Constants
- DEFAULT_CHARSET
- DEFAULT_RETRY_COUNT
- DEFAULT_TIMEOUT
- EXPECTED_ERRORS
- RETRYABLE_ERRORS
- VERSION
Attributes
configuration[RW]
Public Class Methods
configure() { |configuration| ... }
click to toggle source
# File lib/silent_fetcher.rb, line 78 def configure self.configuration ||= Configuration.new yield(configuration) end
expected_error_classes()
click to toggle source
# File lib/silent_fetcher.rb, line 83 def expected_error_classes EXPECTED_ERRORS.keys.map(&:constantize) end
fetch(url, retry_count: DEFAULT_RETRY_COUNT, allow_no_response: false)
click to toggle source
# File lib/silent_fetcher.rb, line 50 def fetch(url, retry_count: DEFAULT_RETRY_COUNT, allow_no_response: false) response = HTTParty.get(url, fetch_options) if response.body.size == 0 && !allow_no_response raise SilentFetcher::ExpectedError, "response.body.size == 0: #{url}" end response.body rescue *RETRYABLE_ERRORS => e if retry_count > 0 retry_count -= 1 retry else raise SilentFetcher::ExpectedError, "#{e.message}: #{url}" end rescue => e class_string = e.class.to_s if EXPECTED_ERRORS[class_string] && (EXPECTED_ERRORS[class_string].none? || EXPECTED_ERRORS[class_string].any? {|m| e.message =~ m }) raise SilentFetcher::ExpectedError, "#{e.class}: #{e.message} by #{url}" else raise e end end
parse_feed(url)
click to toggle source
# File lib/silent_fetcher.rb, line 44 def parse_feed(url) Feedjira::Feed.parse( fetch(url) ) end
parse_html(url, charset: DEFAULT_CHARSET)
click to toggle source
# File lib/silent_fetcher.rb, line 40 def parse_html(url, charset: DEFAULT_CHARSET) Nokogiri::HTML(fetch(url), nil, charset) end
Private Class Methods
fetch_options()
click to toggle source
# File lib/silent_fetcher.rb, line 89 def fetch_options { headers: { 'User-Agent' => configuration.user_agent }, timeout: DEFAULT_TIMEOUT } end