class AeEasy::Qa::ValidateCollection

Attributes

collection_name[R]
errors[R]
options[R]
outputs[R]
rules[R]
scraper_name[R]
total_records[R]

Public Class Methods

new(scraper_name, collection_name, total_records, rules, outputs, options) click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 107
def initialize(scraper_name, collection_name, total_records, rules, outputs, options)
  @scraper_name = scraper_name
  @collection_name = collection_name
  @total_records = total_records
  @rules = rules
  @outputs = outputs
  @data = options['data']
  @options = options
  @errors = { errored_items: [] }
end

Public Instance Methods

run() click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 118
def run
  output_collection
  if most_recent_finished_job
    puts "data count #{data.count}"
    if data.any?
      ValidateGroups.new(data, scraper_name, collection_name, errors).run
      ValidateRules.new(data, errors, rules).run if rules
    end
    SaveOutput.new(data.count, rules, errors, outputs_collection_name, outputs, options).run
  else
    puts "No job with status 'done' available"
  end
end

Private Instance Methods

data() click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 149
def data
  @data ||= begin
              data = []
              page = 1
              while data.count < total_records
                records = AnswersEngine::Client::JobOutput.new(per_page:500, page: page).all(most_recent_finished_job['id'], collection_name).parsed_response
                sleep 1
                if records
                  records.each do |record|
                    data << record
                  end
                else
                  puts "All ScraperJobOutput request was nil. Total records: #{total_records}, data count: #{data.count}, page: #{page}"
                  break
                end
                page += 1
              end
              data
            end
end
most_recent_finished_job() click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 138
def most_recent_finished_job
  @most_recent_finished_job ||= begin
                                  jobs_response = AnswersEngine::Client::ScraperJob.new.all(scraper_name)
                                  if jobs_response.code == 200
                                    jobs_response.parsed_response.sort_by { |job| job['created_at'] }.reverse.find{|job| job['status'] == 'active' || job['status'] == 'done' }
                                  else
                                    nil
                                  end
                                end
end
output_collection() click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 134
def output_collection
  puts "Validating collection: #{collection_name}"
end
outputs_collection_name() click to toggle source
# File lib/ae_easy/qa/validate_internal.rb, line 170
def outputs_collection_name
  @outputs_collection_name ||= "#{scraper_name}_#{collection_name}"
end