class GHTMirrorEvents
Public Instance Methods
go()
click to toggle source
# File lib/ghtorrent/commands/ght_mirror_events.rb, line 64 def go @persister = connect(:mongo, @settings) conn = Bunny.new(:host => config(:amqp_host), :port => config(:amqp_port), :username => config(:amqp_username), :password => config(:amqp_password)) conn.start ch = conn.create_channel debug "Connection to #{config(:amqp_host)} succeded" exchange = ch.topic(config(:amqp_exchange), :durable => true, :auto_delete => false) dupl_msgs = new_msgs = loops = 0 stopped = false while not stopped begin (new, dupl) = retrieve exchange dupl_msgs += dupl new_msgs += new loops += 1 sleep(5) if loops >= 12 # One minute ratio = (dupl_msgs.to_f / (dupl_msgs + new_msgs).to_f) info("Stats: #{new_msgs} new, #{dupl_msgs} duplicate, ratio: #{ratio}") dupl_msgs = new_msgs = loops = 0 end rescue Interrupt stopped = true rescue StandardError => e @logger.error e end end end
retrieve(exchange)
click to toggle source
Retrieve events from Github, store them in the DB
# File lib/ghtorrent/commands/ght_mirror_events.rb, line 38 def retrieve(exchange) begin new = dupl = 0 events = api_request "https://api.github.com/events?per_page=100" (new, dupl, stored) = store_count events # This means that the first page does not contain all new events. Do # a paged request and get everything on the queue if dupl == 0 events = paged_api_request "https://api.github.com/events?per_page=100" (new1, dupl1, stored1) = store_count events stored = stored | stored1 new = new + new1 end stored.each do |e| key = "evt.#{e['type']}" exchange.publish e['id'], :persistent => true, :routing_key => key end return new, dupl rescue StandardError => e STDERR.puts e.message STDERR.puts e.backtrace end end
store_count(events)
click to toggle source
# File lib/ghtorrent/commands/ght_mirror_events.rb, line 20 def store_count(events) stored = Array.new new = dupl = 0 events.each do |e| if @persister.find(:events, {'id' => e['id']}).empty? stored << e new += 1 @persister.store(:events, e) info "Added #{e['id']}" else info "Already got #{e['id']}" dupl += 1 end end return new, dupl, stored end