class GHTorrent::MongoPersister
A persistence adapter that saves data into a configurable MongoDB database.
Constants
- IDXS
- LOCALCONFIG
Supported configuration options.
Attributes
settings[R]
Public Class Methods
new(set)
click to toggle source
Creates a new instance of the MongoDB persistence adapter. Expects a parsed YAML settings document as input. Will create indexes on fields most frequently used in queries.
# File lib/ghtorrent/adapters/mongo_persister.rb, line 47 def initialize(set) merge LOCALCONFIG @settings = set @uniq = config(:uniq_id) end
Public Instance Methods
close()
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 91 def close unless @mongo.nil? @mongo.close if @mongo.class == Mongo::ReplSetConnection @mongo.connection.close if @mongo.class == Mongo::Connection @mongo = nil end end
count(entity, query)
click to toggle source
Count the number of items returned by query
Calls superclass method
GHTorrent::BaseAdapter#count
# File lib/ghtorrent/adapters/mongo_persister.rb, line 74 def count(entity, query) super rescue_connection_failure do get_entity(entity).count(:query => query) end end
del(entity, query)
click to toggle source
Calls superclass method
GHTorrent::BaseAdapter#del
# File lib/ghtorrent/adapters/mongo_persister.rb, line 81 def del(entity, query) super raise Exception 'No filter was specified. Cowardily refusing to remove all entries' if query == {} get_entity(entity).remove(query) end
find(entity, query = {})
click to toggle source
Calls superclass method
GHTorrent::BaseAdapter#find
# File lib/ghtorrent/adapters/mongo_persister.rb, line 61 def find(entity, query = {}) super result = rescue_connection_failure do get_entity(entity).find(query) end result.to_a.map { |r| r[@uniq] = r['_id'].to_s; r.to_h } end
get_underlying_connection()
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 87 def get_underlying_connection mongo end
store(entity, data = {})
click to toggle source
Calls superclass method
GHTorrent::BaseAdapter#store
# File lib/ghtorrent/adapters/mongo_persister.rb, line 54 def store(entity, data = {}) super rescue_connection_failure do get_entity(entity).insert(data).to_s end end
Private Instance Methods
get_collection(col)
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 102 def get_collection(col) mongo.collection(col.to_s) end
get_entity(entity)
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 106 def get_entity(entity) case entity when :users get_collection("users") when :commits get_collection("commits") when :repos get_collection("repos") when :followers get_collection("followers") when :org_members get_collection("org_members") when :events get_collection("events") when :commit_comments get_collection("commit_comments") when :repo_collaborators get_collection("repo_collaborators") when :watchers get_collection("watchers") when :pull_requests get_collection("pull_requests") when :forks get_collection("forks") when :pull_request_comments get_collection("pull_request_comments") when :issues get_collection("issues") when :issue_comments get_collection("issue_comments") when :issue_events get_collection("issue_events") when :repo_labels get_collection("repo_labels") end end
init_db(mongo)
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 172 def init_db(mongo) ENTITIES.each {|x| mongo.collection(x.to_s)} # Ensure that the necessary indexes exist IDXS.each do |k,v| col = get_entity(k) name = v.join('_1_') + '_1' exists = col.index_information.find {|k,v| k == name} idx_fields = v.reduce({}){|acc, x| acc.merge({x => 1})} if exists.nil? col.create_index(idx_fields, :background => true) STDERR.puts "Creating index on #{col}(#{v})" else STDERR.puts "Index on #{col}(#{v}) exists" end end end
mongo()
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 143 def mongo if @mongo.nil? replicas = config(:mongo_replicas) @mongo = if replicas.nil? Mongo::Connection.new(config(:mongo_host), config(:mongo_port))\ .db(config(:mongo_db))\ else repl_arr = replicas.strip.split(/ /).map{|x| "#{x}:#{config(:mongo_port)}"} repl_arr << "#{config(:mongo_host)}:#{config(:mongo_port)}" Mongo::ReplSetConnection.new(repl_arr, :read => :secondary)\ .db(config(:mongo_db)) end unless config(:mongo_username).nil? @mongo.authenticate(config(:mongo_username), config(:mongo_passwd)) end stats = @mongo.stats #init_db(@mongo) if stats['collections'] < ENTITIES.size + 2 #init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size @mongo else @mongo end end
rescue_connection_failure(max_retries=60) { || ... }
click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 192 def rescue_connection_failure(max_retries=60) retries = 0 begin yield rescue Mongo::ConnectionFailure => ex retries += 1 raise ex if retries > max_retries sleep(0.5) @mongo.refresh if @mongo.class == Mongo::ReplSetConnection retry end end