class GHTorrent::MongoPersister

A persistence adapter that saves data into a configurable MongoDB database.

Constants

IDXS
LOCALCONFIG

Supported configuration options.

Attributes

settings[R]

Public Class Methods

new(set) click to toggle source

Creates a new instance of the MongoDB persistence adapter. Expects a parsed YAML settings document as input. Will create indexes on fields most frequently used in queries.

# File lib/ghtorrent/adapters/mongo_persister.rb, line 47
def initialize(set)
  merge LOCALCONFIG

  @settings = set
  @uniq = config(:uniq_id)
end

Public Instance Methods

close() click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 91
def close
  unless @mongo.nil?
    @mongo.close if @mongo.class == Mongo::ReplSetConnection
    @mongo.connection.close if @mongo.class == Mongo::Connection

    @mongo = nil
  end
end
count(entity, query) click to toggle source

Count the number of items returned by query

Calls superclass method GHTorrent::BaseAdapter#count
# File lib/ghtorrent/adapters/mongo_persister.rb, line 74
def count(entity, query)
  super
  rescue_connection_failure do
    get_entity(entity).count(:query => query)
  end
end
del(entity, query) click to toggle source
Calls superclass method GHTorrent::BaseAdapter#del
# File lib/ghtorrent/adapters/mongo_persister.rb, line 81
def del(entity, query)
  super
  raise Exception 'No filter was specified. Cowardily refusing to remove all entries' if query == {}
  get_entity(entity).remove(query)
end
find(entity, query = {}) click to toggle source
Calls superclass method GHTorrent::BaseAdapter#find
# File lib/ghtorrent/adapters/mongo_persister.rb, line 61
def find(entity, query = {})
  super
  result = rescue_connection_failure do
    get_entity(entity).find(query)
  end

  result.to_a.map { |r|
    r[@uniq] = r['_id'].to_s;
    r.to_h
  }
end
get_underlying_connection() click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 87
def get_underlying_connection
  mongo
end
store(entity, data = {}) click to toggle source
Calls superclass method GHTorrent::BaseAdapter#store
# File lib/ghtorrent/adapters/mongo_persister.rb, line 54
def store(entity, data = {})
  super
  rescue_connection_failure do
    get_entity(entity).insert(data).to_s
  end
end

Private Instance Methods

get_collection(col) click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 102
def get_collection(col)
  mongo.collection(col.to_s)
end
get_entity(entity) click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 106
def get_entity(entity)
  case entity
    when :users
      get_collection("users")
    when :commits
      get_collection("commits")
    when :repos
      get_collection("repos")
    when :followers
      get_collection("followers")
    when :org_members
      get_collection("org_members")
    when :events
      get_collection("events")
    when :commit_comments
      get_collection("commit_comments")
    when :repo_collaborators
      get_collection("repo_collaborators")
    when :watchers
      get_collection("watchers")
    when :pull_requests
      get_collection("pull_requests")
    when :forks
      get_collection("forks")
    when :pull_request_comments
      get_collection("pull_request_comments")
    when :issues
      get_collection("issues")
    when :issue_comments
      get_collection("issue_comments")
    when :issue_events
      get_collection("issue_events")
    when :repo_labels
      get_collection("repo_labels")
  end
end
init_db(mongo) click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 172
def init_db(mongo)
  ENTITIES.each {|x| mongo.collection(x.to_s)}

  # Ensure that the necessary indexes exist
  IDXS.each do |k,v|
    col = get_entity(k)
    name = v.join('_1_') + '_1'
    exists = col.index_information.find {|k,v| k == name}

    idx_fields = v.reduce({}){|acc, x| acc.merge({x => 1})}
    if exists.nil?
      col.create_index(idx_fields, :background => true)
      STDERR.puts "Creating index on #{col}(#{v})"
    else
      STDERR.puts "Index on #{col}(#{v}) exists"
    end

  end
end
mongo() click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 143
def mongo
  if @mongo.nil?

    replicas = config(:mongo_replicas)

    @mongo = if replicas.nil?
               Mongo::Connection.new(config(:mongo_host),
                                     config(:mongo_port))\
                                .db(config(:mongo_db))\
             else
               repl_arr = replicas.strip.split(/ /).map{|x| "#{x}:#{config(:mongo_port)}"}
               repl_arr << "#{config(:mongo_host)}:#{config(:mongo_port)}"
               Mongo::ReplSetConnection.new(repl_arr, :read => :secondary)\
                                       .db(config(:mongo_db))
             end

    unless config(:mongo_username).nil?
      @mongo.authenticate(config(:mongo_username), config(:mongo_passwd))
    end
    stats = @mongo.stats
    #init_db(@mongo) if stats['collections'] < ENTITIES.size + 2
    #init_db(@mongo) if stats['indexes'] < IDXS.keys.size + ENTITIES.size

    @mongo
  else
    @mongo
  end
end
rescue_connection_failure(max_retries=60) { || ... } click to toggle source
# File lib/ghtorrent/adapters/mongo_persister.rb, line 192
def rescue_connection_failure(max_retries=60)
  retries = 0
  begin
    yield
  rescue Mongo::ConnectionFailure => ex
    retries += 1
    raise ex if retries > max_retries
    sleep(0.5)
    @mongo.refresh if @mongo.class == Mongo::ReplSetConnection
    retry
  end
end