class DbCharmer::Sharding::Method::DbBlockGroupMap

Attributes

block_size[RW]

Sharding keys block size

connection[RW]

Mapping db connection

connection_name[RW]

Mapping db connection

groups_table[RW]

Tablegroups table name

map_table[RW]

Mapping table name

name[RW]
shards_table[RW]

Shards table name

Public Class Methods

new(config) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 48
def initialize(config)
  @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!")
  @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true)
  @block_size = (config[:block_size] || 10000).to_i

  @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!")
  @groups_table = config[:groups_table] or raise(ArgumentError, "Missing required :groups_table parameter!")
  @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!")

  # Local caches
  @shard_info_cache = {}
  @group_info_cache = {}

  @blocks_cache = Rails.cache
  @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:"
end

Public Instance Methods

allocate_new_block_for_key(key) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 151
        def allocate_new_block_for_key(key)
          # Can't find any groups to use for blocks allocation!
          return nil unless group = least_loaded_group

          # Figure out block limits
          start_id = block_start_for_key(key)
          end_id = block_end_for_key(key)

          # Try to insert a new mapping (ignore duplicate key errors)
          sql = <<-SQL
            INSERT IGNORE INTO #{map_table}
                           SET start_id = #{start_id},
                               end_id = #{end_id},
                               group_id = #{group.id},
                               block_size = #{block_size},
                               created_at = NOW(),
                               updated_at = NOW()
          SQL
          connection.execute(sql, "Allocate new block")

          # Increment the blocks counter on the shard
          Group.update_counters(group.id, :blocks_count => +1)

          # Retry block search after creation
          block_for_key(key)
        end
block_end_for_key(key) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 192
def block_end_for_key(key)
  block_size.to_i + block_start_for_key(key)
end
block_for_key(key, cache = true) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 83
def block_for_key(key, cache = true)
  # Cleanup the cache if asked to
  key_range = [ block_start_for_key(key), block_end_for_key(key) ]
  block_cache_key = "%d-%d" % key_range

  if cache
    cached_block = get_cached_block(block_cache_key)
    return cached_block if cached_block
  end

  # Fetch cached value or load from db
  block = begin
    sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1"
    connection.select_one(sql, 'Find a shard block')
  end

  set_cached_block(block_cache_key, block)

  return block
end
block_start_for_key(key) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 188
def block_start_for_key(key)
  block_size.to_i * (key.to_i / block_size.to_i)
end
create_shard(params) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 222
def create_shard(params)
  params = params.symbolize_keys
  [ :db_host, :db_port, :db_user, :db_pass, :db_name_prefix ].each do |arg|
    raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg]
  end

  # Prepare model
  prepare_shard_models

  # Create the record
  Shard.create! do |shard|
    shard.db_host = params[:db_host]
    shard.db_port = params[:db_port]
    shard.db_user = params[:db_user]
    shard.db_pass = params[:db_pass]
    shard.db_name_prefix = params[:db_name_prefix]
  end
end
get_cached_block(block_cache_key) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 105
def get_cached_block(block_cache_key)
  @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}")
end
group_database_name(shard, group_id) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 217
def group_database_name(shard, group_id)
  "%s_%05d" % [ shard.db_name_prefix, group_id ]
end
group_info_by_id(group_id, cache = true) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 115
def group_info_by_id(group_id, cache = true)
  # Cleanup the cache if asked to
  @group_info_cache[group_id] = nil unless cache

  # Either load from cache or from db
  @group_info_cache[group_id] ||= begin
    prepare_shard_models
    Group.find_by_id(group_id)
  end
end
least_loaded_group() click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 178
def least_loaded_group
  prepare_shard_models

  # Select group
  group = Group.first(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC')
  raise "Can't find any tablegroups to use for blocks allocation!" unless group
  return group
end
prepare_shard_models() click to toggle source

Prepare model for working with our shards table

# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 250
def prepare_shard_models
  Shard.set_table_name(shards_table)
  Shard.switch_connection_to(connection)

  Group.set_table_name(groups_table)
  Group.switch_connection_to(connection)
end
set_cached_block(block_cache_key, block) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 109
def set_cached_block(block_cache_key, block)
  @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block)
end
shard_connection_config(shard, group_id) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 198
def shard_connection_config(shard, group_id)
  # Format connection name
  shard_name = "db_charmer_db_block_group_map_#{name}_s%d_g%d" % [ shard.id, group_id]

  # Here we get the mapping connection's configuration
  # They do not expose configs so we hack in and get the instance var
  # FIXME: Find a better way, maybe move config method to our ar extenstions
  connection.instance_variable_get(:@config).clone.merge(
    # Name for the connection factory
    :connection_name => shard_name,
    # Connection params
    :host => shard.db_host,
    :port => shard.db_port,
    :username => shard.db_user,
    :password => shard.db_pass,
    :database => group_database_name(shard, group_id)
  )
end
shard_connections() click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 241
def shard_connections
  # Find all groups
  prepare_shard_models
  groups = Group.all(:conditions => { :enabled => true }, :include => :shard)
  # Map them to shards
  groups.map { |group| shard_connection_config(group.shard, group.id) }
end
shard_for_key(key) click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 66
def shard_for_key(key)
  block = block_for_key(key)

  # Auto-allocate new blocks
  block ||= allocate_new_block_for_key(key)
  raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block

  # Load shard
  group_id = block['group_id'].to_i
  shard_info = shard_info_by_group_id(group_id)

  # Get config
  shard_connection_config(shard_info, group_id)
end
shard_info_by_group_id(group_id) click to toggle source

Load shard info using mapping info for a group

# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 139
def shard_info_by_group_id(group_id)
  # Load group
  group_info = group_info_by_id(group_id)
  raise ArgumentError, "Invalid group_id: #{group_id}" unless group_info

  shard_info = shard_info_by_id(group_info.shard_id)
  raise ArgumentError, "Invalid shard_id: #{group_info.shard_id}" unless shard_info

  return shard_info
end
shard_info_by_id(shard_id, cache = true) click to toggle source

Load shard info

# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 127
def shard_info_by_id(shard_id, cache = true)
  # Cleanup the cache if asked to
  @shard_info_cache[shard_id] = nil unless cache

  # Either load from cache or from db
  @shard_info_cache[shard_id] ||= begin
    prepare_shard_models
    Shard.find_by_id(shard_id)
  end
end