class DbCharmer::Sharding::Method::DbBlockGroupMap
Attributes
block_size[RW]
Sharding
keys block size
connection[RW]
Mapping db connection
connection_name[RW]
Mapping db connection
groups_table[RW]
Tablegroups table name
map_table[RW]
Mapping table name
name[RW]
shards_table[RW]
Shards table name
Public Class Methods
new(config)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 48 def initialize(config) @name = config[:name] or raise(ArgumentError, "Missing required :name parameter!") @connection = DbCharmer::ConnectionFactory.connect(config[:connection], true) @block_size = (config[:block_size] || 10000).to_i @map_table = config[:map_table] or raise(ArgumentError, "Missing required :map_table parameter!") @groups_table = config[:groups_table] or raise(ArgumentError, "Missing required :groups_table parameter!") @shards_table = config[:shards_table] or raise(ArgumentError, "Missing required :shards_table parameter!") # Local caches @shard_info_cache = {} @group_info_cache = {} @blocks_cache = Rails.cache @blocks_cache_prefix = config[:blocks_cache_prefix] || "#{@name}_block:" end
Public Instance Methods
allocate_new_block_for_key(key)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 151 def allocate_new_block_for_key(key) # Can't find any groups to use for blocks allocation! return nil unless group = least_loaded_group # Figure out block limits start_id = block_start_for_key(key) end_id = block_end_for_key(key) # Try to insert a new mapping (ignore duplicate key errors) sql = <<-SQL INSERT IGNORE INTO #{map_table} SET start_id = #{start_id}, end_id = #{end_id}, group_id = #{group.id}, block_size = #{block_size}, created_at = NOW(), updated_at = NOW() SQL connection.execute(sql, "Allocate new block") # Increment the blocks counter on the shard Group.update_counters(group.id, :blocks_count => +1) # Retry block search after creation block_for_key(key) end
block_end_for_key(key)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 192 def block_end_for_key(key) block_size.to_i + block_start_for_key(key) end
block_for_key(key, cache = true)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 83 def block_for_key(key, cache = true) # Cleanup the cache if asked to key_range = [ block_start_for_key(key), block_end_for_key(key) ] block_cache_key = "%d-%d" % key_range if cache cached_block = get_cached_block(block_cache_key) return cached_block if cached_block end # Fetch cached value or load from db block = begin sql = "SELECT * FROM #{map_table} WHERE start_id = #{key_range.first} AND end_id = #{key_range.last} LIMIT 1" connection.select_one(sql, 'Find a shard block') end set_cached_block(block_cache_key, block) return block end
block_start_for_key(key)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 188 def block_start_for_key(key) block_size.to_i * (key.to_i / block_size.to_i) end
create_shard(params)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 222 def create_shard(params) params = params.symbolize_keys [ :db_host, :db_port, :db_user, :db_pass, :db_name_prefix ].each do |arg| raise ArgumentError, "Missing required parameter: #{arg}" unless params[arg] end # Prepare model prepare_shard_models # Create the record Shard.create! do |shard| shard.db_host = params[:db_host] shard.db_port = params[:db_port] shard.db_user = params[:db_user] shard.db_pass = params[:db_pass] shard.db_name_prefix = params[:db_name_prefix] end end
get_cached_block(block_cache_key)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 105 def get_cached_block(block_cache_key) @blocks_cache.read("#{@blocks_cache_prefix}#{block_cache_key}") end
group_database_name(shard, group_id)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 217 def group_database_name(shard, group_id) "%s_%05d" % [ shard.db_name_prefix, group_id ] end
group_info_by_id(group_id, cache = true)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 115 def group_info_by_id(group_id, cache = true) # Cleanup the cache if asked to @group_info_cache[group_id] = nil unless cache # Either load from cache or from db @group_info_cache[group_id] ||= begin prepare_shard_models Group.find_by_id(group_id) end end
least_loaded_group()
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 178 def least_loaded_group prepare_shard_models # Select group group = Group.first(:conditions => { :enabled => true, :open => true }, :order => 'blocks_count ASC') raise "Can't find any tablegroups to use for blocks allocation!" unless group return group end
prepare_shard_models()
click to toggle source
Prepare model for working with our shards table
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 250 def prepare_shard_models Shard.set_table_name(shards_table) Shard.switch_connection_to(connection) Group.set_table_name(groups_table) Group.switch_connection_to(connection) end
set_cached_block(block_cache_key, block)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 109 def set_cached_block(block_cache_key, block) @blocks_cache.write("#{@blocks_cache_prefix}#{block_cache_key}", block) end
shard_connection_config(shard, group_id)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 198 def shard_connection_config(shard, group_id) # Format connection name shard_name = "db_charmer_db_block_group_map_#{name}_s%d_g%d" % [ shard.id, group_id] # Here we get the mapping connection's configuration # They do not expose configs so we hack in and get the instance var # FIXME: Find a better way, maybe move config method to our ar extenstions connection.instance_variable_get(:@config).clone.merge( # Name for the connection factory :connection_name => shard_name, # Connection params :host => shard.db_host, :port => shard.db_port, :username => shard.db_user, :password => shard.db_pass, :database => group_database_name(shard, group_id) ) end
shard_connections()
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 241 def shard_connections # Find all groups prepare_shard_models groups = Group.all(:conditions => { :enabled => true }, :include => :shard) # Map them to shards groups.map { |group| shard_connection_config(group.shard, group.id) } end
shard_for_key(key)
click to toggle source
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 66 def shard_for_key(key) block = block_for_key(key) # Auto-allocate new blocks block ||= allocate_new_block_for_key(key) raise ArgumentError, "Invalid key value, no shards found for this key and could not create a new block!" unless block # Load shard group_id = block['group_id'].to_i shard_info = shard_info_by_group_id(group_id) # Get config shard_connection_config(shard_info, group_id) end
shard_info_by_group_id(group_id)
click to toggle source
Load shard info using mapping info for a group
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 139 def shard_info_by_group_id(group_id) # Load group group_info = group_info_by_id(group_id) raise ArgumentError, "Invalid group_id: #{group_id}" unless group_info shard_info = shard_info_by_id(group_info.shard_id) raise ArgumentError, "Invalid shard_id: #{group_info.shard_id}" unless shard_info return shard_info end
shard_info_by_id(shard_id, cache = true)
click to toggle source
Load shard info
# File lib/db_charmer/sharding/method/db_block_group_map.rb, line 127 def shard_info_by_id(shard_id, cache = true) # Cleanup the cache if asked to @shard_info_cache[shard_id] = nil unless cache # Either load from cache or from db @shard_info_cache[shard_id] ||= begin prepare_shard_models Shard.find_by_id(shard_id) end end