class Redis::Dump

Constants

VALID_TYPES

Attributes

chunk_size[RW]
debug[RW]
encoder[RW]
host[RW]
parser[RW]
password[RW]
port[RW]
safe[RW]
with_base64[RW]
with_optimizations[RW]
dbs[RW]
redis_connections[R]
uri[RW]

Public Class Methods

check_utf8=(check) click to toggle source
# File lib/redis/dump.rb, line 34
def check_utf8=(check)
  if check == false
    @parser = Yajl::Parser.new(:check_utf8 => false)
  end
end
ld(msg) click to toggle source
# File lib/redis/dump.rb, line 28
def ld(msg)
  STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg] if debug
end
le(msg) click to toggle source
# File lib/redis/dump.rb, line 25
def le(msg)
  STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg]
end
memory_usage() click to toggle source
# File lib/redis/dump.rb, line 31
def memory_usage
  `ps -o rss= -p #{Process.pid}`.to_i # in kb
end
new(dbs=nil, uri="redis:// click to toggle source
# File lib/redis/dump.rb, line 42
def initialize(dbs=nil, uri="redis://#{Redis::Dump.host}:#{Redis::Dump.port}")
  @redis_connections = {}
  @uri = uri
  unless dbs.nil?
    @dbs = Range === dbs ? dbs : (dbs..dbs)
    @dbs = (@dbs.first.to_i..@dbs.last.to_i) # enforce integers
    @dbs.to_a.each { |db| redis(db) } # open_all_connections
  end
end

Public Instance Methods

connect(this_uri) click to toggle source
# File lib/redis/dump.rb, line 54
def connect(this_uri)
  self.class.ld 'CONNECT: ' << this_uri
  opts = {
    :url => this_uri
  }
  opts[:password] = Redis::Dump.password if Redis::Dump.password
  Redis.new **opts
end
dump(filter=nil) { |output_buffer| ... } click to toggle source
# File lib/redis/dump.rb, line 69
def dump filter=nil
  filter ||= '*'
  entries = []
  each_database do |redis|
    chunk_entries = []
    dump_keys = redis.keys(filter)
    dump_keys_size = dump_keys.size
    Redis::Dump.ld "Memory after loading keys: #{Redis::Dump.memory_usage}kb"
    dump_keys.each_with_index do |key,idx|
      entry, idxplus = key, idx+1
      if block_given?
        chunk_entries << entry
        process_chunk idx, dump_keys_size do |count|
          Redis::Dump.ld " dumping #{chunk_entries.size} (#{count}) from #{redis.connection[:id]}"
          output_buffer = []
          chunk_entries = chunk_entries.select do |key|
            type = Redis::Dump.type(redis, key)
            if self.class.with_optimizations && type == 'string'
              true
            else
              output_buffer.push self.class.encoder.encode(Redis::Dump.dump(redis, key, type))
              false
            end
          end
          unless output_buffer.empty?
            yield output_buffer
          end
          unless chunk_entries.empty?
            yield Redis::Dump.dump_strings(redis, chunk_entries) { |obj| self.class.encoder.encode(obj) }
          end
          output_buffer.clear
          chunk_entries.clear
        end
      else
        entries << self.class.encoder.encode(Redis::Dump.dump(redis, entry))
      end
    end
  end
  entries
end
each_database() { |redis_connections| ... } click to toggle source
# File lib/redis/dump.rb, line 63
def each_database
  @redis_connections.keys.sort.each do |redis_uri|
    yield redis_connections[redis_uri]
  end
end
load(string_or_stream, &each_record) click to toggle source
# File lib/redis/dump.rb, line 143
def load(string_or_stream, &each_record)
  count = 0
  Redis::Dump.ld " LOAD SOURCE: #{string_or_stream}"
  Redis::Dump.parser.parse string_or_stream do |obj|
    unless @dbs.member?(obj["db"].to_i)
      Redis::Dump.ld "db out of range: #{obj["db"]}"
      next
    end
    this_redis = redis(obj["db"])
    Redis::Dump.ld "load[#{this_redis.hash}, #{obj}]"
    if each_record.nil?
      if Redis::Dump.safe && this_redis.exists(obj['key'])
        Redis::Dump.ld " record exists (no change)"
        next
      end
      begin
        val, type = obj['value'], obj['type']
        if Redis::Dump.with_base64 && type === 'string'
          val = Base64.decode64 val
        end
        ret = Redis::Dump.set_value this_redis, obj['key'], type, val, obj['ttl']
      rescue => ex
        Redis::Dump.le '(key: %s) %s' % [obj['key'], ex.message]
      end
    else
      each_record.call obj
    end
    count += 1
  end
  count
end
redis(db) click to toggle source
# File lib/redis/dump.rb, line 51
def redis(db)
  redis_connections["#{uri}/#{db}"] ||= connect("#{uri}/#{db}")
end
report(filter='*') click to toggle source
# File lib/redis/dump.rb, line 116
def report filter='*'
  values = []
  total_size, dbs = 0, {}
  each_database do |redis|
    chunk_entries = []
    dump_keys = redis.keys(filter)
    dump_keys_size = dump_keys.size
    dump_keys.each_with_index do |key,idx|
      entry, idxplus = Redis::Dump.report(redis, key), idx+1
      chunk_entries << entry
      process_chunk idx, dump_keys_size do |count|
        Redis::Dump.ld " reporting on #{chunk_entries.size} (#{idxplus}) from #{redis.connection[:id]}"
        chunk_entries.each do |e|
          puts record if obj.global.verbose >= 1
          dbs[e['db']] ||= 0
          dbs[e['db']] += e['size']
          total_size += e['size']
        end
        chunk_entries.clear
      end
    end
  end
  puts dbs.keys.sort.collect { |db| "  db#{db}: #{dbs[db].to_bytes}" }
  puts "total: #{total_size.to_bytes}"
  values
end

Private Instance Methods

process_chunk(idx, total_size) { |idxplus| ... } click to toggle source
# File lib/redis/dump.rb, line 110
def process_chunk idx, total_size
  idxplus = idx+1
  yield idxplus if (idxplus % self.class.chunk_size).zero? || idxplus >= total_size
end