class S3TarBackup::Main

Constants

UPLOAD_TRIES

Public Instance Methods

absolute_path_from_config_file(config, path) click to toggle source
# File lib/s3_tar_backup.rb, line 81
def absolute_path_from_config_file(config, path)
  File.expand_path(File.join(File.expand_path(File.dirname(config.file_path)), path))
end
backup(config, backup, verbose=false) click to toggle source
# File lib/s3_tar_backup.rb, line 215
def backup(config, backup, verbose=false)
  FileUtils.rm(backup.tmp_snar_path) if File.exists?(backup.tmp_snar_path)
  FileUtils.cp(backup.snar_path, backup.tmp_snar_path) if backup.snar_exists?
  exec(backup.backup_cmd(verbose))
  puts "Uploading #{config[:backend].prefix}/#{File.basename(backup.archive)} (#{bytes_to_human(File.size(backup.archive))})"
  upload(config[:backend], backup.archive, File.basename(backup.archive), true)
  FileUtils.mv(backup.tmp_snar_path, backup.snar_path, :force => true)
  puts "Uploading snar (#{bytes_to_human(File.size(backup.snar_path))})"
  upload(config[:backend], backup.snar_path, File.basename(backup.snar), false)
end
backup_full(config, verbose=false) click to toggle source
# File lib/s3_tar_backup.rb, line 207
def backup_full(config, verbose=false)
  puts "Starting new full backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:encryption])
  # Nuke the snar file -- forces a full backup
  File.delete(backup.snar_path) if File.exists?(backup.snar_path)
  backup(config, backup, verbose)
end
backup_incr(config, verbose=false) click to toggle source

Config should have the keys backup_dir, name, soruces, exclude

# File lib/s3_tar_backup.rb, line 189
def backup_incr(config, verbose=false)
  puts "Starting new incremental backup"
  backup = Backup.new(config[:backup_dir], config[:name], config[:sources], config[:exclude], config[:compression], config[:encryption])

  # Try and get hold of the snar file
  unless backup.snar_exists?
    puts "Failed to find snar file. Attempting to download..."
    if config[:backend].item_exists?(backup.snar)
      puts "Found file on S3. Downloading"
      config[:backend].download_item(backup.snar, backup.snar_path)
    else
      puts "Failed to download snar file. Defaulting to full backup"
    end
  end

  backup(config, backup, verbose)
end
bytes_to_human(n) click to toggle source
# File lib/s3_tar_backup.rb, line 335
def bytes_to_human(n)
  count = 0
  while n >= 1014 && count < 4
    n /= 1024.0
    count += 1
  end
  fmt = (count == 0) ? '%i' : '%.2f'
  format(fmt, n) << %w(B KB MB GB TB)[count]
end
create_backend(config, dest_prefix) click to toggle source
# File lib/s3_tar_backup.rb, line 85
def create_backend(config, dest_prefix)
    if dest_prefix.start_with?('file://')
      Backend::FileBackend.new(dest_prefix['file://'.length..-1])
    elsif dest_prefix.start_with?('/')
      Backend::FileBackend.new(dest_prefix)
    else
      Backend::S3Backend.new(
        ENV['AWS_ACCESS_KEY_ID'] || config['settings.aws_access_key_id'],
        ENV['AWS_SECRET_ACCESS_KEY'] || config['settings.aws_secret_access_key'],
        config.get('settings.aws_region', false),
        (config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]).sub(%r{^s3://}, '')
      )
    end
end
exec(cmd) click to toggle source
# File lib/s3_tar_backup.rb, line 345
def exec(cmd)
  puts "Executing: #{cmd}"
  result = system(cmd)
  unless result
    raise "Unable to run command. See above output for clues."
  end
end
full_required?(interval_str, objects) click to toggle source
# File lib/s3_tar_backup.rb, line 330
def full_required?(interval_str, objects)
  time = parse_interval(interval_str)
  objects.select{ |o| o[:type] == :full && o[:date] > time }.empty?
end
gen_backup_config(profile, config) click to toggle source
# File lib/s3_tar_backup.rb, line 100
def gen_backup_config(profile, config)
  top_gpg_key = config.get('settings.gpg_key', false)
  profile_gpg_key = config.get("profile.#{profile}.gpg_key", false)
  top_password_file = config.get('settings.password_file', false)
  profile_password_file = config.get("profile.#{profile}.password_file", false)
  raise "Cannot specify gpg_key and password_file together at the top level" if top_gpg_key && top_password_file 
  raise "Cannot specify both gpg_key and password_file for profile #{profile}" if profile_gpg_key && profile_password_file

  encryption = nil
  if profile_password_file
    encryption = profile_password_file.empty? ? nil : { :type => :password_file, :password_file => absolute_path_from_config_file(config, profile_password_file) }
  elsif profile_gpg_key
    encryption = profile_gpg_key.empty? ? nil : { :type => :gpg_key, :gpg_key => profile_gpg_key }
  elsif top_password_file
    encryption = top_password_file.empty? ? nil : { :type => :password_file, :password_file => absolute_path_from_config_file(config, top_password_file) }
  elsif top_gpg_key
    encryption = top_gpg_key.empty? ? nil : { :type => :gpg_key, :gpg_key => top_gpg_key }
  end

  backup_config = {
    :backup_dir => config.get("profile.#{profile}.backup_dir", false) || config.get('settings.backup_dir', '~/.s3-tar-backup/tmp'),
    :name => profile,
    :encryption => encryption,
    :password_file => profile_password_file || top_password_file || '',
    :sources => [*config.get("profile.#{profile}.source", [])] + [*config.get("settings.source", [])],
    :exclude => [*config.get("profile.#{profile}.exclude", [])] + [*config.get("settings.exclude", [])],
    :pre_backup => [*config.get("profile.#{profile}.pre-backup", [])] + [*config.get('settings.pre-backup', [])],
    :post_backup => [*config.get("profile.#{profile}.post-backup", [])] + [*config.get('settings.post-backup', [])],
    :full_if_older_than => config.get("profile.#{profile}.full_if_older_than", false) || config['settings.full_if_older_than'],
    :remove_older_than => config.get("profile.#{profile}.remove_older_than", false) || config.get('settings.remove_older_than', false),
    :remove_all_but_n_full => config.get("profile.#{profile}.remove_all_but_n_full", false) || config.get('settings.remove_all_but_n_full', false),
    :compression => (config.get("profile.#{profile}.compression", false) || config.get('settings.compression', 'none')).to_sym,
    :always_full => config.get('settings.always_full', false) || config.get("profile.#{profile}.always_full", false),
    :backend => create_backend(config,config.get("profile.#{profile}.dest", false) || config['settings.dest']),
  }
  backup_config
end
get_objects(config, profile) click to toggle source
# File lib/s3_tar_backup.rb, line 311
def get_objects(config, profile)
  objects = config[:backend].list_items.map do |object|
    Backup.parse_object(object, profile)
  end
  objects.compact.sort_by{ |o| o[:date] }
end
parse_interval(interval_str) click to toggle source
# File lib/s3_tar_backup.rb, line 318
def parse_interval(interval_str)
  time = Time.now
  time -= $1.to_i if interval_str =~ /(\d+)s/
  time -= $1.to_i*60 if interval_str =~ /(\d+)m/
  time -= $1.to_i*3600 if interval_str =~ /(\d+)h/
  time -= $1.to_i*86400 if interval_str =~ /(\d+)D/
  time -= $1.to_i*604800 if interval_str =~ /(\d+)W/
  time -= $1.to_i*2592000 if interval_str =~ /(\d+)M/
  time -= $1.to_i*31536000 if interval_str =~ /(\d+)Y/
  time
end
perform_backup(opts, prev_backups, backup_config) click to toggle source
# File lib/s3_tar_backup.rb, line 138
def perform_backup(opts, prev_backups, backup_config)
  puts "===== Backing up profile #{backup_config[:name]} ====="
  backup_config[:pre_backup].each_with_index do |cmd, i|
    puts "Executing pre-backup hook #{i+1}"
    exec(cmd)
  end
  full_required = full_required?(backup_config[:full_if_older_than], prev_backups)
  puts "Last full backup is too old. Forcing a full backup" if full_required && !opts[:full] && backup_config[:always_full]
  if full_required || opts[:full] || backup_config[:always_full]
    backup_full(backup_config, opts[:verbose])
  else
    backup_incr(backup_config, opts[:verbose])
  end
  backup_config[:post_backup].each_with_index do |cmd, i|
    puts "Executing post-backup hook #{i+1}"
    exec(cmd)
  end
end
perform_cleanup(prev_backups, backup_config) click to toggle source
# File lib/s3_tar_backup.rb, line 157
def perform_cleanup(prev_backups, backup_config)
  puts "===== Cleaning up profile #{backup_config[:name]} ====="
  remove = []
  if age_str = backup_config[:remove_older_than]
    age = parse_interval(age_str)
    remove = prev_backups.select{ |o| o[:date] < age }
    # Don't want to delete anything before the last full backup
    unless remove.empty?
      kept = remove.slice!(remove.rindex{ |o| o[:type] == :full }..-1).count
      puts "Keeping #{kept} old backups as part of a chain" if kept > 1
    end
  elsif keep_n = backup_config[:remove_all_but_n_full]
    keep_n = keep_n.to_i
    # Get the date of the last full backup to keep
    if last_full_to_keep = prev_backups.select{ |o| o[:type] == :full }[-keep_n]
      # If there is a last full one...
      remove = prev_backups.select{ |o| o[:date] < last_full_to_keep[:date] }
    end
  end

  if remove.empty?
    puts "Nothing to do"
  else
    puts "Removing #{remove.count} old backup files"
  end
  remove.each do |object|
    backup_config[:backend].remove_item(object[:name])
  end
end
perform_list_backups(prev_backups, backup_config) click to toggle source
# File lib/s3_tar_backup.rb, line 276
def perform_list_backups(prev_backups, backup_config)
  # prev_backups alreays contains just the files for the current profile
  puts "===== Backups list for #{backup_config[:name]} ====="
  puts "Type: N:  Date:#{' '*18}Size:       Chain Size:   Compression:   Encryption:\n\n"
  prev_type = ''
  total_size = 0
  chain_length = 0
  chain_cum_size = 0
  prev_backups.each do |object|
    type = object[:type] == prev_type && object[:type] == :incr ? " -- " : object[:type].to_s.capitalize
    prev_type = object[:type]
    chain_length += 1
    chain_length = 0 if object[:type] == :full
    chain_cum_size = 0 if object[:type] == :full
    chain_cum_size += object[:size]

    chain_length_str = (chain_length == 0 ? '' : chain_length.to_s).ljust(3)
    chain_cum_size_str = (object[:type] == :full ? '' : bytes_to_human(chain_cum_size)).ljust(8)
    encryption = case object[:encryption]
    when :gpg_key
      'Key'
    when :password_file
      'Password'
    else
      'None'
    end
    puts "#{type}  #{chain_length_str} #{object[:date].strftime('%F %T')}    #{bytes_to_human(object[:size]).ljust(8)}    " \
      "#{chain_cum_size_str}      #{object[:compression].to_s.ljust(12)}   #{encryption}"
    total_size += object[:size]
  end
  puts "\n"
  puts "Total size: #{bytes_to_human(total_size)}"
  puts "\n"
end
perform_restore(opts, prev_backups, backup_config) click to toggle source
# File lib/s3_tar_backup.rb, line 243
def perform_restore(opts, prev_backups, backup_config)
  puts "===== Restoring profile #{backup_config[:name]} ====="
  # If restore date given, parse
  if opts[:restore_date_given]
    m = opts[:restore_date].match(/(\d\d\d\d)(\d\d)(\d\d)?(\d\d)?(\d\d)?(\d\d)?/)
    raise "Unknown date format in --restore-to" if m.nil?
    restore_to = Time.new(*m[1..-1].map{ |s| s.to_i if s })
  else
    restore_to = Time.now
  end

  # Find the index of the first backup, incremental or full, before that date
  restore_end_index = prev_backups.rindex{ |o| o[:date] < restore_to }
  raise "Failed to find a backup for that date" unless restore_end_index

  # Find the first full backup before that one
  restore_start_index = prev_backups[0..restore_end_index].rindex{ |o| o[:type] == :full }

  restore_dir = opts[:restore].chomp('/') << '/'

  Dir.mkdir(restore_dir) unless Dir.exists?(restore_dir)
  raise "Destination dir is not a directory" unless File.directory?(restore_dir)

  prev_backups[restore_start_index..restore_end_index].each do |object|
    puts "Fetching #{backup_config[:backend].prefix}/#{object[:name]} (#{bytes_to_human(object[:size])})"
    dl_file = "#{backup_config[:backup_dir]}/#{object[:name]}"
    backup_config[:backend].download_item(object[:name], dl_file)
    puts "Extracting..."
    exec(Backup.restore_cmd(restore_dir, dl_file, opts[:verbose], opts[:password_file] || backup_config[:password_file]))
    File.delete(dl_file)
  end
end
run() click to toggle source
# File lib/s3_tar_backup.rb, line 12
def run
  opts = Trollop::options do
    version VERSION
    banner "Backs up files to, and restores files from, Amazon's S3 storage, using tar incremental backups\n\n" \
      "Usage:\ns3-tar-backup -c config.ini [-p profile] --backup [--full] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --cleanup [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --restore restore_dir\n\t[--restore_date date] [-v]\n" \
      "s3-tar-backup -c config.ini [-p profile] --backup-config [--verbose]\n" \
      "s3-tar-backup -c config.ini [-p profile] --list-backups\n\n" \
      "Option details:\n"
    opt :config, "Configuration file", :short => 'c', :type => :string
    opt :backup, "Make an incremental backup"
    opt :full, "Make the backup a full backup"
    opt :profile, "The backup profile(s) to use (default all)", :short => 'p', :type => :strings
    opt :cleanup, "Clean up old backups"
    opt :restore, "Restore a backup to the specified dir", :type => :string
    opt :restore_date, "Restore a backup from the specified date. Format YYYYMM[DD[hh[mm[ss]]]]", :type => :string
    opt :backup_config, "Backs up the specified configuration file"
    opt :list_backups, "List the stored backup info for one or more profiles"
    opt :password_file, "Override the password file used to decrypt backups", :type => :string
    opt :verbose, "Show verbose output", :short => 'v'
    conflicts :backup, :cleanup, :restore, :backup_config, :list_backups
  end


  Trollop::die "--full requires --backup" if opts[:full] && !opts[:backup]
  Trollop::die "--restore-date requires --restore" if opts[:restore_date_given] && !opts[:restore_given]
  Trollop::die "--password-file requires --restore" if opts[:password_file_given] && !opts[:restore_given]
  unless opts[:backup] || opts[:cleanup] || opts[:restore_given] || opts[:backup_config] || opts[:list_backups]
    Trollop::die "Need one of --backup, --cleanup, --restore, --backup-config, --list-backups"
  end

  config_file = opts[:config] || '~/.s3-tar-backup/config.ini'

  begin
    raise "Config file #{config_file} not found.#{opts[:config] ? '' : ' You can specify a config file to use with --config'}" unless File.exists?(config_file)
    config = IniParser.new(config_file).load
    profiles = opts[:profile] || config.find_sections(/^profile\./).keys.map{ |k| k.to_s.split('.', 2)[1] }

    # This is a bit of a special case
    if opts[:backup_config]
      dest = config.get('settings.dest', false)
      raise "You must specify a single profile (used to determine the location to back up to) " \
        "if backing up config and dest key is not in [settings]" if !dest && profiles.count != 1
      dest ||= config["profile.#{profiles[0]}.dest"]
      puts "===== Backing up config file #{config_file} ====="
      prefix = config.get('settings.dest', false) || config["profile.#{profiles[0]}.dest"]
      puts "Uploading #{config_file} to #{prefix}/#{File.basename(config_file)}"
      backend = create_backend(config, prefix)
      upload(backend, config_file, File.basename(config_file), false)
      return
    end

    profiles.dup.each do |profile|
      raise "No such profile: #{profile}" unless config.has_section?("profile.#{profile}")
      opts[:profile] = profile
      backup_config = gen_backup_config(opts[:profile], config)
      prev_backups = get_objects(backup_config, opts[:profile])
      perform_backup(opts, prev_backups, backup_config) if opts[:backup]
      perform_cleanup(prev_backups, backup_config) if opts[:backup] || opts[:cleanup]
      perform_restore(opts, prev_backups, backup_config) if opts[:restore_given]
      perform_list_backups(prev_backups, backup_config) if opts[:list_backups]
    end
  rescue Exception => e
    raise e
    Trollop::die e.to_s
  end
end
upload(backend, source, dest_name, remove_original) click to toggle source
# File lib/s3_tar_backup.rb, line 226
def upload(backend, source, dest_name, remove_original)
  tries = 0
  begin
    backend.upload_item(dest_name, source, remove_original)
  rescue Backend::UploadItemFailedError => e
    tries += 1
    if tries <= UPLOAD_TRIES
      puts "Upload Exception: #{e}"
      puts "Retrying #{tries}/#{UPLOAD_TRIES}..."
      retry
    else
      raise e
    end
  end
  puts "Succeeded" if tries > 0
end