class Kitchen::Transport::Sftpgz::Connection

Public Class Methods

new(config, options, &block) click to toggle source
Calls superclass method
# File lib/kitchen/transport/sftpgz.rb, line 55
def initialize(config, options, &block)
  @config = config
  super(options, &block)
end

Public Instance Methods

close() click to toggle source

Wrap Ssh::Connection#close to also shut down the SFTP connection.

Calls superclass method
# File lib/kitchen/transport/sftpgz.rb, line 61
def close
  if @sftp_session
    logger.debug("[SFTP] closing connection to #{self}")
    begin
      sftp_session.close_channel
    rescue Net::SSH::Disconnect
      # Welp, we tried.
    rescue IOError
      # Can happen with net-ssh 4.x, no idea why.
      # See https://github.com/net-ssh/net-ssh/pull/493
    end
  end
ensure
  @sftp_session = nil
  # Make sure we can turn down the session even if closing the channels
  # fails in the middle because of a remote disconnect.
  saved_session = @session
  begin
    super
  rescue Net::SSH::Disconnect
    # Boooo zlib warnings.
    saved_session.transport.close if saved_session
  end
end
upload(locals, remote) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 86
def upload(locals, remote)
  Array(locals).each do |local|
    full_remote = File.join(remote, File.basename(local))
    options = {
        recursive: File.directory?(local),
        purge: File.basename(local) != 'cache',
    }
    recursive = File.directory?(local)
    time = Benchmark.realtime do
      sftp_upload!(local, full_remote, options)
    end
    logger.info("[SFTP] Time taken to upload #{local} to #{self}:#{full_remote}: %.2f sec" % time)
  end
end

Private Instance Methods

add_xfer(xfer) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 259
def add_xfer(xfer)
  sftp_xfers << xfer
  sftp_loop
end
copy_checksums_script!() click to toggle source

Upload the checksum script if needed.

@return [void]

# File lib/kitchen/transport/sftpgz.rb, line 203
def copy_checksums_script!
  # Fast path because upload itself is called multiple times.
  return if @checksums_copied
  # Only try to transfer the script if it isn't present. a stat takes about
  # 1/3rd the time of the transfer, so worst case here is still okay.
  sftp_session.upload!(CHECKSUMS_PATH, CHECKSUMS_REMOTE_PATH) unless safe_stat(CHECKSUMS_REMOTE_PATH)
  @checksums_copied = true
end
execute_with_exit_code(command) click to toggle source

Bug fix for session.loop never terminating if there is an SFTP conn active since as far as it is concerned there is still active stuff. This function is Copyright Fletcher Nichol Tracked in github.com/test-kitchen/test-kitchen/pull/724

# File lib/kitchen/transport/sftpgz.rb, line 149
def execute_with_exit_code(command)
  exit_code = nil
  closed = false
  session.open_channel do |channel|

    channel.request_pty

    channel.exec(command) do |_ch, _success|

      channel.on_data do |_ch, data|
        logger << data
      end

      channel.on_extended_data do |_ch, _type, data|
        logger << data
      end

      channel.on_request("exit-status") do |_ch, data|
        exit_code = data.read_long
      end

      channel.on_close do |ch| # This block is new.
        closed = true
      end
    end
  end
  session.loop { exit_code.nil? && !closed } # THERE IS A CHANGE ON THIS LINE, PAY ATTENTION!!!!!!
  exit_code
end
files_to_upload(checksums, local, recursive) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 212
def files_to_upload(checksums, local, recursive)
  glob_path = if recursive
                File.join(local, '**', '*')
              else
                local
              end
  pending = []
  Dir.glob(glob_path, File::FNM_PATHNAME | File::FNM_DOTMATCH).each do |path|
    next unless File.file?(path)
    rel_path = path[local.length..-1]
    remote_hash = checksums.delete(rel_path)
    pending << rel_path unless remote_hash && remote_hash == Digest::SHA1.file(path).hexdigest
  end
  pending
end
gzip(tarfile, output_file) click to toggle source

gzips the underlying string in the given StringIO, returning a new StringIO representing the compressed file.

# File lib/kitchen/transport/sftpgz.rb, line 295
def gzip(tarfile, output_file)
  z = Zlib::GzipWriter.open(output_file)
  z.write tarfile.read
  z.close
end
purge_files(checksums, remote) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 245
def purge_files(checksums, remote)
  checksums.each do |key, value|
    # Check if the file was uploaded in #upload_file.
    if value != true
      logger.debug("[SFTP] Removing #{remote}#{key}")
      add_xfer(sftp_session.remove("#{remote}#{key}"))
    end
  end
end
safe_stat(path) click to toggle source

Return if the path exists (because net::sftp uses exceptions for that and it makes code gross) and also raise an exception if the path is a symlink.

@param path [String] Remote path to check. @return [Boolean]

# File lib/kitchen/transport/sftpgz.rb, line 192
def safe_stat(path)
  stat = sftp_session.lstat!(path)
  raise "#{path} is a symlink, possible security threat, bailing out" if stat.symlink?
  true
rescue Net::SFTP::StatusException
  false
end
sftp_loop(n_xfers=MAX_TRANSFERS) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 264
def sftp_loop(n_xfers=MAX_TRANSFERS)
  sftp_session.loop do
    sftp_xfers.delete_if {|x| !(x.is_a?(Net::SFTP::Request) ? x.pending? : x.active?) } # Purge any completed operations, which has two different APIs for some reason
    sftp_xfers.length > n_xfers # Run until we have fewer than max
  end
end
sftp_session() click to toggle source

Create the SFTP session and block until it is ready.

@return [Net::SFTP::Session]

# File lib/kitchen/transport/sftpgz.rb, line 182
def sftp_session
  @sftp_session ||= session.sftp
end
sftp_upload!(local, remote, recursive: true, purge: true) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 103
def sftp_upload!(local, remote, recursive: true, purge: true)
  # Fast path check, if the remote path doesn't exist at all we just run a direct transfer
  unless safe_stat(remote)
    logger.debug("[SFTP] Fast path upload from #{local} to #{remote}")
    sftp_session.mkdir!(remote) if recursive
    gzip_uploaded = false
    if File.directory?(local)
      logger.debug("[SFTP] Attempting to upload gzip of folder")
      # tar.gz folder
      temp_file_name = 'xfer_tmp.tar.gz'
      gzipped_file_path = File.join(local, temp_file_name)
      gzipped_data = gzip(tar(local), gzipped_file_path)
      # Upload tar.gz to remote
      remote_path = "#{remote}/#{temp_file_name}"
      sftp_session.upload!(gzipped_file_path, remote_path, requests: MAX_TRANSFERS)
      # Unzip tar.gz @ remote
      exit_code = execute_with_exit_code("cd #{remote} && tar -zxvf #{temp_file_name}")
      # Cleanup
      sftp_session.remove(remote_path)
      # Validate success
      gzip_uploaded = exit_code == 0
    end
    if not gzip_uploaded
      sftp_session.upload!(local, remote, requests: MAX_TRANSFERS)
    end
    return
  end
  # Get checksums for existing files on the remote side.
  logger.debug("[SFTP] Slow path upload from #{local} to #{remote}")
  copy_checksums_script!
  checksum_cmd = "#{@config[:ruby_path]} #{CHECKSUMS_REMOTE_PATH} #{remote}"
  logger.debug("[SFTP] Running #{checksum_cmd}")
  checksums = JSON.parse(session.exec!(checksum_cmd))
  # Sync files that have changed.
  files_to_upload(checksums, local, recursive).each do |rel_path|
    upload_file(checksums, local, remote, rel_path)
  end
  purge_files(checksums, remote) if purge
  # Wait until all xfers are complete.
  sftp_loop(0)
end
sftp_xfers() click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 255
def sftp_xfers
  @sftp_xfers ||= []
end
tar(path) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 271
def tar(path)
  tarfile = StringIO.new("")
  Gem::Package::TarWriter.new(tarfile) do |tar|
    Dir[File.join(path, "**/*")].each do |file|
      mode = File.stat(file).mode
      relative_file = file.sub /^#{Regexp::escape path}\/?/, ''

      if File.directory?(file)
        tar.mkdir relative_file, mode
      else
        tar.add_file relative_file, mode do |tf|
          File.open(file, "rb") { |f| tf.write f.read }
        end
      end
    end
  end

  tarfile.rewind
  tarfile
end
upload_file(checksums, local, remote, rel_path) click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 228
def upload_file(checksums, local, remote, rel_path)
  parts = rel_path.split('/')
  parts.pop # Drop the filename since we are only checking dirs
  parts_to_check = []
  until parts.empty?
    parts_to_check << parts.shift
    path_to_check = parts_to_check.join('/')
    unless checksums[path_to_check]
      logger.debug("[SFTP] Creating directory #{remote}#{path_to_check}")
      add_xfer(sftp_session.mkdir("#{remote}#{path_to_check}"))
      checksums[path_to_check] = true
    end
  end
  logger.debug("[SFTP] Uploading #{local}#{rel_path} to #{remote}#{rel_path}")
  add_xfer(sftp_session.upload("#{local}#{rel_path}", "#{remote}#{rel_path}"))
end