class Kitchen::Transport::Sftpgz::Connection
Public Class Methods
new(config, options, &block)
click to toggle source
Calls superclass method
# File lib/kitchen/transport/sftpgz.rb, line 55 def initialize(config, options, &block) @config = config super(options, &block) end
Public Instance Methods
close()
click to toggle source
Wrap Ssh::Connection#close to also shut down the SFTP connection.
Calls superclass method
# File lib/kitchen/transport/sftpgz.rb, line 61 def close if @sftp_session logger.debug("[SFTP] closing connection to #{self}") begin sftp_session.close_channel rescue Net::SSH::Disconnect # Welp, we tried. rescue IOError # Can happen with net-ssh 4.x, no idea why. # See https://github.com/net-ssh/net-ssh/pull/493 end end ensure @sftp_session = nil # Make sure we can turn down the session even if closing the channels # fails in the middle because of a remote disconnect. saved_session = @session begin super rescue Net::SSH::Disconnect # Boooo zlib warnings. saved_session.transport.close if saved_session end end
upload(locals, remote)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 86 def upload(locals, remote) Array(locals).each do |local| full_remote = File.join(remote, File.basename(local)) options = { recursive: File.directory?(local), purge: File.basename(local) != 'cache', } recursive = File.directory?(local) time = Benchmark.realtime do sftp_upload!(local, full_remote, options) end logger.info("[SFTP] Time taken to upload #{local} to #{self}:#{full_remote}: %.2f sec" % time) end end
Private Instance Methods
add_xfer(xfer)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 259 def add_xfer(xfer) sftp_xfers << xfer sftp_loop end
copy_checksums_script!()
click to toggle source
Upload the checksum script if needed.
@return [void]
# File lib/kitchen/transport/sftpgz.rb, line 203 def copy_checksums_script! # Fast path because upload itself is called multiple times. return if @checksums_copied # Only try to transfer the script if it isn't present. a stat takes about # 1/3rd the time of the transfer, so worst case here is still okay. sftp_session.upload!(CHECKSUMS_PATH, CHECKSUMS_REMOTE_PATH) unless safe_stat(CHECKSUMS_REMOTE_PATH) @checksums_copied = true end
execute_with_exit_code(command)
click to toggle source
Bug fix for session.loop never terminating if there is an SFTP conn active since as far as it is concerned there is still active stuff. This function is Copyright Fletcher Nichol Tracked in github.com/test-kitchen/test-kitchen/pull/724
# File lib/kitchen/transport/sftpgz.rb, line 149 def execute_with_exit_code(command) exit_code = nil closed = false session.open_channel do |channel| channel.request_pty channel.exec(command) do |_ch, _success| channel.on_data do |_ch, data| logger << data end channel.on_extended_data do |_ch, _type, data| logger << data end channel.on_request("exit-status") do |_ch, data| exit_code = data.read_long end channel.on_close do |ch| # This block is new. closed = true end end end session.loop { exit_code.nil? && !closed } # THERE IS A CHANGE ON THIS LINE, PAY ATTENTION!!!!!! exit_code end
files_to_upload(checksums, local, recursive)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 212 def files_to_upload(checksums, local, recursive) glob_path = if recursive File.join(local, '**', '*') else local end pending = [] Dir.glob(glob_path, File::FNM_PATHNAME | File::FNM_DOTMATCH).each do |path| next unless File.file?(path) rel_path = path[local.length..-1] remote_hash = checksums.delete(rel_path) pending << rel_path unless remote_hash && remote_hash == Digest::SHA1.file(path).hexdigest end pending end
gzip(tarfile, output_file)
click to toggle source
gzips the underlying string in the given StringIO, returning a new StringIO representing the compressed file.
# File lib/kitchen/transport/sftpgz.rb, line 295 def gzip(tarfile, output_file) z = Zlib::GzipWriter.open(output_file) z.write tarfile.read z.close end
purge_files(checksums, remote)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 245 def purge_files(checksums, remote) checksums.each do |key, value| # Check if the file was uploaded in #upload_file. if value != true logger.debug("[SFTP] Removing #{remote}#{key}") add_xfer(sftp_session.remove("#{remote}#{key}")) end end end
safe_stat(path)
click to toggle source
Return if the path exists (because net::sftp uses exceptions for that and it makes code gross) and also raise an exception if the path is a symlink.
@param path [String] Remote path to check. @return [Boolean]
# File lib/kitchen/transport/sftpgz.rb, line 192 def safe_stat(path) stat = sftp_session.lstat!(path) raise "#{path} is a symlink, possible security threat, bailing out" if stat.symlink? true rescue Net::SFTP::StatusException false end
sftp_loop(n_xfers=MAX_TRANSFERS)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 264 def sftp_loop(n_xfers=MAX_TRANSFERS) sftp_session.loop do sftp_xfers.delete_if {|x| !(x.is_a?(Net::SFTP::Request) ? x.pending? : x.active?) } # Purge any completed operations, which has two different APIs for some reason sftp_xfers.length > n_xfers # Run until we have fewer than max end end
sftp_session()
click to toggle source
Create the SFTP session and block until it is ready.
@return [Net::SFTP::Session]
# File lib/kitchen/transport/sftpgz.rb, line 182 def sftp_session @sftp_session ||= session.sftp end
sftp_upload!(local, remote, recursive: true, purge: true)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 103 def sftp_upload!(local, remote, recursive: true, purge: true) # Fast path check, if the remote path doesn't exist at all we just run a direct transfer unless safe_stat(remote) logger.debug("[SFTP] Fast path upload from #{local} to #{remote}") sftp_session.mkdir!(remote) if recursive gzip_uploaded = false if File.directory?(local) logger.debug("[SFTP] Attempting to upload gzip of folder") # tar.gz folder temp_file_name = 'xfer_tmp.tar.gz' gzipped_file_path = File.join(local, temp_file_name) gzipped_data = gzip(tar(local), gzipped_file_path) # Upload tar.gz to remote remote_path = "#{remote}/#{temp_file_name}" sftp_session.upload!(gzipped_file_path, remote_path, requests: MAX_TRANSFERS) # Unzip tar.gz @ remote exit_code = execute_with_exit_code("cd #{remote} && tar -zxvf #{temp_file_name}") # Cleanup sftp_session.remove(remote_path) # Validate success gzip_uploaded = exit_code == 0 end if not gzip_uploaded sftp_session.upload!(local, remote, requests: MAX_TRANSFERS) end return end # Get checksums for existing files on the remote side. logger.debug("[SFTP] Slow path upload from #{local} to #{remote}") copy_checksums_script! checksum_cmd = "#{@config[:ruby_path]} #{CHECKSUMS_REMOTE_PATH} #{remote}" logger.debug("[SFTP] Running #{checksum_cmd}") checksums = JSON.parse(session.exec!(checksum_cmd)) # Sync files that have changed. files_to_upload(checksums, local, recursive).each do |rel_path| upload_file(checksums, local, remote, rel_path) end purge_files(checksums, remote) if purge # Wait until all xfers are complete. sftp_loop(0) end
sftp_xfers()
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 255 def sftp_xfers @sftp_xfers ||= [] end
tar(path)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 271 def tar(path) tarfile = StringIO.new("") Gem::Package::TarWriter.new(tarfile) do |tar| Dir[File.join(path, "**/*")].each do |file| mode = File.stat(file).mode relative_file = file.sub /^#{Regexp::escape path}\/?/, '' if File.directory?(file) tar.mkdir relative_file, mode else tar.add_file relative_file, mode do |tf| File.open(file, "rb") { |f| tf.write f.read } end end end end tarfile.rewind tarfile end
upload_file(checksums, local, remote, rel_path)
click to toggle source
# File lib/kitchen/transport/sftpgz.rb, line 228 def upload_file(checksums, local, remote, rel_path) parts = rel_path.split('/') parts.pop # Drop the filename since we are only checking dirs parts_to_check = [] until parts.empty? parts_to_check << parts.shift path_to_check = parts_to_check.join('/') unless checksums[path_to_check] logger.debug("[SFTP] Creating directory #{remote}#{path_to_check}") add_xfer(sftp_session.mkdir("#{remote}#{path_to_check}")) checksums[path_to_check] = true end end logger.debug("[SFTP] Uploading #{local}#{rel_path} to #{remote}#{rel_path}") add_xfer(sftp_session.upload("#{local}#{rel_path}", "#{remote}#{rel_path}")) end