class Autoproj::Sync::Remote
Attributes
name[R]
uri[R]
Public Class Methods
from_uri(uri, name: uri, enabled: true)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 11 def self.from_uri(uri, name: uri, enabled: true) if uri.scheme != "ssh" raise ArgumentError, "unsupported protocol #{uri.scheme}" end Remote.new(uri, name: name, enabled: enabled) end
new(uri, name: uri, enabled: true)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 18 def initialize(uri, name: uri, enabled: true) @uri = uri @enabled = enabled @name = name end
Public Instance Methods
autoproj_annex_files(ws)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 105 def autoproj_annex_files(ws) user_files = %w[env.sh]. map do |file| File.join(ws.root_dir, file) end autoproj_files = %w[env.yml installation-manifest]. map do |file| File.join(ws.root_dir, '.autoproj', file) end bundler_files = %w[gems/Gemfile gems/Gemfile.lock]. map do |file| File.join(ws.prefix_dir, file) end [*user_files, *ws.env.source_before, *ws.env.source_after, *autoproj_files, *bundler_files]. find_all { |file| File.exist?(file) } end
bootstrap_or_update_autoproj(sftp, ws)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 313 def bootstrap_or_update_autoproj(sftp, ws) gemfile_lock_path = File.join(ws.root_dir, ".autoproj/Gemfile.lock") if remote_file_exist?(sftp, gemfile_lock_path) remote_gemfile_lock = remote_file_get(sftp, gemfile_lock_path) local_gemfile_lock = local_file_get(gemfile_lock_path) if remote_gemfile_lock == local_gemfile_lock info "remote Autoproj install up-to-date" info "sync: Autoproj install up-to-date on #{name}" return end info "sync: updating the Autoproj install on #{name}" remote_file_put(sftp, gemfile_lock_path, local_gemfile_lock) remote_file_transfer( sftp, File.join(ws.root_dir, ".autoproj/Gemfile")) remote_file_transfer( sftp, File.join(ws.root_dir, ".autoproj/config.yml")) result = remote_autoproj( sftp, ws.root_dir, "update", "--autoproj") unless result[:exit_code] == 0 raise FailedRemoteCommand, "failed to update Autoproj:\n"\ "autoproj update --autoproj finished with exit status "\ "#{result[:exit_code]}\n"\ "#{result}" end else info "sync: installing Autoproj on #{name}" autoproj_spec = Bundler.definition.specs. find { |spec| spec.name == "autoproj" } autoproj_dir = autoproj_spec.full_gem_path install_script = File.join(autoproj_dir, "bin", "autoproj_install") remote_mkdir_p(sftp, ws.root_dir) sftp.upload!(install_script, remote_path(File.join(ws.root_dir, "autoproj_install"))) remote_file_transfer(sftp, File.join(ws.root_dir, ".autoproj/config.yml"), target: remote_path(File.join(ws.root_dir, 'bootstrap-config.yml'))) remote_file_transfer(sftp, File.join(ws.root_dir, ".autoproj/Gemfile"), target: remote_path(File.join(ws.root_dir, 'bootstrap-Gemfile'))) result = sftp.session.exec!("cd '#{remote_path(ws.root_dir)}' && "\ "#{ws.config.ruby_executable} autoproj_install "\ "--gemfile bootstrap-Gemfile "\ "--seed-config bootstrap-config.yml") if result.exitstatus != 0 raise RuntimeError, "failed to install autoproj: #{result}" end end end
create_package_directories(sftp, pkg)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 189 def create_package_directories(sftp, pkg) Autobuild.progress_start pkg, "sync: preparing #{pkg.name}@#{name}", done_message: "sync: prepared #{pkg.name}@#{name}" do remote_mkdir_p(sftp, pkg.autobuild.prefix) remote_mkdir_p(sftp, File.dirname(pkg.autobuild.installstamp)) end end
each_outdated_package(sftp, ws, packages) { |package| ... }
click to toggle source
Enumerate the packages that are outdated on the remote
@yieldparam [Net::SFTP::Session] sftp the opened SFTP session, that can
be used to do further operations on the remote
@yieldparam [Autoproj::PackageDescription] an outdated package
# File lib/autoproj/sync/remote.rb, line 45 def each_outdated_package(sftp, ws, packages) return enum_for(__method__, sftp, ws, packages) unless block_given? stat = packages.map do |package| autobuild = package.autobuild installstamp = autobuild.installstamp next unless File.exist?(installstamp) local_stat = File.stat(installstamp) remote_path = File.join(uri.path, installstamp) begin remote_stat = sftp.file.open(remote_path) do |f| f.stat end [package, local_stat, remote_stat, remote_path] rescue Net::SFTP::StatusException => e if e.code != Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE raise end package end end stat.compact.map do |package, local_stat, remote_stat, remote_path| yield(package) if !local_stat || changed_stat?(local_stat, remote_stat) end.compact end
enabled?()
click to toggle source
# File lib/autoproj/sync/remote.rb, line 24 def enabled? @enabled end
info(message)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 309 def info(message) Autoproj.message " #{message}", force: true end
install_osdep_packages(sftp, ws, manager_name, packages)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 170 def install_osdep_packages(sftp, ws, manager_name, packages) Autobuild.progress_start "sync-#{name}-osdeps-#{manager_name}", "sync: handling #{packages.size} osdeps #{manager_name} packages "\ "on #{name}", done_message: "sync: handled #{packages.size} "\ "osdeps #{manager_name} packages on #{name}" do result = remote_autoproj(sftp, ws.root_dir, "sync", "install-osdeps", manager_name, *packages) if result[:exit_code] != 0 raise RuntimeError, "remote autoproj command failed\n"\ "autoproj exited with status "\ "#{result[:exit_code]}\n#{result}" end end end
local_file_get(local_path)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 305 def local_file_get(local_path) File.read(local_path) end
osdeps(sftp, ws, osdep_packages)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 145 def osdeps(sftp, ws, osdep_packages) installer = ws.os_package_installer installer.setup_package_managers all = ws.all_os_packages partitioned_packages = installer. resolve_and_partition_osdep_packages(osdep_packages, all) os_packages = partitioned_packages.delete(installer.os_package_manager) if os_packages partitioned_packages = [[installer.os_package_manager, os_packages]]. concat(partitioned_packages.to_a) end partitioned_packages = partitioned_packages.map do |manager, packages| manager_name, _ = installer.package_managers. find { |key, obj| manager == obj } [manager_name, packages] end partitioned_packages.each do |manager_name, packages| install_osdep_packages(sftp, ws, manager_name, packages) end end
remote_autoproj(sftp, root_dir, *command, chdir: nil, interactive: false)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 238 def remote_autoproj(sftp, root_dir, *command, chdir: nil, interactive: false) remote_exec(sftp, remote_path(File.join(root_dir, ".autoproj/bin/autoproj")), *command, chdir: chdir, interactive: interactive) end
remote_exec(sftp, *command, chdir: nil, interactive: false)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 244 def remote_exec(sftp, *command, chdir: nil, interactive: false) if interactive remote_interactive_exec(sftp, *command, chdir: chdir) else status = Hash.new ios = Hash[:stdout => STDOUT, :stderr => STDOUT] target_dir = @uri.path target_dir = File.join(target_dir, chdir) if chdir pid = nil command = "cd '#{target_dir}' && "\ "echo \"AUTOPROJ_SYNC_PID=$$\" && "\ "exec '" + command.join("' '") + "'" ch = sftp.session.exec(command, status: status) do |channel, stream, data| if !pid && (m = /^AUTOPROJ_SYNC_PID=(\d+)/.match(data)) pid = Integer(m[1]) else ios[stream].print(data) ios[stream].flush end end begin ch.wait status rescue Interrupt sftp.session.exec!("kill #{pid}") if pid ch.wait raise end end end
remote_file_exist?(sftp, path)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 215 def remote_file_exist?(sftp, path) sftp.stat!(remote_path(path)) true rescue Net::SFTP::StatusException => e if e.code == Net::SFTP::Constants::StatusCodes::FX_NO_SUCH_FILE false else raise end end
remote_file_get(sftp, local_path)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 226 def remote_file_get(sftp, local_path) sftp.download!(remote_path(local_path)) end
remote_file_put(sftp, local_path, content)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 230 def remote_file_put(sftp, local_path, content) sftp.upload!(StringIO.new(content), remote_path(local_path)) end
remote_file_transfer(sftp, local_path, target: remote_path(local_path))
click to toggle source
# File lib/autoproj/sync/remote.rb, line 234 def remote_file_transfer(sftp, local_path, target: remote_path(local_path)) sftp.upload!(local_path, target) end
remote_interactive_exec(sftp, *command, chdir: nil)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 276 def remote_interactive_exec(sftp, *command, chdir: nil) channel = sftp.session.open_channel do |ch| ch.on_data do |ch, data| STDOUT.print data STDOUT.flush end ch.on_extended_data do |ch, type, data| STDERR.print data end ch.request_pty ch.exec("cd '#{chdir}' && '" + command.join("' '") + "'") end ssh = sftp.session while channel.active? ssh.process(0.1) begin while true data = STDIN.read_nonblock(4096) channel.send_data(data) end rescue IO::WaitReadable end end rescue EOFError channel.close end
remote_path()
click to toggle source
# File lib/autoproj/sync/remote.rb, line 28 def remote_path @uri.path end
rsync_dir(sftp, local_dir)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 133 def rsync_dir(sftp, local_dir) remote_dir = remote_path(local_dir) ["rsync", "-a", "--delete-after", "#{local_dir}/", "#{rsync_target}:#{remote_dir}/"] end
rsync_file(sftp, local_file)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 139 def rsync_file(sftp, local_file) remote_file = remote_path(local_file) ["rsync", "-a", local_file, "#{rsync_target}:#{remote_file}"] end
rsync_package(sftp, pkg)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 198 def rsync_package(sftp, pkg) Autobuild.progress_start pkg, "sync: updating #{pkg.name}@#{name}", done_message: "sync: updated #{pkg.name}@#{name}" do ops = [rsync_dir(sftp, pkg.autobuild.prefix), rsync_file(sftp, pkg.autobuild.installstamp)] ops.each do |op| if !system(*op) raise "update of #{pkg.name} failed" end end end end
rsync_target()
click to toggle source
# File lib/autoproj/sync/remote.rb, line 123 def rsync_target if @uri.user && @uri.password "#{@uri.user}:#{@uri.password}@#{@uri.host}" elsif @uri.user "#{@uri.user}@#{@uri.host}" else @uri.host end end
start() { |sftp| ... }
click to toggle source
# File lib/autoproj/sync/remote.rb, line 32 def start result = nil Net::SFTP.start(@uri.host, @uri.user, password: @uri.password) do |sftp| result = yield(sftp) end result end
update(sftp, ws, packages)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 363 def update(sftp, ws, packages) # First check if autoproj is bootstrapped on the target already bootstrap_or_update_autoproj(sftp, ws) packages = each_outdated_package(sftp, ws, packages).to_a info "sync: #{packages.size} outdated packages on #{name}" executor = Concurrent::FixedThreadPool.new(6) futures = packages.map do |pkg| create_package_directories(sftp, pkg) Concurrent::Future.execute(executor: executor) do rsync_package(sftp, pkg) end end # Copy some autoproj installation-manifest files Autobuild.progress_start "sync-#{name}-autoproj", "sync: updating Autoproj configuration files on #{name}", done_message: "sync: updated Autoproj configuration files on #{name}" do autoproj_annex_files(ws).each do |file| missing = [] dir = File.join(@uri.path, File.dirname(file)) while dir != '/' begin sftp.stat!(dir) break rescue Net::SFTP::StatusException missing.unshift dir dir = File.dirname(dir) end end missing.each do |dir| sftp.mkdir!(dir) end sftp.upload!(file, File.join(@uri.path, file)) end end futures.each_with_index do |f, i| f.value! end ensure if executor executor.shutdown executor.wait_for_termination end end
Private Instance Methods
changed_stat?(local, remote)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 74 def changed_stat?(local, remote) return true if local.size != remote.size local_sec = local.mtime.tv_sec local_usec = local.mtime.tv_usec if remote.mtime != local_sec true elsif !remote.respond_to?(:remote_nseconds) false else (remote.mtime_nseconds / 1000) != local_usec end end
remote_mkdir_p(sftp, local_path)
click to toggle source
# File lib/autoproj/sync/remote.rb, line 88 def remote_mkdir_p(sftp, local_path) remote_path = File.join(@uri.path, local_path) ops = [] while remote_path != '/' ops << [remote_path, sftp.stat(remote_path)] remote_path = File.dirname(remote_path) end missing = ops.take_while do |_, op| op.wait !op.response.ok? end mkdirs = missing.reverse.map do |path, _| sftp.mkdir(path) end mkdirs.each(&:wait) end