class Autoproj::CLI::CI

Actual implementation of the functionality for the `autoproj ci` subcommand

Autoproj internally splits the CLI definition (Thor subclass) and the underlying functionality of each CLI subcommand. `autoproj-ci` follows the same pattern, and registers its subcommand in {MainCI} while implementing the functionality in this class

Constants

PHASES

Public Instance Methods

cache_pull(dir, ignore: []) click to toggle source
# File lib/autoproj/cli/ci.rb, line 41
def cache_pull(dir, ignore: [])
    packages = resolve_packages

    memo = {}
    results = packages.each_with_object({}) do |pkg, h|
        if ignore.include?(pkg.name)
            pkg.message '%s: ignored by command line'
            fingerprint = pkg.fingerprint(memo: memo)
            h[pkg.name] = {
                'cached' => false,
                'fingerprint' => fingerprint
            }
            next
        end

        state, fingerprint, metadata =
            pull_package_from_cache(dir, pkg, memo: memo)
        if state
            pkg.message "%s: pulled #{fingerprint}", :green
        else
            pkg.message "%s: #{fingerprint} not in cache, "\
                        'or not pulled from cache'
        end

        h[pkg.name] = metadata.merge(
            'cached' => state,
            'fingerprint' => fingerprint
        )
    end

    hit = results.count { |_, info| info['cached'] }
    Autoproj.message "#{hit} hits, #{results.size - hit} misses"

    results
end
cache_push(dir) click to toggle source
# File lib/autoproj/cli/ci.rb, line 77
def cache_push(dir)
    packages = resolve_packages
    metadata = consolidated_report['packages']

    memo = {}
    results = packages.each_with_object({}) do |pkg, h|
        if !(pkg_metadata = metadata[pkg.name])
            pkg.message '%s: no metadata in build report', :magenta
            next
        elsif !(build_info = pkg_metadata['build'])
            pkg.message '%s: no build info in build report', :magenta
            next
        elsif build_info['cached']
            pkg.message '%s: was pulled from cache, not pushing'
            next
        elsif !build_info['success']
            pkg.message '%s: build failed, not pushing', :magenta
            next
        end

        # Remove cached flags before saving
        pkg_metadata = pkg_metadata.dup
        PHASES.each do |phase_name|
            pkg_metadata[phase_name]&.delete('cached')
        end

        state, fingerprint = push_package_to_cache(
            dir, pkg, pkg_metadata, force: true, memo: memo
        )
        if state
            pkg.message "%s: pushed #{fingerprint}", :green
        else
            pkg.message "%s: #{fingerprint} already in cache"
        end

        h[pkg.name] = {
            'updated' => state,
            'fingerprint' => fingerprint
        }
    end

    hit = results.count { |_, info| info['updated'] }
    Autoproj.message "#{hit} updated packages, #{results.size - hit} "\
                     'reused entries'

    results
end
cache_state(dir, ignore: []) click to toggle source
# File lib/autoproj/cli/ci.rb, line 27
def cache_state(dir, ignore: [])
    packages = resolve_packages

    memo = {}
    packages.each_with_object({}) do |pkg, h|
        state = package_cache_state(dir, pkg, memo: memo)
        if ignore.include?(pkg.name)
            state = state.merge('cached' => false, 'metadata' => false)
        end

        h[pkg.name] = state
    end
end
cleanup_build_cache(dir, size_limit) click to toggle source
# File lib/autoproj/cli/ci.rb, line 274
def cleanup_build_cache(dir, size_limit)
    all_files = Find.enum_for(:find, dir).map do |path|
        next unless File.file?(path) && File.file?("#{path}.json")

        [path, File.stat(path)]
    end.compact

    total_size = all_files.map { |_, s| s.size }.sum
    lru = all_files.sort_by { |_, s| s.mtime }

    while total_size > size_limit
        path, stat = lru.shift
        Autoproj.message "removing #{path} (size=#{stat.size}, mtime=#{stat.mtime})"

        FileUtils.rm_f path
        FileUtils.rm_f "#{path}.json"
        total_size -= stat.size
    end

    Autoproj.message format("current build cache size: %.1f GB", Float(total_size) / 1_000_000_000)
    total_size
end
consolidated_report() click to toggle source
# File lib/autoproj/cli/ci.rb, line 314
def consolidated_report
    # NOTE: keys must match PHASES
    new_reports = {
        'import' => @ws.import_report_path,
        'build' => @ws.build_report_path,
        'test' => @ws.utility_report_path('test')
    }

    # We start with the cached info (if any) and override with
    # information from the other phase reports
    cache_report_path = File.join(@ws.root_dir, 'cache-pull.json')
    result = load_report(cache_report_path, 'cache_pull_report')['packages']
    result.delete_if do |_name, info|
        next true unless info.delete('cached')

        PHASES.each do |phase_name|
            if (phase_info = info[phase_name])
                phase_info['cached'] = true
            end
        end
        false
    end

    new_reports.each do |phase_name, path|
        report = load_report(path, "#{phase_name}_report")
        report['packages'].each do |pkg_name, pkg_info|
            result[pkg_name] ||= {}
            if pkg_info['invoked']
                result[pkg_name][phase_name] = pkg_info.merge(
                    'cached' => false,
                    'timestamp' => report['timestamp']
                )
            end
        end
    end
    { 'packages' => result }
end
create_report(dir) click to toggle source

Build a report in a given directory

The method itself will not archive the directory, only gather the information in a consistent way

# File lib/autoproj/cli/ci.rb, line 176
def create_report(dir)
    initialize_and_load
    finalize_setup([], non_imported_packages: :ignore)

    report = consolidated_report
    FileUtils.mkdir_p(dir)
    File.open(File.join(dir, 'report.json'), 'w') do |io|
        JSON.dump(report, io)
    end

    installation_manifest = InstallationManifest
                            .from_workspace_root(@ws.root_dir)
    logs = File.join(dir, 'logs')

    # Pre-create the logs, or cp_r will have a different behavior
    # if the directory exists or not
    FileUtils.mkdir_p(logs)
    installation_manifest.each_package do |pkg|
        glob = Dir.glob(File.join(pkg.logdir, '*'))
        FileUtils.cp_r(glob, logs) if File.directory?(pkg.logdir)
    end
end
load_built_flags() click to toggle source
# File lib/autoproj/cli/ci.rb, line 297
def load_built_flags
    path = @ws.build_report_path
    return {} unless File.file?(path)

    report = JSON.parse(File.read(path))
    report['build_report']['packages']
        .each_with_object({}) do |pkg_report, h|
            h[pkg_report['name']] = pkg_report['built']
        end
end
load_report(path, root_name, default: { 'packages' => {} }) click to toggle source
# File lib/autoproj/cli/ci.rb, line 308
def load_report(path, root_name, default: { 'packages' => {} })
    return default unless File.file?(path)

    JSON.parse(File.read(path)).fetch(root_name)
end
need_xunit_processing?(results_dir, xunit_output, force: false) click to toggle source

Checks if a package's test results should be processed with xunit-viewer

@param [String] results_dir the directory where the @param [String] xunit_output path to the xunit-viewer output. An

existing file is re-generated only if force is true

@param [Boolean] force re-generation of the xunit-viewer output

# File lib/autoproj/cli/ci.rb, line 131
def need_xunit_processing?(results_dir, xunit_output, force: false)
    # We don't re-generate if the xunit-processed files were cached
    return if !force && File.file?(xunit_output)

    # We only check whether there are xml files in the
    # package's test dir. That's the only check we do ... if
    # the XML files are not JUnit, we'll finish with an empty
    # xunit html file
    Dir.enum_for(:glob, File.join(results_dir, '*.xml'))
       .first
end
package_cache_path(dir, pkg, fingerprint: nil, memo: {}) click to toggle source
# File lib/autoproj/cli/ci.rb, line 199
def package_cache_path(dir, pkg, fingerprint: nil, memo: {})
    fingerprint ||= pkg.fingerprint(memo: memo)
    File.join(dir, pkg.name, fingerprint)
end
package_cache_state(dir, pkg, memo: {}) click to toggle source
# File lib/autoproj/cli/ci.rb, line 204
def package_cache_state(dir, pkg, memo: {})
    fingerprint = pkg.fingerprint(memo: memo)
    path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)

    {
        'path' => path,
        'cached' => File.file?(path),
        'metadata' => File.file?("#{path}.json"),
        'fingerprint' => fingerprint
    }
end
process_test_results(force: false, xunit_viewer: 'xunit-viewer') click to toggle source

Post-processing of test results

# File lib/autoproj/cli/ci.rb, line 168
def process_test_results(force: false, xunit_viewer: 'xunit-viewer')
    process_test_results_xunit(force: force, xunit_viewer: xunit_viewer)
end
process_test_results_xunit(force: false, xunit_viewer: 'xunit-viewer') click to toggle source

Process the package's test results with xunit-viewer

@param [String] xunit_viewer path to xunit-viewer @param [Boolean] force re-generation of the xunit-viewer output. If

false, packages that already have a xunit-viewer output will be skipped
# File lib/autoproj/cli/ci.rb, line 148
def process_test_results_xunit(force: false, xunit_viewer: 'xunit-viewer')
    consolidated_report['packages'].each_value do |info|
        next unless info['test']
        next unless (results_dir = info['test']['target_dir'])

        xunit_output = "#{results_dir}.html"
        next unless need_xunit_processing?(results_dir, xunit_output,
                                           force: force)

        success = system(xunit_viewer,
                         "--results=#{results_dir}",
                         "--output=#{xunit_output}")
        unless success
            Autoproj.warn 'xunit-viewer conversion failed '\
                          "for '#{results_dir}'"
        end
    end
end
pull_package_from_cache(dir, pkg, memo: {}) click to toggle source
# File lib/autoproj/cli/ci.rb, line 219
def pull_package_from_cache(dir, pkg, memo: {})
    fingerprint = pkg.fingerprint(memo: memo)
    path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)
    return [false, fingerprint, {}] unless File.file?(path)

    metadata_path = "#{path}.json"
    metadata =
        if File.file?(metadata_path)
            JSON.parse(File.read(metadata_path))
        else
            {}
        end

    # Do not pull packages for which we should run tests
    tests_enabled = pkg.test_utility.enabled?
    tests_invoked = metadata['test'] && metadata['test']['invoked']
    if tests_enabled && !tests_invoked
        pkg.message '%s: has tests that have never '\
                    'been invoked, not pulling from cache'
        return [false, fingerprint, {}]
    end

    FileUtils.mkdir_p(pkg.prefix)
    unless system('tar', 'xzf', path, chdir: pkg.prefix, out: '/dev/null')
        raise PullError, "tar failed when pulling cache file for #{pkg.name}"
    end

    [true, fingerprint, metadata]
end
push_package_to_cache(dir, pkg, metadata, force: false, memo: {}) click to toggle source
# File lib/autoproj/cli/ci.rb, line 249
def push_package_to_cache(dir, pkg, metadata, force: false, memo: {})
    fingerprint = pkg.fingerprint(memo: memo)
    path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)
    temppath = "#{path}.#{Process.pid}.#{rand(256)}"

    FileUtils.mkdir_p(File.dirname(path))
    if force || !File.file?("#{path}.json")
        File.open(temppath, 'w') { |io| JSON.dump(metadata, io) }
        FileUtils.mv(temppath, "#{path}.json")
    end

    if !force && File.file?(path)
        # Update modification time for the cleanup process
        FileUtils.touch(path)
        return [false, fingerprint]
    end

    result = system('tar', 'czf', temppath, '.',
                    chdir: pkg.prefix, out: '/dev/null')
    raise "tar failed when pushing cache file for #{pkg.name}" unless result

    FileUtils.mv(temppath, path)
    [true, fingerprint]
end
resolve_packages() click to toggle source
# File lib/autoproj/cli/ci.rb, line 17
def resolve_packages
    initialize_and_load
    source_packages, * = finalize_setup(
        [], non_imported_packages: :ignore
    )
    source_packages.map do |pkg_name|
        ws.manifest.find_autobuild_package(pkg_name)
    end
end