module MyPrecious

Declare the module here so it doesn't cause problems for files in the “myprecious” directory (or does cause problems if they try to declare it a class)

Constants

DATA_DIR
ONE_DAY
Program

Attributes

caching_disabled[RW]

Public Class Methods

common_program_args(parser, args) click to toggle source
# File lib/myprecious.rb, line 28
def self.common_program_args(parser, args)
  parser.on(
    '-o', '--out FILE',
    "Output file to generate",
  ) {|fpath| args.output_file = Pathname(fpath)}
  
  args.target = Pathname.pwd
  parser.on(
    '-C', '--dir PATH',
    "Path to inspect",
  ) do |fpath|
    fpath = Pathname(fpath)
    parser.invalid_args!("#{fpath} does not exist.") unless fpath.exist?
    args.target = fpath
    CVEs.config_dir = fpath
  end
  
  parser.on(
    '--[no-]cache',
    "Control caching of dependency information"
  ) {|v| MyPrecious.caching_disabled = v}
end
data_cache(fpath) click to toggle source

Declare a path as a data cache

This method returns the path it was given in fpath.

# File lib/myprecious/data_caches.rb, line 12
def data_cache(fpath)
  (@data_caches ||= []) << fpath
  return fpath
end
data_caches() click to toggle source

Retrieve an Array of all known data caches

# File lib/myprecious/data_caches.rb, line 20
def data_caches
  (@data_caches || [])
end
tracing_errors?() click to toggle source
# File lib/myprecious.rb, line 24
def self.tracing_errors?
  !ENV['TRACE_ERRORS'].to_s.empty?
end
yes_no(prompt) click to toggle source

Prompt user for a yes/no answer

It doesn't matter if they've redirected STDIN/STDOUT – this grabs the TTY directly.

# File lib/myprecious.rb, line 169
def self.yes_no(prompt)
  Pathname('/dev/tty').open('r+') do |term|
    loop do
      term.write("#{prompt} ")
      case term.gets[0..-2]
      when 'y', 'Y', 'yes', 'Yes', 'YES'
        return true
      when 'n', 'N', 'no', 'No', 'NO'
        return false
      end
    end
  end
end

Public Instance Methods

pypi_release_url(release) click to toggle source
# File lib/myprecious/python_packages.rb, line 925
def pypi_release_url(release)
  "https://pypi.org/pypi/#{name}/#{release}/json"
end
pypi_url() click to toggle source
# File lib/myprecious/python_packages.rb, line 921
def pypi_url
  "https://pypi.org/pypi/#{name}/json"
end

Private Instance Methods

extracted_url(archive_type) { |url_f, extraction_path| ... } click to toggle source
# File lib/myprecious/python_packages.rb, line 1174
def extracted_url(archive_type, &blk)
  puts "Downloading #{self.url}"
  extraction_path = CODE_CACHE_DIR.join(
    "#{archive_type}_#{Digest::MD5.hexdigest(self.url.to_s)}"
  )
  CODE_CACHE_DIR.mkpath
  
  if %w[http https].include?(self.url.scheme)
    # TODO: Make a HEAD request to see if re-download is necessary
  end
  
  self.url.open('rb') {|url_f| yield url_f, extraction_path}
  
  return extraction_path
end
get_age() click to toggle source
# File lib/myprecious/python_packages.rb, line 946
def get_age
  versions_with_release.each do |vnum, released|
    return ((Time.now - released) / ONE_DAY).to_i if vnum == current_version
  end
  return nil
end
get_package_info() click to toggle source
# File lib/myprecious/python_packages.rb, line 930
def get_package_info
  cache = PACKAGE_CACHE_DIR.join("#{name}.json")
  apply_cache(cache) do
    pypi_response = RestClient.get(pypi_url)
    JSON.parse(pypi_response)
  end
end
get_release_info(release) click to toggle source
# File lib/myprecious/python_packages.rb, line 938
def get_release_info(release)
  cache = PACKAGE_CACHE_DIR.join(name, "#{release}.json")
  apply_cache(cache) do
    pypi_response = RestClient.get(pypi_release_url(release))
    JSON.parse(pypi_response)
  end
end
get_url_content_type() click to toggle source
# File lib/myprecious/python_packages.rb, line 1107
def get_url_content_type
  # TODO: Make a HEAD request to the URL to find out the content type
  return 'application/octet-stream'
end
nonpatch_versegs(ver) click to toggle source

Given a version, return the parts that we expect to define the major/minor release series

Returns an Array

# File lib/myprecious/python_packages.rb, line 959
def nonpatch_versegs(ver)
  return nil if ver.nil?
  [ver.epoch] + ver.final.take(2)
end
setup_data() click to toggle source

Get data from the setup.py file of the package

# File lib/myprecious/python_packages.rb, line 967
      def setup_data
        return @setup_data if defined? @setup_data
        unless self.url
          raise "#setup_data called for #{name}, may only be called for packages specified by URL"
        end
        
        python_code = <<~END_OF_PYTHON
          import json, sys
          from unittest.mock import patch

          sys.path[0:0] = ['.']

          def capture_setup(**kwargs):
              capture_setup.captured = kwargs

          with patch('setuptools.setup', capture_setup):
              import setup

          json.dump(
            capture_setup.captured,
            sys.stdout,
            default=lambda o: "<{}.{}>".format(type(o).__module__, type(o).__qualname__),
          )
        END_OF_PYTHON
        
        output, status = with_package_files do |workdir|
          Dir.chdir(workdir) do
            Open3.capture2('python3', stdin_data: python_code)
          end
        end || []
        
        @setup_data = begin
          case status
          when nil
            warn("Package files unavailable, could not read setup.py")
            {}
          when :success?.to_proc
            JSON.parse(output)
          else
            warn("Failed to read setup.py in for #{self.url}")
            {}
          end
        rescue StandardError => ex
          warn("Failed to read setup.py in for #{self.url}: #{ex}")
          {}
        end
      end
tgz_url?() click to toggle source
# File lib/myprecious/python_packages.rb, line 1142
def tgz_url?
  case get_url_content_type
  when %r{^application/(x-tar(\+gzip)?|gzip)$} then true
  when 'application/octet-stream'
    !!(self.url.path.downcase =~ /\.(tar\.gz|tgz)$/)
  else false
  end
end
with_git_worktree(uri) { |package_dir)| ... } click to toggle source

Implementation of with_package_files for git URIs

# File lib/myprecious/python_packages.rb, line 1048
def with_git_worktree(uri)
  git_url = uri.dup
  git_url.path, committish = uri.path.split('@', 2)
  uri_fragment, git_url.fragment = uri.fragment, nil
  repo_path = CODE_CACHE_DIR.join("git_#{Digest::MD5.hexdigest(git_url.to_s)}.git")
  
  CODE_CACHE_DIR.mkpath
  
  in_dir_git_cmd = ['git', '-C', repo_path.to_s]
  
  if repo_path.exist?
    puts "Fetching #{git_url} to #{repo_path}..."
    cmd = in_dir_git_cmd + ['fetch', '--tags', 'origin', '+refs/heads/*:refs/heads/*']
    output, status = Open3.capture2(*cmd)
    unless status.success?
      warn("Failed to fetch 'origin' in #{repo_path}")
      return
    end
  else
    cmd = ['git', 'clone', '--bare', git_url.to_s, repo_path.to_s]
    output, status = Open3.capture2(*cmd)
    unless status.success?
      warn("Failed to clone #{git_url}")
      return
    end
  end
  
  committish ||= (
    cmd = in_dir_git_cmd + ['ls-remote', 'origin', 'HEAD']
    output, status = Open3.capture2(*cmd)
    unless status.success?
      raise "Unable to read the HEAD of orgin"
    end
    output.split("\t")[0]
  )
  Dir.mktmpdir("myprecious-git-") do |workdir|
    cmds = [
      in_dir_git_cmd + ['archive', committish],
      ['tar', '-x', '-C', workdir.to_s],
    ]
    statuses = Open3.pipeline(*cmds, in: :close)
    if failed_i = statuses.find {|s| s.exited? && !s.success?}
      exitstatus = statuses[failed_i].exitstatus
      failed_cmd_str = cmds[failed_i].shelljoin
      warn(
        "Failed to create temporary folder at command:\n" +
        "    #{failed_cmd.light_red} (exited with code #{exitstatus})"
      )
      return
    end
    
    fragment_parts = Hash[URI.decode_www_form(uri.fragment || '')]
    package_dir = Pathname(workdir).join(
      fragment_parts.fetch('subdirectory', '.')
    )
    return (yield package_dir)
  end
end
with_package_files(&blk) click to toggle source

Yield a Pathname for the directory containing the package files

Returns the result of the block, or nil if the block is not executed. The directory with the package files may be removed when the block exits.

# File lib/myprecious/python_packages.rb, line 1022
def with_package_files(&blk)
  case self.url.scheme
  when 'git'
    return with_git_worktree(self.url, &blk)
  when /^git\+/
    git_uri = self.url.dup
    git_uri.scheme = self.url.scheme[4..-1]
    return with_git_worktree(git_uri, &blk)
  when 'http', 'https'
    case
    when zip_url?
      return with_unzipped_files(&blk)
    when tgz_url?
      return with_untarred_files(&blk)
    else
      warn("Unknown archive type for URL: #{self.url}")
      return nil
    end
  else
    warn("Unable to process URI package requirement: #{self.url}")
  end
end
with_untarred_files() { |tar_path)| ... } click to toggle source

Implementation of with_package_files for TGZ file URLs

# File lib/myprecious/python_packages.rb, line 1154
def with_untarred_files
  tar_path = extracted_url("tar") do |url_f, tar_path|
    Gem::Package::TarReader.new(Zlib::GzipReader.new(url_f)) do |tar_file|
      tar_file.each do |entry|
        if entry.full_name =~ %r{(^|/)\.\./}
          warn("Did not extract #{entry.name} from #{self.url}")
        elsif entry.file?
          dest_file = tar_path.join(entry.full_name.split('/', 2)[1])
          dest_file.dirname.mkpath
          dest_file.open('wb') do |df|
            IO.copy_stream(entry, df)
          end
        end
      end
    end
  end
  
  return (yield tar_path)
end
with_unzipped_files() { |zip_path)| ... } click to toggle source

Implementation of with_package_files for ZIP file URLs

# File lib/myprecious/python_packages.rb, line 1124
def with_unzipped_files
  zip_path = extracted_url("zip") do |url_f, zip_path|
    Zip::File.open_buffer(url_f) do |zip_file|
      zip_file.each do |entry|
        if entry.name_safe?
          dest_file = zip_path.join(entry.name.split('/', 2)[1])
          dest_file.dirname.mkpath
          entry.extract(dest_file.to_s) {:overwrite}
        else
          warn("Did not extract #{entry.name} from #{self.url}")
        end
      end
    end
  end
  
  return (yield zip_path)
end
zip_url?() click to toggle source
# File lib/myprecious/python_packages.rb, line 1112
def zip_url?
  case get_url_content_type
  when 'application/zip' then true
  when 'application/octet-stream'
    self.url.path.downcase.end_with?('.zip')
  else false
  end
end