module MyPrecious
Declare the module here so it doesn't cause problems for files in the “myprecious” directory (or does cause problems if they try to declare it a class)
Constants
- DATA_DIR
- ONE_DAY
- Program
Attributes
Public Class Methods
# File lib/myprecious.rb, line 28 def self.common_program_args(parser, args) parser.on( '-o', '--out FILE', "Output file to generate", ) {|fpath| args.output_file = Pathname(fpath)} args.target = Pathname.pwd parser.on( '-C', '--dir PATH', "Path to inspect", ) do |fpath| fpath = Pathname(fpath) parser.invalid_args!("#{fpath} does not exist.") unless fpath.exist? args.target = fpath CVEs.config_dir = fpath end parser.on( '--[no-]cache', "Control caching of dependency information" ) {|v| MyPrecious.caching_disabled = v} end
Declare a path as a data cache
This method returns the path it was given in fpath
.
# File lib/myprecious/data_caches.rb, line 12 def data_cache(fpath) (@data_caches ||= []) << fpath return fpath end
Retrieve an Array of all known data caches
# File lib/myprecious/data_caches.rb, line 20 def data_caches (@data_caches || []) end
# File lib/myprecious.rb, line 24 def self.tracing_errors? !ENV['TRACE_ERRORS'].to_s.empty? end
Prompt user for a yes/no answer
It doesn't matter if they've redirected STDIN/STDOUT – this grabs the TTY directly.
# File lib/myprecious.rb, line 169 def self.yes_no(prompt) Pathname('/dev/tty').open('r+') do |term| loop do term.write("#{prompt} ") case term.gets[0..-2] when 'y', 'Y', 'yes', 'Yes', 'YES' return true when 'n', 'N', 'no', 'No', 'NO' return false end end end end
Public Instance Methods
# File lib/myprecious/python_packages.rb, line 925 def pypi_release_url(release) "https://pypi.org/pypi/#{name}/#{release}/json" end
# File lib/myprecious/python_packages.rb, line 921 def pypi_url "https://pypi.org/pypi/#{name}/json" end
Private Instance Methods
# File lib/myprecious/python_packages.rb, line 1174 def extracted_url(archive_type, &blk) puts "Downloading #{self.url}" extraction_path = CODE_CACHE_DIR.join( "#{archive_type}_#{Digest::MD5.hexdigest(self.url.to_s)}" ) CODE_CACHE_DIR.mkpath if %w[http https].include?(self.url.scheme) # TODO: Make a HEAD request to see if re-download is necessary end self.url.open('rb') {|url_f| yield url_f, extraction_path} return extraction_path end
# File lib/myprecious/python_packages.rb, line 946 def get_age versions_with_release.each do |vnum, released| return ((Time.now - released) / ONE_DAY).to_i if vnum == current_version end return nil end
# File lib/myprecious/python_packages.rb, line 930 def get_package_info cache = PACKAGE_CACHE_DIR.join("#{name}.json") apply_cache(cache) do pypi_response = RestClient.get(pypi_url) JSON.parse(pypi_response) end end
# File lib/myprecious/python_packages.rb, line 938 def get_release_info(release) cache = PACKAGE_CACHE_DIR.join(name, "#{release}.json") apply_cache(cache) do pypi_response = RestClient.get(pypi_release_url(release)) JSON.parse(pypi_response) end end
# File lib/myprecious/python_packages.rb, line 1107 def get_url_content_type # TODO: Make a HEAD request to the URL to find out the content type return 'application/octet-stream' end
Given a version, return the parts that we expect to define the major/minor release series
Returns an Array
# File lib/myprecious/python_packages.rb, line 959 def nonpatch_versegs(ver) return nil if ver.nil? [ver.epoch] + ver.final.take(2) end
Get data from the setup.py file of the package
# File lib/myprecious/python_packages.rb, line 967 def setup_data return @setup_data if defined? @setup_data unless self.url raise "#setup_data called for #{name}, may only be called for packages specified by URL" end python_code = <<~END_OF_PYTHON import json, sys from unittest.mock import patch sys.path[0:0] = ['.'] def capture_setup(**kwargs): capture_setup.captured = kwargs with patch('setuptools.setup', capture_setup): import setup json.dump( capture_setup.captured, sys.stdout, default=lambda o: "<{}.{}>".format(type(o).__module__, type(o).__qualname__), ) END_OF_PYTHON output, status = with_package_files do |workdir| Dir.chdir(workdir) do Open3.capture2('python3', stdin_data: python_code) end end || [] @setup_data = begin case status when nil warn("Package files unavailable, could not read setup.py") {} when :success?.to_proc JSON.parse(output) else warn("Failed to read setup.py in for #{self.url}") {} end rescue StandardError => ex warn("Failed to read setup.py in for #{self.url}: #{ex}") {} end end
# File lib/myprecious/python_packages.rb, line 1142 def tgz_url? case get_url_content_type when %r{^application/(x-tar(\+gzip)?|gzip)$} then true when 'application/octet-stream' !!(self.url.path.downcase =~ /\.(tar\.gz|tgz)$/) else false end end
Implementation of with_package_files
for git URIs
# File lib/myprecious/python_packages.rb, line 1048 def with_git_worktree(uri) git_url = uri.dup git_url.path, committish = uri.path.split('@', 2) uri_fragment, git_url.fragment = uri.fragment, nil repo_path = CODE_CACHE_DIR.join("git_#{Digest::MD5.hexdigest(git_url.to_s)}.git") CODE_CACHE_DIR.mkpath in_dir_git_cmd = ['git', '-C', repo_path.to_s] if repo_path.exist? puts "Fetching #{git_url} to #{repo_path}..." cmd = in_dir_git_cmd + ['fetch', '--tags', 'origin', '+refs/heads/*:refs/heads/*'] output, status = Open3.capture2(*cmd) unless status.success? warn("Failed to fetch 'origin' in #{repo_path}") return end else cmd = ['git', 'clone', '--bare', git_url.to_s, repo_path.to_s] output, status = Open3.capture2(*cmd) unless status.success? warn("Failed to clone #{git_url}") return end end committish ||= ( cmd = in_dir_git_cmd + ['ls-remote', 'origin', 'HEAD'] output, status = Open3.capture2(*cmd) unless status.success? raise "Unable to read the HEAD of orgin" end output.split("\t")[0] ) Dir.mktmpdir("myprecious-git-") do |workdir| cmds = [ in_dir_git_cmd + ['archive', committish], ['tar', '-x', '-C', workdir.to_s], ] statuses = Open3.pipeline(*cmds, in: :close) if failed_i = statuses.find {|s| s.exited? && !s.success?} exitstatus = statuses[failed_i].exitstatus failed_cmd_str = cmds[failed_i].shelljoin warn( "Failed to create temporary folder at command:\n" + " #{failed_cmd.light_red} (exited with code #{exitstatus})" ) return end fragment_parts = Hash[URI.decode_www_form(uri.fragment || '')] package_dir = Pathname(workdir).join( fragment_parts.fetch('subdirectory', '.') ) return (yield package_dir) end end
Yield a Pathname for the directory containing the package files
Returns the result of the block, or nil
if the block is not executed. The directory with the package files may be removed when the block exits.
# File lib/myprecious/python_packages.rb, line 1022 def with_package_files(&blk) case self.url.scheme when 'git' return with_git_worktree(self.url, &blk) when /^git\+/ git_uri = self.url.dup git_uri.scheme = self.url.scheme[4..-1] return with_git_worktree(git_uri, &blk) when 'http', 'https' case when zip_url? return with_unzipped_files(&blk) when tgz_url? return with_untarred_files(&blk) else warn("Unknown archive type for URL: #{self.url}") return nil end else warn("Unable to process URI package requirement: #{self.url}") end end
Implementation of with_package_files
for TGZ file URLs
# File lib/myprecious/python_packages.rb, line 1154 def with_untarred_files tar_path = extracted_url("tar") do |url_f, tar_path| Gem::Package::TarReader.new(Zlib::GzipReader.new(url_f)) do |tar_file| tar_file.each do |entry| if entry.full_name =~ %r{(^|/)\.\./} warn("Did not extract #{entry.name} from #{self.url}") elsif entry.file? dest_file = tar_path.join(entry.full_name.split('/', 2)[1]) dest_file.dirname.mkpath dest_file.open('wb') do |df| IO.copy_stream(entry, df) end end end end end return (yield tar_path) end
Implementation of with_package_files
for ZIP file URLs
# File lib/myprecious/python_packages.rb, line 1124 def with_unzipped_files zip_path = extracted_url("zip") do |url_f, zip_path| Zip::File.open_buffer(url_f) do |zip_file| zip_file.each do |entry| if entry.name_safe? dest_file = zip_path.join(entry.name.split('/', 2)[1]) dest_file.dirname.mkpath entry.extract(dest_file.to_s) {:overwrite} else warn("Did not extract #{entry.name} from #{self.url}") end end end end return (yield zip_path) end
# File lib/myprecious/python_packages.rb, line 1112 def zip_url? case get_url_content_type when 'application/zip' then true when 'application/octet-stream' self.url.path.downcase.end_with?('.zip') else false end end