class Dependabot::NpmAndYarn::FileFetcher
Constants
- NPM_PATH_DEPENDENCY_STARTS
Npm always prefixes file paths in the lockfile “version” with “file:” even when a naked path is used (e.g. “../dep”)
- PATH_DEPENDENCY_CLEAN_REGEX
- PATH_DEPENDENCY_STARTS
“link:” is only supported by Yarn but is interchangeable with “file:” when it specifies a path. Only include Yarn “”‘s that start with a path and ignore symlinked package names that have been registered with “yarn link”, e.g. “react”
Public Class Methods
required_files_in?(filenames)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 24 def self.required_files_in?(filenames) filenames.include?("package.json") end
required_files_message()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 28 def self.required_files_message "Repo must contain a package.json." end
Private Instance Methods
build_unfetchable_deps(unfetchable_deps)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 347 def build_unfetchable_deps(unfetchable_deps) return [] unless package_lock || yarn_lock unfetchable_deps.map do |name, path| PathDependencyBuilder.new( dependency_name: name, path: path, directory: directory, package_lock: package_lock, yarn_lock: yarn_lock ).dependency_file end end
convert_dependency_path_to_name(path, value)
click to toggle source
Re-write the glob name to the targeted dependency name (which is used in the lockfile), for example “parent-package/**/sub-dep/target-dep” > “target-dep”
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 218 def convert_dependency_path_to_name(path, value) # Picking the last two parts that might include a scope parts = path.split("/").last(2) parts.shift if parts.count == 2 && !parts.first.start_with?("@") [parts.join("/"), value] end
expanded_paths(path)
click to toggle source
Only expands globs one level deep, so path/*/ gets expanded to path/
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 304 def expanded_paths(path) ignored_paths = path.scan(/!\((.*?)\)/).flatten dir = directory.gsub(%r{(^/|/$)}, "") path = path.gsub(%r{^\./}, "").gsub(/!\(.*?\)/, "*") unglobbed_path = path.split("*").first&.gsub(%r{(?<=/)[^/]*$}, "") || "." repo_contents(dir: unglobbed_path, raise_errors: false). select { |file| file.type == "dir" }. map { |f| f.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "") }. select { |filename| File.fnmatch?(path, filename) }. reject { |fn| ignored_paths.any? { |p| fn.include?(p) } } end
fetch_files()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 34 def fetch_files fetched_files = [] fetched_files << package_json fetched_files << package_lock if package_lock && !ignore_package_lock? fetched_files << yarn_lock if yarn_lock fetched_files << shrinkwrap if shrinkwrap fetched_files << lerna_json if lerna_json fetched_files << npmrc if npmrc fetched_files << yarnrc if yarnrc fetched_files += workspace_package_jsons fetched_files += lerna_packages fetched_files += path_dependencies(fetched_files) fetched_files.uniq end
fetch_lerna_packages()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 243 def fetch_lerna_packages return [] unless parsed_lerna_json["packages"] dependency_files = [] workspace_paths(parsed_lerna_json["packages"]).each do |workspace| dependency_files += fetch_lerna_packages_from_path(workspace) end dependency_files end
fetch_lerna_packages_from_path(path, nested = false)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 255 def fetch_lerna_packages_from_path(path, nested = false) dependency_files = [] package_json_path = File.join(path, "package.json") begin dependency_files << fetch_file_from_host(package_json_path) dependency_files += [ fetch_file_if_present(File.join(path, "package-lock.json")), fetch_file_if_present(File.join(path, "yarn.lock")), fetch_file_if_present(File.join(path, "npm-shrinkwrap.json")) ].compact rescue Dependabot::DependencyFileNotFound matches_double_glob = parsed_lerna_json["packages"].any? do |globbed_path| next false unless globbed_path.include?("**") File.fnmatch?(globbed_path, path) end if matches_double_glob && !nested dependency_files += expanded_paths(File.join(path, "*")).flat_map do |nested_path| fetch_lerna_packages_from_path(nested_path, true) end end end dependency_files end
fetch_workspace_package_jsons()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 225 def fetch_workspace_package_jsons return [] unless parsed_package_json["workspaces"] package_json_files = [] workspace_paths(parsed_package_json["workspaces"]).each do |workspace| file = File.join(workspace, "package.json") begin package_json_files << fetch_file_from_host(file) rescue Dependabot::DependencyFileNotFound nil end end package_json_files end
ignore_package_lock?()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 341 def ignore_package_lock? return false unless npmrc npmrc.content.match?(/^package-lock\s*=\s*false/) end
lerna_json()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 104 def lerna_json @lerna_json ||= fetch_file_if_present("lerna.json")&. tap { |f| f.support_file = true } end
lerna_packages()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 113 def lerna_packages @lerna_packages ||= fetch_lerna_packages end
npmrc()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 66 def npmrc @npmrc ||= fetch_file_if_present(".npmrc")&. tap { |f| f.support_file = true } return @npmrc if @npmrc || directory == "/" # Loop through parent directories looking for an npmrc (1..directory.split("/").count).each do |i| @npmrc = fetch_file_from_host("../" * i + ".npmrc")&. tap { |f| f.support_file = true } break if @npmrc rescue Dependabot::DependencyFileNotFound # Ignore errors (.npmrc may not be present) nil end @npmrc end
package_json()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 50 def package_json @package_json ||= fetch_file_from_host("package.json") end
package_lock()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 54 def package_lock @package_lock ||= fetch_file_if_present("package-lock.json") end
parsed_lerna_json()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 361 def parsed_lerna_json return {} unless lerna_json JSON.parse(lerna_json.content) rescue JSON::ParserError raise Dependabot::DependencyFileNotParseable, lerna_json.path end
parsed_package_json()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 319 def parsed_package_json JSON.parse(package_json.content) rescue JSON::ParserError raise Dependabot::DependencyFileNotParseable, package_json.path end
parsed_package_lock()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 325 def parsed_package_lock return {} unless package_lock JSON.parse(package_lock.content) rescue JSON::ParserError {} end
parsed_shrinkwrap()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 333 def parsed_shrinkwrap return {} unless shrinkwrap JSON.parse(shrinkwrap.content) rescue JSON::ParserError {} end
path_dependencies(fetched_files)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 117 def path_dependencies(fetched_files) package_json_files = [] unfetchable_deps = [] path_dependency_details(fetched_files).each do |name, path| path = path.gsub(PATH_DEPENDENCY_CLEAN_REGEX, "") filename = path # NPM/Yarn support loading path dependencies from tarballs: # https://docs.npmjs.com/cli/pack.html filename = File.join(filename, "package.json") unless filename.end_with?(".tgz") cleaned_name = Pathname.new(filename).cleanpath.to_path next if fetched_files.map(&:name).include?(cleaned_name) begin file = fetch_file_from_host(filename, fetch_submodules: true) package_json_files << file rescue Dependabot::DependencyFileNotFound # Unfetchable tarballs should not be re-fetched as a package unfetchable_deps << [name, path] unless path.end_with?(".tgz") end end package_json_files += build_unfetchable_deps(unfetchable_deps) if package_json_files.any? package_json_files += path_dependencies(fetched_files + package_json_files) end package_json_files.tap { |fs| fs.each { |f| f.support_file = true } } end
path_dependency_details(fetched_files)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 149 def path_dependency_details(fetched_files) package_json_path_deps = [] fetched_files.each do |file| package_json_path_deps += path_dependency_details_from_manifest(file) end package_lock_path_deps = path_dependency_details_from_npm_lockfile( parsed_package_lock ) shrinkwrap_path_deps = path_dependency_details_from_npm_lockfile( parsed_shrinkwrap ) [ *package_json_path_deps, *package_lock_path_deps, *shrinkwrap_path_deps ].uniq end
path_dependency_details_from_manifest(file)
click to toggle source
rubocop:disable Metrics/PerceivedComplexity rubocop:disable Metrics/AbcSize
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 173 def path_dependency_details_from_manifest(file) return [] unless file.name.end_with?("package.json") current_dir = file.name.rpartition("/").first current_dir = nil if current_dir == "" dep_types = NpmAndYarn::FileParser::DEPENDENCY_TYPES parsed_manifest = JSON.parse(file.content) dependency_objects = parsed_manifest.values_at(*dep_types).compact # Fetch yarn "file:" path "resolutions" so the lockfile can be resolved resolution_objects = parsed_manifest.values_at("resolutions").compact manifest_objects = dependency_objects + resolution_objects raise Dependabot::DependencyFileNotParseable, file.path unless manifest_objects.all? { |o| o.is_a?(Hash) } resolution_deps = resolution_objects.flat_map(&:to_a). map do |path, value| convert_dependency_path_to_name(path, value) end path_starts = PATH_DEPENDENCY_STARTS (dependency_objects.flat_map(&:to_a) + resolution_deps). select { |_, v| v.is_a?(String) && v.start_with?(*path_starts) }. map do |name, path| path = path.gsub(PATH_DEPENDENCY_CLEAN_REGEX, "") path = File.join(current_dir, path) unless current_dir.nil? [name, Pathname.new(path).cleanpath.to_path] end rescue JSON::ParserError raise Dependabot::DependencyFileNotParseable, file.path end
path_dependency_details_from_npm_lockfile(parsed_lockfile)
click to toggle source
rubocop:enable Metrics/AbcSize rubocop:enable Metrics/PerceivedComplexity
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 207 def path_dependency_details_from_npm_lockfile(parsed_lockfile) path_starts = NPM_PATH_DEPENDENCY_STARTS parsed_lockfile.fetch("dependencies", []).to_a. select { |_, v| v.is_a?(Hash) }. select { |_, v| v.fetch("version", "").start_with?(*path_starts) }. map { |k, v| [k, v.fetch("version")] } end
shrinkwrap()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 62 def shrinkwrap @shrinkwrap ||= fetch_file_if_present("npm-shrinkwrap.json") end
workspace_package_jsons()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 109 def workspace_package_jsons @workspace_package_jsons ||= fetch_workspace_package_jsons end
workspace_paths(workspace_object)
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 286 def workspace_paths(workspace_object) paths_array = if workspace_object.is_a?(Hash) workspace_object.values_at("packages", "nohoist").flatten.compact elsif workspace_object.is_a?(Array) then workspace_object else [] # Invalid lerna.json, which must not be in use end paths_array.flat_map do |path| # The packages/!(not-this-package) syntax is unique to Yarn if path.include?("*") || path.include?("!(") expanded_paths(path) else path end end end
yarn_lock()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 58 def yarn_lock @yarn_lock ||= fetch_file_if_present("yarn.lock") end
yarnrc()
click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 85 def yarnrc @yarnrc ||= fetch_file_if_present(".yarnrc")&. tap { |f| f.support_file = true } return @yarnrc if @yarnrc || directory == "/" # Loop through parent directories looking for an yarnrc (1..directory.split("/").count).each do |i| @yarnrc = fetch_file_from_host("../" * i + ".yarnrc")&. tap { |f| f.support_file = true } break if @yarnrc rescue Dependabot::DependencyFileNotFound # Ignore errors (.yarnrc may not be present) nil end @yarnrc end