class Dependabot::NpmAndYarn::FileFetcher

Constants

NPM_PATH_DEPENDENCY_STARTS

Npm always prefixes file paths in the lockfile “version” with “file:” even when a naked path is used (e.g. “../dep”)

PATH_DEPENDENCY_CLEAN_REGEX
PATH_DEPENDENCY_STARTS

“link:” is only supported by Yarn but is interchangeable with “file:” when it specifies a path. Only include Yarn “”‘s that start with a path and ignore symlinked package names that have been registered with “yarn link”, e.g. “react

Public Class Methods

required_files_in?(filenames) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 24
def self.required_files_in?(filenames)
  filenames.include?("package.json")
end
required_files_message() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 28
def self.required_files_message
  "Repo must contain a package.json."
end

Private Instance Methods

build_unfetchable_deps(unfetchable_deps) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 347
def build_unfetchable_deps(unfetchable_deps)
  return [] unless package_lock || yarn_lock

  unfetchable_deps.map do |name, path|
    PathDependencyBuilder.new(
      dependency_name: name,
      path: path,
      directory: directory,
      package_lock: package_lock,
      yarn_lock: yarn_lock
    ).dependency_file
  end
end
convert_dependency_path_to_name(path, value) click to toggle source

Re-write the glob name to the targeted dependency name (which is used in the lockfile), for example “parent-package/**/sub-dep/target-dep” > “target-dep”

# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 218
def convert_dependency_path_to_name(path, value)
  # Picking the last two parts that might include a scope
  parts = path.split("/").last(2)
  parts.shift if parts.count == 2 && !parts.first.start_with?("@")
  [parts.join("/"), value]
end
expanded_paths(path) click to toggle source

Only expands globs one level deep, so path/*/ gets expanded to path/

# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 304
def expanded_paths(path)
  ignored_paths = path.scan(/!\((.*?)\)/).flatten

  dir = directory.gsub(%r{(^/|/$)}, "")
  path = path.gsub(%r{^\./}, "").gsub(/!\(.*?\)/, "*")
  unglobbed_path = path.split("*").first&.gsub(%r{(?<=/)[^/]*$}, "") ||
                   "."

  repo_contents(dir: unglobbed_path, raise_errors: false).
    select { |file| file.type == "dir" }.
    map { |f| f.path.gsub(%r{^/?#{Regexp.escape(dir)}/?}, "") }.
    select { |filename| File.fnmatch?(path, filename) }.
    reject { |fn| ignored_paths.any? { |p| fn.include?(p) } }
end
fetch_files() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 34
def fetch_files
  fetched_files = []
  fetched_files << package_json
  fetched_files << package_lock if package_lock && !ignore_package_lock?
  fetched_files << yarn_lock if yarn_lock
  fetched_files << shrinkwrap if shrinkwrap
  fetched_files << lerna_json if lerna_json
  fetched_files << npmrc if npmrc
  fetched_files << yarnrc if yarnrc
  fetched_files += workspace_package_jsons
  fetched_files += lerna_packages
  fetched_files += path_dependencies(fetched_files)

  fetched_files.uniq
end
fetch_lerna_packages() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 243
def fetch_lerna_packages
  return [] unless parsed_lerna_json["packages"]

  dependency_files = []

  workspace_paths(parsed_lerna_json["packages"]).each do |workspace|
    dependency_files += fetch_lerna_packages_from_path(workspace)
  end

  dependency_files
end
fetch_lerna_packages_from_path(path, nested = false) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 255
def fetch_lerna_packages_from_path(path, nested = false)
  dependency_files = []

  package_json_path = File.join(path, "package.json")

  begin
    dependency_files << fetch_file_from_host(package_json_path)
    dependency_files += [
      fetch_file_if_present(File.join(path, "package-lock.json")),
      fetch_file_if_present(File.join(path, "yarn.lock")),
      fetch_file_if_present(File.join(path, "npm-shrinkwrap.json"))
    ].compact
  rescue Dependabot::DependencyFileNotFound
    matches_double_glob =
      parsed_lerna_json["packages"].any? do |globbed_path|
        next false unless globbed_path.include?("**")

        File.fnmatch?(globbed_path, path)
      end

    if matches_double_glob && !nested
      dependency_files +=
        expanded_paths(File.join(path, "*")).flat_map do |nested_path|
          fetch_lerna_packages_from_path(nested_path, true)
        end
    end
  end

  dependency_files
end
fetch_workspace_package_jsons() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 225
def fetch_workspace_package_jsons
  return [] unless parsed_package_json["workspaces"]

  package_json_files = []

  workspace_paths(parsed_package_json["workspaces"]).each do |workspace|
    file = File.join(workspace, "package.json")

    begin
      package_json_files << fetch_file_from_host(file)
    rescue Dependabot::DependencyFileNotFound
      nil
    end
  end

  package_json_files
end
ignore_package_lock?() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 341
def ignore_package_lock?
  return false unless npmrc

  npmrc.content.match?(/^package-lock\s*=\s*false/)
end
lerna_json() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 104
def lerna_json
  @lerna_json ||= fetch_file_if_present("lerna.json")&.
                  tap { |f| f.support_file = true }
end
lerna_packages() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 113
def lerna_packages
  @lerna_packages ||= fetch_lerna_packages
end
npmrc() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 66
def npmrc
  @npmrc ||= fetch_file_if_present(".npmrc")&.
             tap { |f| f.support_file = true }

  return @npmrc if @npmrc || directory == "/"

  # Loop through parent directories looking for an npmrc
  (1..directory.split("/").count).each do |i|
    @npmrc = fetch_file_from_host("../" * i + ".npmrc")&.
             tap { |f| f.support_file = true }
    break if @npmrc
  rescue Dependabot::DependencyFileNotFound
    # Ignore errors (.npmrc may not be present)
    nil
  end

  @npmrc
end
package_json() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 50
def package_json
  @package_json ||= fetch_file_from_host("package.json")
end
package_lock() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 54
def package_lock
  @package_lock ||= fetch_file_if_present("package-lock.json")
end
parsed_lerna_json() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 361
def parsed_lerna_json
  return {} unless lerna_json

  JSON.parse(lerna_json.content)
rescue JSON::ParserError
  raise Dependabot::DependencyFileNotParseable, lerna_json.path
end
parsed_package_json() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 319
def parsed_package_json
  JSON.parse(package_json.content)
rescue JSON::ParserError
  raise Dependabot::DependencyFileNotParseable, package_json.path
end
parsed_package_lock() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 325
def parsed_package_lock
  return {} unless package_lock

  JSON.parse(package_lock.content)
rescue JSON::ParserError
  {}
end
parsed_shrinkwrap() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 333
def parsed_shrinkwrap
  return {} unless shrinkwrap

  JSON.parse(shrinkwrap.content)
rescue JSON::ParserError
  {}
end
path_dependencies(fetched_files) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 117
def path_dependencies(fetched_files)
  package_json_files = []
  unfetchable_deps = []

  path_dependency_details(fetched_files).each do |name, path|
    path = path.gsub(PATH_DEPENDENCY_CLEAN_REGEX, "")
    filename = path
    # NPM/Yarn support loading path dependencies from tarballs:
    # https://docs.npmjs.com/cli/pack.html
    filename = File.join(filename, "package.json") unless filename.end_with?(".tgz")
    cleaned_name = Pathname.new(filename).cleanpath.to_path
    next if fetched_files.map(&:name).include?(cleaned_name)

    begin
      file = fetch_file_from_host(filename, fetch_submodules: true)
      package_json_files << file
    rescue Dependabot::DependencyFileNotFound
      # Unfetchable tarballs should not be re-fetched as a package
      unfetchable_deps << [name, path] unless path.end_with?(".tgz")
    end
  end

  package_json_files += build_unfetchable_deps(unfetchable_deps)

  if package_json_files.any?
    package_json_files +=
      path_dependencies(fetched_files + package_json_files)
  end

  package_json_files.tap { |fs| fs.each { |f| f.support_file = true } }
end
path_dependency_details(fetched_files) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 149
def path_dependency_details(fetched_files)
  package_json_path_deps = []

  fetched_files.each do |file|
    package_json_path_deps +=
      path_dependency_details_from_manifest(file)
  end

  package_lock_path_deps = path_dependency_details_from_npm_lockfile(
    parsed_package_lock
  )
  shrinkwrap_path_deps = path_dependency_details_from_npm_lockfile(
    parsed_shrinkwrap
  )

  [
    *package_json_path_deps,
    *package_lock_path_deps,
    *shrinkwrap_path_deps
  ].uniq
end
path_dependency_details_from_manifest(file) click to toggle source

rubocop:disable Metrics/PerceivedComplexity rubocop:disable Metrics/AbcSize

# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 173
def path_dependency_details_from_manifest(file)
  return [] unless file.name.end_with?("package.json")

  current_dir = file.name.rpartition("/").first
  current_dir = nil if current_dir == ""

  dep_types = NpmAndYarn::FileParser::DEPENDENCY_TYPES
  parsed_manifest = JSON.parse(file.content)
  dependency_objects = parsed_manifest.values_at(*dep_types).compact
  # Fetch yarn "file:" path "resolutions" so the lockfile can be resolved
  resolution_objects = parsed_manifest.values_at("resolutions").compact
  manifest_objects = dependency_objects + resolution_objects

  raise Dependabot::DependencyFileNotParseable, file.path unless manifest_objects.all? { |o| o.is_a?(Hash) }

  resolution_deps = resolution_objects.flat_map(&:to_a).
                    map do |path, value|
                      convert_dependency_path_to_name(path, value)
                    end

  path_starts = PATH_DEPENDENCY_STARTS
  (dependency_objects.flat_map(&:to_a) + resolution_deps).
    select { |_, v| v.is_a?(String) && v.start_with?(*path_starts) }.
    map do |name, path|
      path = path.gsub(PATH_DEPENDENCY_CLEAN_REGEX, "")
      path = File.join(current_dir, path) unless current_dir.nil?
      [name, Pathname.new(path).cleanpath.to_path]
    end
rescue JSON::ParserError
  raise Dependabot::DependencyFileNotParseable, file.path
end
path_dependency_details_from_npm_lockfile(parsed_lockfile) click to toggle source

rubocop:enable Metrics/AbcSize rubocop:enable Metrics/PerceivedComplexity

# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 207
def path_dependency_details_from_npm_lockfile(parsed_lockfile)
  path_starts = NPM_PATH_DEPENDENCY_STARTS
  parsed_lockfile.fetch("dependencies", []).to_a.
    select { |_, v| v.is_a?(Hash) }.
    select { |_, v| v.fetch("version", "").start_with?(*path_starts) }.
    map { |k, v| [k, v.fetch("version")] }
end
shrinkwrap() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 62
def shrinkwrap
  @shrinkwrap ||= fetch_file_if_present("npm-shrinkwrap.json")
end
workspace_package_jsons() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 109
def workspace_package_jsons
  @workspace_package_jsons ||= fetch_workspace_package_jsons
end
workspace_paths(workspace_object) click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 286
def workspace_paths(workspace_object)
  paths_array =
    if workspace_object.is_a?(Hash)
      workspace_object.values_at("packages", "nohoist").flatten.compact
    elsif workspace_object.is_a?(Array) then workspace_object
    else [] # Invalid lerna.json, which must not be in use
    end

  paths_array.flat_map do |path|
    # The packages/!(not-this-package) syntax is unique to Yarn
    if path.include?("*") || path.include?("!(")
      expanded_paths(path)
    else path
    end
  end
end
yarn_lock() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 58
def yarn_lock
  @yarn_lock ||= fetch_file_if_present("yarn.lock")
end
yarnrc() click to toggle source
# File lib/dependabot/npm_and_yarn/file_fetcher.rb, line 85
def yarnrc
  @yarnrc ||= fetch_file_if_present(".yarnrc")&.
             tap { |f| f.support_file = true }

  return @yarnrc if @yarnrc || directory == "/"

  # Loop through parent directories looking for an yarnrc
  (1..directory.split("/").count).each do |i|
    @yarnrc = fetch_file_from_host("../" * i + ".yarnrc")&.
             tap { |f| f.support_file = true }
    break if @yarnrc
  rescue Dependabot::DependencyFileNotFound
    # Ignore errors (.yarnrc may not be present)
    nil
  end

  @yarnrc
end