class Starscope::DB

Constants

DB_FORMAT
EXTRACTORS
FRAGMENT
LANGS

Public Class Methods

new(output, config = {}) click to toggle source
# File lib/starscope/db.rb, line 35
def initialize(output, config = {})
  @output = output
  @meta = { paths: [], files: {}, excludes: [],
            langs: LANGS.dup, version: Starscope::VERSION }
  @tables = {}
  @config = config
end

Private Class Methods

extractors() click to toggle source
# File lib/starscope/db.rb, line 337
def extractors # so we can stub it in tests
  EXTRACTORS
end
normalize_fnmatch(path) click to toggle source

File.fnmatch treats a “**” to match files and directories recursively

# File lib/starscope/db.rb, line 309
def normalize_fnmatch(path)
  if path == '.'
    '**'
  elsif File.directory?(path)
    File.join(path, '**')
  else
    path
  end
end
normalize_glob(path) click to toggle source

Dir.glob treats a “**” to only match directories recursively; you need “*/” to match all files recursively

# File lib/starscope/db.rb, line 321
def normalize_glob(path)
  if path == '.'
    File.join('**', '*')
  elsif File.directory?(path)
    File.join(path, '**', '*')
  else
    path
  end
end
normalize_record(file, name, args) click to toggle source
# File lib/starscope/db.rb, line 331
def normalize_record(file, name, args)
  args[:file] = file
  args[:name] = Array(name).map(&:to_sym)
  args
end

Public Instance Methods

add_excludes(paths) click to toggle source
# File lib/starscope/db.rb, line 68
def add_excludes(paths)
  @output.extra("Excluding files in paths #{paths}...")
  @meta[:paths] -= paths.map { |p| self.class.normalize_glob(p) }
  paths = paths.map { |p| self.class.normalize_fnmatch(p) }
  @meta[:excludes] += paths
  @meta[:excludes].uniq!
  @all_excludes = nil # clear cache

  excluded = @meta[:files].keys.select { |name| matches_exclude?(name, paths) }
  remove_files(excluded)
end
add_paths(paths) click to toggle source
# File lib/starscope/db.rb, line 80
def add_paths(paths)
  @output.extra("Adding files in paths #{paths}...")
  @meta[:excludes] -= paths.map { |p| self.class.normalize_fnmatch(p) }
  @all_excludes = nil # clear cache
  paths = paths.map { |p| self.class.normalize_glob(p) }
  @meta[:paths] += paths
  @meta[:paths].uniq!
  files = Dir.glob(paths).select { |f| File.file? f }
  files.delete_if { |f| matches_exclude?(f) }
  return if files.empty?
  @output.new_pbar('Building', files.length)
  add_files(files)
  @output.finish_pbar
end
drop_all() click to toggle source
# File lib/starscope/db.rb, line 143
def drop_all
  @meta[:files] = {}
  @tables = {}
end
line_for_record(rec) click to toggle source
# File lib/starscope/db.rb, line 117
def line_for_record(rec)
  return rec[:line] if rec[:line]

  file = @meta[:files][rec[:file]]

  return file[:lines][rec[:line_no] - 1] if file[:lines]
end
load(filename) click to toggle source

returns true iff the database was already in the most recent format

# File lib/starscope/db.rb, line 44
def load(filename)
  @output.extra("Reading database from `#{filename}`... ")
  current_fmt = open_db(filename)
  fixup if current_fmt
  current_fmt
end
metadata(key = nil) click to toggle source
# File lib/starscope/db.rb, line 135
def metadata(key = nil)
  return @meta.keys if key.nil?

  raise NoTableError unless @meta[key]

  @meta[key]
end
records(table) click to toggle source
# File lib/starscope/db.rb, line 129
def records(table)
  raise NoTableError unless @tables[table]

  @tables[table]
end
save(filename) click to toggle source
# File lib/starscope/db.rb, line 51
def save(filename)
  @output.extra("Writing database to `#{filename}`...")

  # regardless of what the old version was, the new version is written by us
  @meta[:version] = Starscope::VERSION

  @meta[:langs].merge!(LANGS)

  File.open(filename, 'wb') do |file|
    Zlib::GzipWriter.wrap(file) do |stream|
      stream.puts DB_FORMAT
      stream.puts Oj.dump @meta
      stream.puts Oj.dump @tables
    end
  end
end
tables() click to toggle source
# File lib/starscope/db.rb, line 125
def tables
  @tables.keys
end
update() click to toggle source
# File lib/starscope/db.rb, line 95
def update
  changes = @meta[:files].keys.group_by { |name| file_changed(name) }
  changes[:modified] ||= []
  changes[:deleted] ||= []

  new_files = (Dir.glob(@meta[:paths]).select { |f| File.file? f }) - @meta[:files].keys
  new_files.delete_if { |f| matches_exclude?(f) }

  if changes[:deleted].empty? && changes[:modified].empty? && new_files.empty?
    @output.normal('No changes detected.')
    return false
  end

  @output.new_pbar('Updating', changes[:modified].length + new_files.length)
  remove_files(changes[:deleted])
  update_files(changes[:modified])
  add_files(new_files)
  @output.finish_pbar

  true
end

Private Instance Methods

add_files(files) click to toggle source
# File lib/starscope/db.rb, line 206
def add_files(files)
  files.each do |file|
    @output.extra("Adding `#{file}`")
    parse_file(file)
    @output.inc_pbar
  end
end
all_excludes() click to toggle source
# File lib/starscope/db.rb, line 198
def all_excludes
  @all_excludes ||= @meta[:excludes] + (@config[:excludes] || []).map { |x| self.class.normalize_fnmatch(x) }
end
extract_file(extractor, file, line_cache, lines) click to toggle source
# File lib/starscope/db.rb, line 250
def extract_file(extractor, file, line_cache, lines)
  fragment_cache = {}

  extractor_metadata = extractor.extract(file, File.read(file)) do |tbl, name, args|
    case tbl
    when FRAGMENT
      fragment_cache[name] ||= []
      fragment_cache[name] << args
    else
      @tables[tbl] ||= []
      @tables[tbl] << self.class.normalize_record(file, name, args)

      if args[:line_no]
        line_cache ||= File.readlines(file)
        lines ||= Array.new(line_cache.length)
        lines[args[:line_no] - 1] = line_cache[args[:line_no] - 1].chomp
      end
    end
  end

  fragment_cache.each do |lang, frags|
    extract_file(Starscope::FragmentExtractor.new(lang, frags), file, line_cache, lines)
    @meta[:files][file][:sublangs] << lang
  end

rescue => e
  @output.normal("#{extractor} raised \"#{e}\" while extracting #{file}")
ensure
  # metadata must be created for any record that was inserted into a tbl
  # even if there was later a rescued exception
  @meta[:files][file][:lang] = extractor.name.split('::').last.to_sym
  @meta[:files][file][:lines] = lines

  if extractor_metadata.is_a? Hash
    @meta[:files][file] = extractor_metadata.merge!(@meta[:files][file])
  end
end
file_changed(name) click to toggle source
# File lib/starscope/db.rb, line 288
def file_changed(name)
  file_meta = @meta[:files][name]
  if matches_exclude?(name) || !File.exist?(name) || !File.file?(name)
    :deleted
  elsif (file_meta[:last_updated] < File.mtime(name).to_i) ||
        language_out_of_date(file_meta[:lang]) ||
        (file_meta[:sublangs] || []).any? { |lang| language_out_of_date(lang) }
    :modified
  else
    :unchanged
  end
end
fixup() click to toggle source
# File lib/starscope/db.rb, line 193
def fixup
  # misc things that were't worth bumping the format for, but which might not be written by old versions
  @meta[:langs] ||= {}
end
language_out_of_date(lang) click to toggle source
# File lib/starscope/db.rb, line 301
def language_out_of_date(lang)
  return false unless lang
  return true unless LANGS[lang]
  (@meta[:langs][lang] || 0) < LANGS[lang]
end
matches_exclude?(file, patterns = all_excludes) click to toggle source
# File lib/starscope/db.rb, line 202
def matches_exclude?(file, patterns = all_excludes)
  patterns.map { |p| File.fnmatch(p, file) }.any?
end
open_db(filename) click to toggle source
# File lib/starscope/db.rb, line 150
def open_db(filename)
  File.open(filename, 'rb') do |file|
    begin
      Zlib::GzipReader.wrap(file) do |stream|
        parse_db(stream)
      end
    rescue Zlib::GzipFile::Error
      file.rewind
      parse_db(file)
    end
  end
end
parse_db(stream) click to toggle source

returns true iff the database is in the most recent format

# File lib/starscope/db.rb, line 164
def parse_db(stream)
  case stream.gets.to_i
  when DB_FORMAT
    @meta   = Oj.load(stream.gets)
    @tables = Oj.load(stream.gets)
    return true
  when 3..4
    # Old format, so read the directories segment then rebuild
    add_paths(Oj.load(stream.gets))
    return false
  when 0..2
    # Old format (pre-json), so read the directories segment then rebuild
    len = stream.gets.to_i
    add_paths(Marshal.load(stream.read(len)))
    return false
  else
    raise UnknownDBFormatError
  end
rescue Oj::ParseError
  stream.rewind
  raise unless stream.gets.to_i == DB_FORMAT
  # try reading as formated json, which is much slower, but it is sometimes
  # useful to be able to directly read your db
  objects = []
  Oj.load(stream) { |obj| objects << obj }
  @meta, @tables = objects
  return true
end
parse_file(file) click to toggle source
# File lib/starscope/db.rb, line 230
def parse_file(file)
  @meta[:files][file] = { last_updated: File.mtime(file).to_i }

  self.class.extractors.each do |extractor|
    begin
      next unless extractor.match_file file
    rescue => e
      @output.normal("#{extractor} raised \"#{e}\" while matching #{file}")
      next
    end

    line_cache = File.readlines(file)
    lines = Array.new(line_cache.length)
    @meta[:files][file][:sublangs] = []
    extract_file(extractor, file, line_cache, lines)

    break
  end
end
remove_files(files) click to toggle source
# File lib/starscope/db.rb, line 214
def remove_files(files)
  files.each do |file|
    @output.extra("Removing `#{file}`")
    @meta[:files].delete(file)
  end
  files = files.to_set
  @tables.each do |_, tbl|
    tbl.delete_if { |val| files.include?(val[:file]) }
  end
end
update_files(files) click to toggle source
# File lib/starscope/db.rb, line 225
def update_files(files)
  remove_files(files)
  add_files(files)
end