class Starscope::DB
Constants
- DB_FORMAT
- EXTRACTORS
- FRAGMENT
- LANGS
Public Class Methods
new(output, config = {})
click to toggle source
# File lib/starscope/db.rb, line 35 def initialize(output, config = {}) @output = output @meta = { paths: [], files: {}, excludes: [], langs: LANGS.dup, version: Starscope::VERSION } @tables = {} @config = config end
Private Class Methods
extractors()
click to toggle source
# File lib/starscope/db.rb, line 337 def extractors # so we can stub it in tests EXTRACTORS end
normalize_fnmatch(path)
click to toggle source
File.fnmatch treats a “**” to match files and directories recursively
# File lib/starscope/db.rb, line 309 def normalize_fnmatch(path) if path == '.' '**' elsif File.directory?(path) File.join(path, '**') else path end end
normalize_glob(path)
click to toggle source
Dir.glob treats a “**” to only match directories recursively; you need “*/” to match all files recursively
# File lib/starscope/db.rb, line 321 def normalize_glob(path) if path == '.' File.join('**', '*') elsif File.directory?(path) File.join(path, '**', '*') else path end end
normalize_record(file, name, args)
click to toggle source
# File lib/starscope/db.rb, line 331 def normalize_record(file, name, args) args[:file] = file args[:name] = Array(name).map(&:to_sym) args end
Public Instance Methods
add_excludes(paths)
click to toggle source
# File lib/starscope/db.rb, line 68 def add_excludes(paths) @output.extra("Excluding files in paths #{paths}...") @meta[:paths] -= paths.map { |p| self.class.normalize_glob(p) } paths = paths.map { |p| self.class.normalize_fnmatch(p) } @meta[:excludes] += paths @meta[:excludes].uniq! @all_excludes = nil # clear cache excluded = @meta[:files].keys.select { |name| matches_exclude?(name, paths) } remove_files(excluded) end
add_paths(paths)
click to toggle source
# File lib/starscope/db.rb, line 80 def add_paths(paths) @output.extra("Adding files in paths #{paths}...") @meta[:excludes] -= paths.map { |p| self.class.normalize_fnmatch(p) } @all_excludes = nil # clear cache paths = paths.map { |p| self.class.normalize_glob(p) } @meta[:paths] += paths @meta[:paths].uniq! files = Dir.glob(paths).select { |f| File.file? f } files.delete_if { |f| matches_exclude?(f) } return if files.empty? @output.new_pbar('Building', files.length) add_files(files) @output.finish_pbar end
drop_all()
click to toggle source
# File lib/starscope/db.rb, line 143 def drop_all @meta[:files] = {} @tables = {} end
line_for_record(rec)
click to toggle source
# File lib/starscope/db.rb, line 117 def line_for_record(rec) return rec[:line] if rec[:line] file = @meta[:files][rec[:file]] return file[:lines][rec[:line_no] - 1] if file[:lines] end
load(filename)
click to toggle source
returns true iff the database was already in the most recent format
# File lib/starscope/db.rb, line 44 def load(filename) @output.extra("Reading database from `#{filename}`... ") current_fmt = open_db(filename) fixup if current_fmt current_fmt end
metadata(key = nil)
click to toggle source
# File lib/starscope/db.rb, line 135 def metadata(key = nil) return @meta.keys if key.nil? raise NoTableError unless @meta[key] @meta[key] end
records(table)
click to toggle source
# File lib/starscope/db.rb, line 129 def records(table) raise NoTableError unless @tables[table] @tables[table] end
save(filename)
click to toggle source
# File lib/starscope/db.rb, line 51 def save(filename) @output.extra("Writing database to `#{filename}`...") # regardless of what the old version was, the new version is written by us @meta[:version] = Starscope::VERSION @meta[:langs].merge!(LANGS) File.open(filename, 'wb') do |file| Zlib::GzipWriter.wrap(file) do |stream| stream.puts DB_FORMAT stream.puts Oj.dump @meta stream.puts Oj.dump @tables end end end
tables()
click to toggle source
# File lib/starscope/db.rb, line 125 def tables @tables.keys end
update()
click to toggle source
# File lib/starscope/db.rb, line 95 def update changes = @meta[:files].keys.group_by { |name| file_changed(name) } changes[:modified] ||= [] changes[:deleted] ||= [] new_files = (Dir.glob(@meta[:paths]).select { |f| File.file? f }) - @meta[:files].keys new_files.delete_if { |f| matches_exclude?(f) } if changes[:deleted].empty? && changes[:modified].empty? && new_files.empty? @output.normal('No changes detected.') return false end @output.new_pbar('Updating', changes[:modified].length + new_files.length) remove_files(changes[:deleted]) update_files(changes[:modified]) add_files(new_files) @output.finish_pbar true end
Private Instance Methods
add_files(files)
click to toggle source
# File lib/starscope/db.rb, line 206 def add_files(files) files.each do |file| @output.extra("Adding `#{file}`") parse_file(file) @output.inc_pbar end end
all_excludes()
click to toggle source
# File lib/starscope/db.rb, line 198 def all_excludes @all_excludes ||= @meta[:excludes] + (@config[:excludes] || []).map { |x| self.class.normalize_fnmatch(x) } end
extract_file(extractor, file, line_cache, lines)
click to toggle source
# File lib/starscope/db.rb, line 250 def extract_file(extractor, file, line_cache, lines) fragment_cache = {} extractor_metadata = extractor.extract(file, File.read(file)) do |tbl, name, args| case tbl when FRAGMENT fragment_cache[name] ||= [] fragment_cache[name] << args else @tables[tbl] ||= [] @tables[tbl] << self.class.normalize_record(file, name, args) if args[:line_no] line_cache ||= File.readlines(file) lines ||= Array.new(line_cache.length) lines[args[:line_no] - 1] = line_cache[args[:line_no] - 1].chomp end end end fragment_cache.each do |lang, frags| extract_file(Starscope::FragmentExtractor.new(lang, frags), file, line_cache, lines) @meta[:files][file][:sublangs] << lang end rescue => e @output.normal("#{extractor} raised \"#{e}\" while extracting #{file}") ensure # metadata must be created for any record that was inserted into a tbl # even if there was later a rescued exception @meta[:files][file][:lang] = extractor.name.split('::').last.to_sym @meta[:files][file][:lines] = lines if extractor_metadata.is_a? Hash @meta[:files][file] = extractor_metadata.merge!(@meta[:files][file]) end end
file_changed(name)
click to toggle source
# File lib/starscope/db.rb, line 288 def file_changed(name) file_meta = @meta[:files][name] if matches_exclude?(name) || !File.exist?(name) || !File.file?(name) :deleted elsif (file_meta[:last_updated] < File.mtime(name).to_i) || language_out_of_date(file_meta[:lang]) || (file_meta[:sublangs] || []).any? { |lang| language_out_of_date(lang) } :modified else :unchanged end end
fixup()
click to toggle source
# File lib/starscope/db.rb, line 193 def fixup # misc things that were't worth bumping the format for, but which might not be written by old versions @meta[:langs] ||= {} end
language_out_of_date(lang)
click to toggle source
# File lib/starscope/db.rb, line 301 def language_out_of_date(lang) return false unless lang return true unless LANGS[lang] (@meta[:langs][lang] || 0) < LANGS[lang] end
matches_exclude?(file, patterns = all_excludes)
click to toggle source
# File lib/starscope/db.rb, line 202 def matches_exclude?(file, patterns = all_excludes) patterns.map { |p| File.fnmatch(p, file) }.any? end
open_db(filename)
click to toggle source
# File lib/starscope/db.rb, line 150 def open_db(filename) File.open(filename, 'rb') do |file| begin Zlib::GzipReader.wrap(file) do |stream| parse_db(stream) end rescue Zlib::GzipFile::Error file.rewind parse_db(file) end end end
parse_db(stream)
click to toggle source
returns true iff the database is in the most recent format
# File lib/starscope/db.rb, line 164 def parse_db(stream) case stream.gets.to_i when DB_FORMAT @meta = Oj.load(stream.gets) @tables = Oj.load(stream.gets) return true when 3..4 # Old format, so read the directories segment then rebuild add_paths(Oj.load(stream.gets)) return false when 0..2 # Old format (pre-json), so read the directories segment then rebuild len = stream.gets.to_i add_paths(Marshal.load(stream.read(len))) return false else raise UnknownDBFormatError end rescue Oj::ParseError stream.rewind raise unless stream.gets.to_i == DB_FORMAT # try reading as formated json, which is much slower, but it is sometimes # useful to be able to directly read your db objects = [] Oj.load(stream) { |obj| objects << obj } @meta, @tables = objects return true end
parse_file(file)
click to toggle source
# File lib/starscope/db.rb, line 230 def parse_file(file) @meta[:files][file] = { last_updated: File.mtime(file).to_i } self.class.extractors.each do |extractor| begin next unless extractor.match_file file rescue => e @output.normal("#{extractor} raised \"#{e}\" while matching #{file}") next end line_cache = File.readlines(file) lines = Array.new(line_cache.length) @meta[:files][file][:sublangs] = [] extract_file(extractor, file, line_cache, lines) break end end
remove_files(files)
click to toggle source
# File lib/starscope/db.rb, line 214 def remove_files(files) files.each do |file| @output.extra("Removing `#{file}`") @meta[:files].delete(file) end files = files.to_set @tables.each do |_, tbl| tbl.delete_if { |val| files.include?(val[:file]) } end end
update_files(files)
click to toggle source
# File lib/starscope/db.rb, line 225 def update_files(files) remove_files(files) add_files(files) end