class Dentaku::TokenScanner
Constants
- DATE_TIME_REGEXP
Attributes
case_sensitive[R]
Public Class Methods
access()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 143 def access names = { lbracket: '[', rbracket: ']' }.invert new(:access, '\[|\]', lambda { |raw| names[raw] }) end
array()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 138 def array names = { array_start: '{', array_end: '}', }.invert new(:array, '\{|\}|,', lambda { |raw| names[raw] }) end
available_scanners()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 35 def available_scanners [ :null, :whitespace, :datetime, # before numeric so it can pick up timestamps :numeric, :hexadecimal, :double_quoted_string, :single_quoted_string, :negate, :combinator, :operator, :grouping, :array, :access, :case_statement, :comparator, :boolean, :function, :identifier, :quoted_identifier ] end
boolean()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 167 def boolean new(:logical, '(true|false)\b', lambda { |raw| raw.strip.downcase == 'true' }) end
case_statement()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 148 def case_statement names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert new(:case, '(case|end|then|when|else)\b', lambda { |raw| names[raw.downcase] }) end
combinator()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 159 def combinator names = { and: '&&', or: '||' }.invert new(:combinator, '(and|or|&&|\|\|)\s', lambda { |raw| norm = raw.strip.downcase names.fetch(norm) { norm.to_sym } }) end
comparator()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 153 def comparator names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert alternate = { ne: '<>', eq: '==' }.invert new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] }) end
datetime()
click to toggle source
NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason
# File lib/dentaku/token_scanner.rb, line 93 def datetime new(:datetime, DATE_TIME_REGEXP, lambda { |raw| Time.parse(raw).to_datetime }) end
double_quoted_string()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 107 def double_quoted_string new(:string, '"[^"]*"', lambda { |raw| raw.gsub(/^"|"$/, '') }) end
function()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 171 def function new(:function, '\w+!?\s*\(', lambda do |raw| function_name = raw.gsub('(', '') [ Token.new(:function, function_name.strip.downcase.to_sym, function_name), Token.new(:grouping, :open, '(') ] end) end
grouping()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 133 def grouping names = { open: '(', close: ')', comma: ',' }.invert new(:grouping, '\(|\)|,', lambda { |raw| names[raw] }) end
hexadecimal()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 103 def hexadecimal new(:numeric, '(0x[0-9a-f]+)\b', lambda { |raw| raw[2..-1].to_i(16) }) end
identifier()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 181 def identifier new(:identifier, '[[[:word:]]\.]+\b', lambda { |raw| standardize_case(raw.strip) }) end
negate()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 115 def negate new(:operator, '-', lambda { |raw| :negate }, lambda { |last_token| last_token.nil? || last_token.is?(:operator) || last_token.is?(:comparator) || last_token.is?(:combinator) || last_token.value == :open || last_token.value == :comma }) end
new(category, regexp, converter = nil, condition = nil)
click to toggle source
# File lib/dentaku/token_scanner.rb, line 12 def initialize(category, regexp, converter = nil, condition = nil) @category = category @regexp = %r{\A(#{ regexp })}i @converter = converter @condition = condition || ->(*) { true } end
null()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 88 def null new(:null, 'null\b') end
numeric()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 97 def numeric new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', lambda { |raw| raw =~ /(\.|e|E)/ ? BigDecimal(raw) : raw.to_i }) end
operator()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 126 def operator names = { pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%', bitor: '|', bitand: '&', bitshiftleft: '<<', bitshiftright: '>>' }.invert new(:operator, '\^|\+|-|\*|\/|%|\||&|<<|>>', lambda { |raw| names[raw] }) end
quoted_identifier()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 185 def quoted_identifier new(:identifier, '`[^`]*`', lambda { |raw| raw.gsub(/^`|`$/, '') }) end
register_default_scanners()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 59 def register_default_scanners register_scanners(available_scanners) end
register_scanner(id, scanner)
click to toggle source
# File lib/dentaku/token_scanner.rb, line 69 def register_scanner(id, scanner) @scanners[id] = scanner end
register_scanners(scanner_ids)
click to toggle source
# File lib/dentaku/token_scanner.rb, line 63 def register_scanners(scanner_ids) @scanners = scanner_ids.each_with_object({}) do |id, scanners| scanners[id] = self.send(id) end end
scanners(options = {})
click to toggle source
# File lib/dentaku/token_scanner.rb, line 77 def scanners(options = {}) @case_sensitive = options.fetch(:case_sensitive, false) raw_date_literals = options.fetch(:raw_date_literals, true) @scanners.select { |k, _| raw_date_literals || k != :datetime }.values end
scanners=(scanner_ids)
click to toggle source
# File lib/dentaku/token_scanner.rb, line 73 def scanners=(scanner_ids) @scanners.select! { |k, v| scanner_ids.include?(k) } end
single_quoted_string()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 111 def single_quoted_string new(:string, "'[^']*'", lambda { |raw| raw.gsub(/^'|'$/, '') }) end
whitespace()
click to toggle source
# File lib/dentaku/token_scanner.rb, line 84 def whitespace new(:whitespace, '\s+') end
Public Instance Methods
scan(string, last_token = nil)
click to toggle source
# File lib/dentaku/token_scanner.rb, line 19 def scan(string, last_token = nil) if (m = @regexp.match(string)) && @condition.call(last_token) value = raw = m.to_s value = @converter.call(raw) if @converter return Array(value).map do |v| Token === v ? v : Token.new(@category, v, raw) end end false end