class Aspen::CustomGrammar::Lexer
Public Class Methods
tokenize(code, env={})
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 6 def self.tokenize(code, env={}) new.tokenize(code, env) end
Public Instance Methods
no_match(scanner, state)
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 69 def no_match(scanner, state) raise Aspen::LexError, Aspen::Errors.messages(:unexpected_token, scanner, state) end
pop_state()
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 65 def pop_state stack.pop end
push_state(state)
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 61 def push_state(state) stack.push(state) end
stack()
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 53 def stack @stack ||= [] end
state()
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 57 def state stack.last || :default end
tokenize(code, env={})
click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 10 def tokenize(code, env={}) scanner = StringScanner.new(code) tokens = [] # puts "tokens: #{tokens} | state: #{state} | stack: #{stack}" until scanner.eos? case state when :default then if scanner.scan(/\(/) tokens << [:OPEN_PARENS] push_state :capture_segment elsif scanner.scan(/[[[:alnum:]][[:blank:]]\!"\#$%&'*+,\-.\/:;<=>?@\[\\\]^_‘\{\|\}~]+/) tokens << [:BARE, scanner.matched] else no_match(scanner, state) end when :capture_segment if scanner.scan(/\s+/) # NO OP elsif scanner.scan(/^(numeric|integer|float|string)/) tokens << [:TYPE, scanner.matched] elsif scanner.scan(Aspen::Lexer::PASCAL_CASE) tokens << [:TYPE, ["node", scanner.matched]] # TODO: This should only accept legal variable names, like `hello_01` elsif scanner.scan(/^\w+/) tokens << [:VAR_NAME, scanner.matched] elsif scanner.scan(/\)/) tokens << [:CLOSE_PARENS] pop_state else no_match(scanner, state) end else raise Aspen::LexError, "There is no matcher for state #{state.inspect}." end end tokens end