class Aspen::CustomGrammar::Lexer

Public Class Methods

tokenize(code, env={}) click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 6
def self.tokenize(code, env={})
  new.tokenize(code, env)
end

Public Instance Methods

no_match(scanner, state) click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 69
def no_match(scanner, state)
  raise Aspen::LexError,
          Aspen::Errors.messages(:unexpected_token, scanner, state)
end
pop_state() click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 65
def pop_state
  stack.pop
end
push_state(state) click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 61
def push_state(state)
  stack.push(state)
end
stack() click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 53
def stack
  @stack ||= []
end
state() click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 57
def state
  stack.last || :default
end
tokenize(code, env={}) click to toggle source
# File lib/aspen/custom_grammar/lexer.rb, line 10
def tokenize(code, env={})
  scanner = StringScanner.new(code)
  tokens = []

  # puts "tokens: #{tokens} | state: #{state} | stack: #{stack}"

  until scanner.eos?
    case state
    when :default then
      if scanner.scan(/\(/)
        tokens << [:OPEN_PARENS]
        push_state :capture_segment
      elsif scanner.scan(/[[[:alnum:]][[:blank:]]\!"\#$%&'*+,\-.\/:;<=>?@\[\\\]^_‘\{\|\}~]+/)
        tokens << [:BARE, scanner.matched]
      else
        no_match(scanner, state)
      end

    when :capture_segment
      if scanner.scan(/\s+/)
        # NO OP
      elsif scanner.scan(/^(numeric|integer|float|string)/)
        tokens << [:TYPE, scanner.matched]
      elsif scanner.scan(Aspen::Lexer::PASCAL_CASE)
        tokens << [:TYPE, ["node", scanner.matched]]
      # TODO: This should only accept legal variable names, like `hello_01`
      elsif scanner.scan(/^\w+/)
        tokens << [:VAR_NAME, scanner.matched]
      elsif scanner.scan(/\)/)
        tokens << [:CLOSE_PARENS]
        pop_state
      else
        no_match(scanner, state)
      end

    else
      raise Aspen::LexError, "There is no matcher for state #{state.inspect}."
    end
  end

  tokens
end