class SrcLexer::Lexer

Constants

END_TOKEN
NUMBER_REGEX
STRING_REGEX

Attributes

comment_markers[R]
keywords[R]
line_comment_marker[R]
str[R]
string_literal_marker[R]
symbols[R]
tokens[R]

Public Class Methods

new(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers) click to toggle source
# File lib/src_lexer.rb, line 24
def initialize(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers)
  @keywords = (keywords ? keywords.uniq.compact : [])
  @symbols = (symbols ? symbols.uniq.compact : [])
  @string_literal_marker = string_literal_marker
  @line_comment_marker = line_comment_marker
  @comment_markers = comment_markers
end

Public Instance Methods

analyze(str) click to toggle source
# File lib/src_lexer.rb, line 32
def analyze(str)
  @str = str
  tokenize
end
pop_token() click to toggle source
# File lib/src_lexer.rb, line 37
def pop_token
  token = @tokens.shift
  return END_TOKEN if token.nil?
  case token[0]
  when NUMBER_REGEX
    [:NUMBER, Token.new(token[0], token[1], token[2])]
  when STRING_REGEX
    [:STRING, Token.new(token[0], token[1], token[2])]
  else
    [is_reserved?(token[0]) ? token[0] : :IDENT, Token.new(token[0], token[1], token[2])]
  end
end

Private Instance Methods

is_reserved?(token) click to toggle source
# File lib/src_lexer.rb, line 178
def is_reserved?(token)
  @keywords.include?(token) || @symbols.include?(token)
end
tokenize() click to toggle source
# File lib/src_lexer.rb, line 139
def tokenize()
  @tokens = []
  iterator = StringIterator.new(@str)

  while iterator < @str.length do
    if iterator.is_white_space then
      @tokens.push iterator.shift if iterator.marked?
      iterator.move_next
    elsif @line_comment_marker && iterator.is(@line_comment_marker) then
      @tokens.push iterator.shift if iterator.marked?
      iterator.move_to_the_end_of_the_line
      iterator.move_next
    elsif @comment_markers && iterator.is(@comment_markers[0]) then
      @tokens.push iterator.shift if iterator.marked?
      iterator.move_to(@comment_markers[1])
      iterator.move_next
    elsif @string_literal_marker && iterator.is(@string_literal_marker[0]) then
      @tokens.push iterator.shift if iterator.marked?
      iterator.mark_set
      iterator.move_next
      iterator.move_to(@string_literal_marker[1])
      iterator.move_next
      @tokens.push iterator.shift
    elsif iterator.is_in(@symbols) then
      @tokens.push iterator.shift if iterator.marked?
      iterator.mark_set
      @symbols.find { |symbol| iterator.is(symbol) }.length.times { iterator.move_next }
      @tokens.push iterator.shift
    elsif !iterator.marked? then
      iterator.mark_set
    else
      iterator.move_next
    end
  end
  @tokens.push iterator.shift if iterator.marked?
  
  return self
end