class SrcLexer::Lexer
Constants
- END_TOKEN
- NUMBER_REGEX
- STRING_REGEX
Attributes
comment_markers[R]
keywords[R]
line_comment_marker[R]
str[R]
string_literal_marker[R]
symbols[R]
tokens[R]
Public Class Methods
new(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers)
click to toggle source
# File lib/src_lexer.rb, line 24 def initialize(keywords, symbols, string_literal_marker, line_comment_marker, comment_markers) @keywords = (keywords ? keywords.uniq.compact : []) @symbols = (symbols ? symbols.uniq.compact : []) @string_literal_marker = string_literal_marker @line_comment_marker = line_comment_marker @comment_markers = comment_markers end
Public Instance Methods
analyze(str)
click to toggle source
# File lib/src_lexer.rb, line 32 def analyze(str) @str = str tokenize end
pop_token()
click to toggle source
# File lib/src_lexer.rb, line 37 def pop_token token = @tokens.shift return END_TOKEN if token.nil? case token[0] when NUMBER_REGEX [:NUMBER, Token.new(token[0], token[1], token[2])] when STRING_REGEX [:STRING, Token.new(token[0], token[1], token[2])] else [is_reserved?(token[0]) ? token[0] : :IDENT, Token.new(token[0], token[1], token[2])] end end
Private Instance Methods
is_reserved?(token)
click to toggle source
# File lib/src_lexer.rb, line 178 def is_reserved?(token) @keywords.include?(token) || @symbols.include?(token) end
tokenize()
click to toggle source
# File lib/src_lexer.rb, line 139 def tokenize() @tokens = [] iterator = StringIterator.new(@str) while iterator < @str.length do if iterator.is_white_space then @tokens.push iterator.shift if iterator.marked? iterator.move_next elsif @line_comment_marker && iterator.is(@line_comment_marker) then @tokens.push iterator.shift if iterator.marked? iterator.move_to_the_end_of_the_line iterator.move_next elsif @comment_markers && iterator.is(@comment_markers[0]) then @tokens.push iterator.shift if iterator.marked? iterator.move_to(@comment_markers[1]) iterator.move_next elsif @string_literal_marker && iterator.is(@string_literal_marker[0]) then @tokens.push iterator.shift if iterator.marked? iterator.mark_set iterator.move_next iterator.move_to(@string_literal_marker[1]) iterator.move_next @tokens.push iterator.shift elsif iterator.is_in(@symbols) then @tokens.push iterator.shift if iterator.marked? iterator.mark_set @symbols.find { |symbol| iterator.is(symbol) }.length.times { iterator.move_next } @tokens.push iterator.shift elsif !iterator.marked? then iterator.mark_set else iterator.move_next end end @tokens.push iterator.shift if iterator.marked? return self end