class Rakko::Scanner
Hand-coded scanner which transforms a stream of characters into a stream of tokens in the Rakko
language.
### Syntax
Comment :: #.*\n Integer :: 0|[1-9][0-9]* Identifier :: [a-zA-Z_][0-9a-zA-Z_]* String :: "([^"]|\")*"
Attributes
columnno[R]
lineno[R]
Public Class Methods
new(input)
click to toggle source
Creates a new Scanner
instance from `input` stream.
The `input` parameter must be an instance of `String` or I/O object can respond to `#getc` method. If the `input` is an instance of `String`, it will be converted to an instance of `StringIO` internally.
@param input [String, getc] A stream of characters.
# File lib/rakko/scanner.rb, line 33 def initialize(input) @nextc = nil @columnno = 0 @lineno = 0 @input = case input when String StringIO.new(input) else unless input.respond_to?(:getc) raise ArgumentError, '`input` must be String or IO like object.' else input end end end
Public Instance Methods
each(&block)
click to toggle source
# File lib/rakko/scanner.rb, line 141 def each(&block) while t = next_token block.yield(t) end end
next_token()
click to toggle source
Reads the next token from the input stream.
@return [Token] The token read, or `nil` if called at end of the input stream.
# File lib/rakko/scanner.rb, line 55 def next_token # Read the first character if needed. advance unless @nextc loop do line = self.lineno column = self.columnno return case @nextc when nil # EOS nil when "\t", "\f", "\n", "\r", " " # Skip white spaces advance next when '#' # Skip line comment advance until @nextc == "\n" || @nextc == nil advance end redo when "0" # TODO octadecimal integer # TODO hexadecimal integer advance Token.new(0, lineno: line, columnno: column) when "1".."9" # decimal integer n = @nextc.ord - "0".ord loop do advance case @nextc when "0".."9" n = n * 10 + (@nextc.ord - "0".ord) else break Token.new(n, lineno: line, columnno: column) end end when "a".."z", "A".."Z", "_" # identifier name = @nextc loop do advance case @nextc when "0".."9", "a".."z", "A".."Z", "_" name << @nextc else break Token.new(name.to_sym, lineno: line, columnno: column) end end when "+", "-", "*", "/", "%", "=", ".", "!", "(", ")", "[", "]", "{", "}" # operator op = @nextc advance Token.new(op.to_sym, lineno: line, columnno: column) when '"' # string str = '' loop do advance case @nextc when nil raise "Premature end of quoted string" when '\\' # Escape sequence advance case @nextc when 't'; str << "\t" when 'n'; str << "\n" when 'r'; str << "\r" when 'f'; str << "\f" when '\\'; str << "\\" when '"'; str << "\"" when nil raise "Premature end of escape sequence" else raise %Q{Unrecognized escape sequence: "\\#{@nextc}"} end when '"' advance break Token.new(str, lineno: line, columnno: column) else str << @nextc end end else raise "Unrecognized token" end end end
Private Instance Methods
advance()
click to toggle source
# File lib/rakko/scanner.rb, line 149 def advance # Update lineno and columnno of _next character_ if @nextc if @nextc == $/ @lineno += 1 @columnno = 0 else @columnno += 1 end end @nextc = @input.getc end