class Rakko::Scanner

Hand-coded scanner which transforms a stream of characters into a stream of tokens in the Rakko language.

### Syntax

Comment    :: #.*\n
Integer    :: 0|[1-9][0-9]*
Identifier :: [a-zA-Z_][0-9a-zA-Z_]*
String     :: "([^"]|\")*"

Attributes

columnno[R]
lineno[R]

Public Class Methods

new(input) click to toggle source

Creates a new Scanner instance from `input` stream.

The `input` parameter must be an instance of `String` or I/O object can respond to `#getc` method. If the `input` is an instance of `String`, it will be converted to an instance of `StringIO` internally.

@param input [String, getc] A stream of characters.

# File lib/rakko/scanner.rb, line 33
def initialize(input)
  @nextc = nil

  @columnno = 0
  @lineno = 0

  @input =
    case input
    when String
      StringIO.new(input)
    else
      unless input.respond_to?(:getc)
        raise ArgumentError, '`input` must be String or IO like object.'
      else
        input
      end
    end
end

Public Instance Methods

each(&block) click to toggle source
# File lib/rakko/scanner.rb, line 141
def each(&block)
  while t = next_token
    block.yield(t)
  end
end
next_token() click to toggle source

Reads the next token from the input stream.

@return [Token] The token read, or `nil` if called at end of the input stream.

# File lib/rakko/scanner.rb, line 55
def next_token
  # Read the first character if needed.
  advance unless @nextc

  loop do
    line = self.lineno
    column = self.columnno

    return case @nextc
    when nil # EOS
      nil
    when "\t", "\f", "\n", "\r", " " # Skip white spaces
      advance
      next
    when '#' # Skip line comment
      advance
      until @nextc == "\n" || @nextc == nil
        advance
      end
      redo
    when "0"
      # TODO octadecimal integer
      # TODO hexadecimal integer
      advance
      Token.new(0, lineno: line, columnno: column)
    when "1".."9" # decimal integer
      n = @nextc.ord - "0".ord
      loop do
        advance
        case @nextc
        when "0".."9"
          n = n * 10 + (@nextc.ord - "0".ord)
        else
          break Token.new(n, lineno: line, columnno: column)
        end
      end
    when "a".."z", "A".."Z", "_" # identifier
      name = @nextc
      loop do
        advance
        case @nextc
        when "0".."9", "a".."z", "A".."Z", "_"
          name << @nextc
        else
          break Token.new(name.to_sym, lineno: line, columnno: column)
        end
      end
    when "+", "-", "*", "/", "%", "=", ".", "!",
         "(", ")", "[", "]", "{", "}" # operator
      op = @nextc
      advance
      Token.new(op.to_sym, lineno: line, columnno: column)
    when '"' # string
      str = ''
      loop do
        advance
        case @nextc
        when nil
          raise "Premature end of quoted string"
        when '\\' # Escape sequence
          advance
          case @nextc
          when 't';  str << "\t"
          when 'n';  str << "\n"
          when 'r';  str << "\r"
          when 'f';  str << "\f"
          when '\\'; str << "\\"
          when '"';  str << "\""
          when nil
            raise "Premature end of escape sequence"
          else
            raise %Q{Unrecognized escape sequence: "\\#{@nextc}"}
          end
        when '"'
          advance
          break Token.new(str, lineno: line, columnno: column)
        else
          str << @nextc
        end
      end
    else
      raise "Unrecognized token"
    end
  end
end

Private Instance Methods

advance() click to toggle source
# File lib/rakko/scanner.rb, line 149
def advance
  # Update lineno and columnno of _next character_
  if @nextc
    if @nextc == $/
      @lineno += 1
      @columnno = 0
    else
      @columnno += 1
    end
  end

  @nextc = @input.getc
end