def parse_scanner(s)
if s.eos?
[]
else
current_match = @token_types.find do |token_type|
s.match?(token_type.regex)
end
token =
if current_match.nil?
next_match_indexes = @token_types.map do |token_type|
next_match = s.check_until(token_type.regex)
if next_match.nil?
nil
else
next_match.length - s.matched.length
end
end.reject { |i| i.nil? }
non_match_size =
if next_match_indexes.length == 0
s.rest_size
else
next_match_indexes.min
end
non_match = s.peek(non_match_size)
s.pos = s.pos + non_match_size
NonMatchToken.new(non_match)
else
current_match.create_token s.scan(current_match.regex)
end
parse_scanner(s) << token
end
end