class MailParser::Loose::Tokenizer
Public Class Methods
new(str)
click to toggle source
@return [String] str source string
# File lib/mailparser/loose.rb, line 278 def initialize(str) @comments = [] @ss = StringScanner.new(str) end
token(str)
click to toggle source
# File lib/mailparser/loose.rb, line 335 def self.token(str) Tokenizer.new(str).token end
token_received(str)
click to toggle source
# File lib/mailparser/loose.rb, line 339 def self.token_received(str) Tokenizer.new(str).token_received end
Public Instance Methods
token()
click to toggle source
tokenize @return [Array<String>] tokens
# File lib/mailparser/loose.rb, line 285 def token() token = [] while @ss.rest? do if s = @ss.scan(/\s+/) then # ignore elsif s = @ss.scan(/\(/) then begin pos = @ss.pos cfws(@ss) rescue ParseError @ss.pos = pos token << s end elsif s = @ss.scan(/\"(\s*(\\[#{TEXT_RE}]|[#{QTEXT_RE}\x80-\xff]))*\s*\"/o) || @ss.scan(/\[(\s*(\\[#{TEXT_RE}]|[#{DTEXT_RE}\x80-\xff]))*\s*\]/o) || @ss.scan(/[#{ATEXT_RE}\x80-\xff]+/o) token << s else token << @ss.scan(/./) end end return token end
token_received()
click to toggle source
tokenize for Received field @return [Array<String>] tokens
# File lib/mailparser/loose.rb, line 311 def token_received() ret = [] while @ss.rest? do if s = @ss.scan(/[\s]+/) then # ignore blank elsif s = @ss.scan(/\(/) then begin pos = @ss.pos cfws(@ss) rescue ParseError @ss.pos = pos ret.last << s unless ret.empty? end elsif s = @ss.scan(/\"([\s]*(\\[#{TEXT_RE}]|[#{QTEXT_RE}]))*[\s]*\"/o) ret << s elsif s = @ss.scan(/;/) ret << s else ret << @ss.scan(/[^\s\(\;]+/o) end end return ret end