class PTB

Tokenization based on the tokenizer developed by Robert Macyntyre in 1995 for the Penn Treebank project. This tokenizer mostly follows the conventions used by the Penn Treebank. Modified from the sript written by Utiyama Masao (mutiyama@nict.go.jp).

Constants

DefaultOptions: Default options for the tokenizer.

Public Instance Methods

tokenize(string, options = {}) click to toggle source

# File lib/tokenator/core.rb, line 15
def tokenize(string, options = {})
  options = DefaultOptions.merge(options)

  s = " " + string + " "
  # Transforms every quote to single quote
  #s.gsub!(/[`‘’“”"]/, "'")
  #Preserves acronyms deleting puncts
  s.gsub!(/((\w\.){2,})/) {$1.gsub!(/\./,'')}

  s.gsub!(/ (([A-Z]\.){2,}) /,' ')
  s.gsub!(/\w\./)
  s.gsub!(/\s+/, ' ')
  s.gsub!(/(\s+)''/, '\1"')
  s.gsub!(/(\s+)``/, '\1"')
  s.gsub!(/''(\s+)/, '"\1')
  s.gsub!(/``(\s+)/, '"\1')
  s.gsub!(/ (['`]+)([^0-9].+) /, ' \1 \2 ')
  s.gsub!(/([ (\[{<])"/, '\1 `` ')
  s.gsub!(/\.\.\./, ' ... ')
  s.gsub!(/[,;:@\#$%&]/, ' \& ')
  s.gsub!(/([^.])([.])([\])}>"']*)[   ]*$/, '\1 \2\3 ')
  s.gsub!(/[?!]/, ' \& ')
  s.gsub!(/[\]\[(){}<>]/, ' \& ')
  s.gsub!(/--/, ' -- ')
  s.sub!(/$/, ' ')
  s.sub!(/^/, ' ')
  s.gsub!(/'ll /, ' will ')
  s.gsub!(/'re /, ' are ')
  s.gsub!(/'ve /, ' have ')
  s.gsub!(/n't /, ' not ')
  s.gsub!(/'LL /, ' WILL ')
  s.gsub!(/'RE /, ' ARE ')
  s.gsub!(/'VE /, ' HAVE ')
  s.gsub!(/N'T /, ' NOT ')
  s.gsub!(/ ([Cc])annot /, ' \1an not ')
  s.gsub!(/ ([Ii])'m /, ' \1 am ')
  s.gsub!(/ ([Hh]|[Ss]h)e's /, ' \1e is ')
  s.gsub!(/ ([Ii])t's /, ' \1t is ')
  s.gsub!(/ ([Ww](hat|ho))'s/, ' \1 is')
  s.gsub!(/ ([Dd])idn't /, ' \1id not ')
  s.gsub!(/ ([Dd])'ye /, ' \1\' ye ')
  s.gsub!(/ ([Gg])imme /, ' \1ive me ')
  s.gsub!(/ ([Gg])onna /, ' \1oing to ')
  s.gsub!(/ ([Gg])otta /, ' \1ot to ')
  s.gsub!(/ ([Ll])emme /, ' \1et me ')
  s.gsub!(/ ([Mm])ore'n /, ' \1ore \'n ')
  s.gsub!(/ '([Tt])is /, ' \'\1 is ')
  s.gsub!(/ '([Tt])was /, ' \'\1 was ')
  s.gsub!(/ ([Ww])anna /, ' \1ant to ')

  while s.sub!(/(\s)([0-9]+) , ([0-9]+)(\s)/, '\1\2,\3\4'); end
  s.gsub!(/\//, ' / ')
  s.gsub!(/\s+/, ' ')
  s.strip!
  # Remove directional quotes.
  if options[:remove_punctuation]
    s.gsub!(/ [-:;?!]/, ' ')
    s.gsub!(/ , /, ' ')
    s.gsub!(/ '(\w+)' /, ' \1 ')
    s.gsub!(/([a-zA-Z]+)\./, '\1 ')
    s.gsub!(/\.\.\./, '')
    s.gsub!(/ \./, ' ')
    s.gsub!(/- /, ' ')
    s.gsub!(/["`'][^\w]/, ' ')
    s.gsub!(/"$/, ' ')
  end
  s.split(/\s+/)
end