class Markovian::Utils::TextSplitter
Constants
- MARKERS_OF_INTEREST
anything that doesn’t contain any letters is not a word we need to care about
Attributes
text[R]
Public Class Methods
new(text)
click to toggle source
# File lib/markovian/utils/text_splitter.rb, line 7 def initialize(text) @text = text end
Public Instance Methods
components()
click to toggle source
# File lib/markovian/utils/text_splitter.rb, line 14 def components split_text.select {|w| w.text.match(MARKERS_OF_INTEREST)} end
Protected Instance Methods
split_text()
click to toggle source
# File lib/markovian/utils/text_splitter.rb, line 20 def split_text Tokeneyes::Tokenizer.new(text.downcase).parse_into_words end