Don't tokenize empty tweets

harrisj · Dec 29, 2013 · 31aa842 · 31aa842 · peteyreplies · Dec 29, 2013
1 parent 3027681
commit 31aa842
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/ebook.rb b/ebook.rb
@@ -82,6 +82,7 @@ def filtered_tweets(tweets)
  tokenizer = Punkt::SentenceTokenizer.new(source_tweets.join(" ")) # init with corpus of all sentences
 
  source_tweets.each do |twt|
+ next if twt.nil? || twt == ''
  sentences = tokenizer.sentences_from_text(twt, :output => :sentences_text)
 
  # sentences = text.split(/[.:;?!]/)