Modified regex to include 30 additional English stop words

2015-11-18 20:18:03 -05:00
parent c148fe403d
commit 5e83ea6935
1 changed files with 1 additions and 1 deletions
@@ -439,7 +439,7 @@ sub hashtagify {
                # drop stop words
 		# TODO : make these overridable
                next if length( $item ) < 3;
-                next if lc( $item ) =~ m/^(and|are|but|can|for|from|had|her|his|have|how|its|that|the|their|they|theyre|this|was|what|when|you)$/;
+                next if lc( $item ) =~ m/^a(lso|nd|ny|re)|been|but|can(not|t)?|e(ach|tc|very)|for|from|g(e|o)t|ha(d|ve)|has(nt)?|hers?|hi(m|s)|how|its|no(r|t)|ours?|she|some|th(an|at|em?|eirs?|(e|o)se|ey|eyre|is)|too|very|was|wh(at|en|o)|with|you(r|rs)?$/;
                # hashtagify it
                $item = '#' . $item;
                # use a hash here instead of an ordered list for auto-dedupe