summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xfilter-tokens23
1 files changed, 23 insertions, 0 deletions
diff --git a/filter-tokens b/filter-tokens
new file mode 100755
index 0000000..00c8f2c
--- /dev/null
+++ b/filter-tokens
@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'zipf'
+
+bad_words = {}
+ReadFile.readlines_strip(ARGV[0]).each { |line|
+ bad_words[line] = true
+}
+
+i = 0
+while line = STDIN.gets
+ bad = false
+ tokens = line.split
+ bad_words.keys.each { |w|
+ if tokens.include? w
+ bad = true
+ break
+ end
+ }
+ puts i if bad
+ i += 1
+end
+