remove-test-from-bitext

1 files changed, 45 insertions, 0 deletions

diff --git a/remove-test-from-bitext b/remove-test-from-bitext
new file mode 100755
index 0000000..43038d3
--- /dev/null
+++ b/remove-test-from-bitext
@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'zipf'
+
+test_source = ReadFile.new ARGV[0]
+test_target = ReadFile.new ARGV[1]
+all_test_source_lines = {}
+all_test_source_lines.default = false
+all_test_target_lines = {}
+all_test_source_lines.default = false
+
+while test_source_line = test_source.gets
+  test_source_line.strip!
+  test_target_line = test_target.gets
+  test_target_line.strip!
+  
+  all_test_source_lines[test_source_line] = true
+  all_test_target_lines[test_target_line] = true
+end
+
+train_source = ReadFile.new ARGV[2]
+train_target = ReadFile.new ARGV[3]
+
+train_source_out = WriteFile.new "#{ARGV[2]}.out"
+train_target_out = WriteFile.new "#{ARGV[3]}.out"
+
+while train_source_line = train_source.gets
+  train_source_line.strip!
+  train_target_line = train_target.gets
+  if train_target_line == nil then train_target_line = "" end
+  train_target_line.strip!
+
+  if not all_test_source_lines[train_source_line] \
+     and not all_test_target_lines[train_target_line]
+    train_source_out.write train_source_line + "\n"
+    train_target_out.write train_target_line + "\n"
+  end
+end
+
+test_source.close
+test_target.close
+train_source.close
+train_target.close
+train_source_out.close
+train_target_out.close