steal tokenizer from moses' scripts

1 files changed, 10 insertions, 0 deletions

diff --git a/lowercase.perl b/lowercase.perl
new file mode 100755
index 0000000..c30e029
--- /dev/null
+++ b/lowercase.perl
@@ -0,0 +1,10 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+
+while(<STDIN>) {
+  print lc($_);
+}