From 2783f837303ae07c4a1d676302bca779abbb1296 Mon Sep 17 00:00:00 2001
From: Patrick Simianer <p@simianer.de>
Date: Sat, 14 Jun 2014 14:43:14 +0200
Subject: steal tokenizer from moses' scripts

---
 lowercase.perl | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100755 lowercase.perl

(limited to 'lowercase.perl')

diff --git a/lowercase.perl b/lowercase.perl
new file mode 100755
index 0000000..c30e029
--- /dev/null
+++ b/lowercase.perl
@@ -0,0 +1,10 @@
+#!/usr/bin/perl -w
+
+use strict;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+
+while(<STDIN>) {
+  print lc($_);
+}
-- 
cgit v1.2.3