summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-12-01 02:46:07 +0000
committerredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-12-01 02:46:07 +0000
commit46882b48148ec36d4eac7a94a3b2bf453ffb0828 (patch)
tree213ea62fe335e5016ae0ddb4ae712dacd0e1f33e
parent619f814c28159e207e1f6d3f8ec872114d12989b (diff)
stemmers
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@737 ec762483-ff6d-05da-a07a-a48fb63a330f
-rwxr-xr-xword-aligner/stemmers/zh.pl27
1 files changed, 27 insertions, 0 deletions
diff --git a/word-aligner/stemmers/zh.pl b/word-aligner/stemmers/zh.pl
new file mode 100755
index 00000000..14ba8aab
--- /dev/null
+++ b/word-aligner/stemmers/zh.pl
@@ -0,0 +1,27 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+ die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+ $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+ chomp;
+ my @words = split /\s+/;
+ my @out = @words;
+ if ($vocab) {
+ die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+ print "$_ @out\n";
+ } else {
+ print "@out\n";
+ }
+}
+