summaryrefslogtreecommitdiff
path: root/word-aligner/stemmers
diff options
context:
space:
mode:
authorKenneth Heafield <github@kheafield.com>2011-11-03 19:53:08 +0000
committerKenneth Heafield <github@kheafield.com>2011-11-03 19:53:08 +0000
commitca5b590478a2cb1c23a706b76e6a881ce8701716 (patch)
tree3faea0090f9a91b52ea2d4c47cfa3b4eb04d15f8 /word-aligner/stemmers
parent437ee131bc4c2574768a187bf29d62dbb715abf4 (diff)
parenta769e4964db2443ce165043095e18dfc0d788910 (diff)
Merge branch 'master' of github.com:redpony/cdec
Diffstat (limited to 'word-aligner/stemmers')
-rwxr-xr-xword-aligner/stemmers/mg.pl39
-rwxr-xr-xword-aligner/stemmers/rw.pl38
2 files changed, 77 insertions, 0 deletions
diff --git a/word-aligner/stemmers/mg.pl b/word-aligner/stemmers/mg.pl
new file mode 100755
index 00000000..2f79a94e
--- /dev/null
+++ b/word-aligner/stemmers/mg.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+ die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+ $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+ chomp;
+ my @words = split /\s+/;
+ my @out = ();
+ for my $w (@words) {
+ my $tw = $dict{$w};
+ if (!defined $tw) {
+ my $el = 5;
+ if ($w =~ /(ndz|ndr|nts|ntr)/) { $el++; }
+ if ($w =~ /^(mp|mb|nd)/) { $el++; }
+ if ($el > length($w)) { $el = length($w); }
+ $tw = substr $w, 0, $el;
+ $dict{$w} = $tw;
+ }
+ push @out, $tw;
+ }
+ if ($vocab) {
+ die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+ print "$_ @out\n";
+ } else {
+ print "@out\n";
+ }
+}
+
diff --git a/word-aligner/stemmers/rw.pl b/word-aligner/stemmers/rw.pl
new file mode 100755
index 00000000..6d873b40
--- /dev/null
+++ b/word-aligner/stemmers/rw.pl
@@ -0,0 +1,38 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+ die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+ $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+ chomp;
+ my @words = split /\s+/;
+ my @out = ();
+ for my $w (@words) {
+ my $tw = $dict{$w};
+ if (!defined $tw) {
+ my $el = 5;
+ if ($w =~ /(ny|jy|nk|nt|sh|cy)/) { $el++; }
+ if ($el > length($w)) { $el = length($w); }
+ $tw = substr $w, 0, $el;
+ $dict{$w} = $tw;
+ }
+ push @out, $tw;
+ }
+ if ($vocab) {
+ die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+ print "$_ @out\n";
+ } else {
+ print "@out\n";
+ }
+}
+