diff options
author | Kenneth Heafield <github@kheafield.com> | 2011-11-03 19:53:08 +0000 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2011-11-03 19:53:08 +0000 |
commit | 976e492a10d88df932acbff3fec8142edc990929 (patch) | |
tree | 308efe2d0d0f72bcecaae4b270c4715cf021a021 /word-aligner/stemmers/mg.pl | |
parent | 3106cf8eca76df8b46d139b8f5ce5002200d660d (diff) | |
parent | 6de8f58cd13813bf33af4903bf386439683c0fd6 (diff) |
Merge branch 'master' of github.com:redpony/cdec
Diffstat (limited to 'word-aligner/stemmers/mg.pl')
-rwxr-xr-x | word-aligner/stemmers/mg.pl | 39 |
1 files changed, 39 insertions, 0 deletions
diff --git a/word-aligner/stemmers/mg.pl b/word-aligner/stemmers/mg.pl new file mode 100755 index 00000000..2f79a94e --- /dev/null +++ b/word-aligner/stemmers/mg.pl @@ -0,0 +1,39 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while(<STDIN>) { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 5; + if ($w =~ /(ndz|ndr|nts|ntr)/) { $el++; } + if ($w =~ /^(mp|mb|nd)/) { $el++; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + |