From 22e05bf807ad59bfad38fcdf35bb51524034e23b Mon Sep 17 00:00:00 2001
From: redpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>
Date: Thu, 28 Oct 2010 00:22:42 +0000
Subject: change stem handling

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@693 ec762483-ff6d-05da-a07a-a48fb63a330f
---
 word-aligner/stemmers/en.pl | 39 +++++++++++++++++++++++++++++++++++++++
 word-aligner/stemmers/fr.pl | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100755 word-aligner/stemmers/en.pl
 create mode 100755 word-aligner/stemmers/fr.pl

(limited to 'word-aligner/stemmers')

diff --git a/word-aligner/stemmers/en.pl b/word-aligner/stemmers/en.pl
new file mode 100755
index 00000000..2c85dd06
--- /dev/null
+++ b/word-aligner/stemmers/en.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 4;
+      if ($w =~ /^(ac|be|un|ad|al|re|co|de|in|im|ab|pr)/) { $el++; }
+      if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+
diff --git a/word-aligner/stemmers/fr.pl b/word-aligner/stemmers/fr.pl
new file mode 100755
index 00000000..fdf9ff6f
--- /dev/null
+++ b/word-aligner/stemmers/fr.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 4;
+      if ($w =~ /^(ac|un|ad|al|ré|co|dé|de|in|im|en|em|ab|pr)/) { $el++; }
+      if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+
-- 
cgit v1.2.3