change stem handling

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@693 ec762483-ff6d-05da-a07a-a48fb63a330f

5 files changed, 109 insertions, 13 deletions

diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl
index e23c2beb..508dbd8d 100755
--- a/word-aligner/aligner.pl
+++ b/word-aligner/aligner.pl
@@ -105,7 +105,9 @@ sub make_stage {
   my ($direction) = @_;
   my $stage_dir = "$align_dir/model-$direction";
   my $first = $direction;
+  my $second = $direction;
   $first =~ s/^(.+)-.*$/$1/;
+  $second =~ s/^.+-(.+)$/$1/;
   mkdir $stage_dir;
   open CDEC, ">$stage_dir/cdec.ini" or die "Can't write $stage_dir/cdec.ini: $!";
   print CDEC <<EOT;
@@ -113,6 +115,8 @@ formalism=lextrans
 intersection_strategy=full
 grammar=$align_dir/grammars/corpus.$direction.lex-grammar.gz
 feature_function=LexicalPairIdentity
+feature_function=LexicalPairIdentity C $align_dir/grammars/corpus.class.$first $align_dir/grammars/voc2class.$second
+feature_function=LexicalPairIdentity S $align_dir/grammars/corpus.stemmed.$first $align_dir/grammars/${second}stem.map
 feature_function=InputIdentity
 feature_function=OutputIdentity
 feature_function=RelativeSentencePosition $align_dir/grammars/corpus.class.$first
diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars
index f4b956bc..8a10cb19 100644
--- a/word-aligner/makefiles/makefile.grammars
+++ b/word-aligner/makefiles/makefile.grammars
@@ -1,8 +1,7 @@
-all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f
+all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f corpus.stemmed.f fstem.map corpus.stemmed.e estem.map
 
 clean:
-	$(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar*
-
+	$(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* *stem* 
 SUPPORT_DIR = $(SCRIPT_DIR)/support
 GZIP = /usr/bin/gzip
 ZCAT = zcat
@@ -12,21 +11,42 @@ SUPPLEMENT_WEIGHTS = $(SUPPORT_DIR)/supplement_weights_file.pl
 EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl
 ORTHONORM_E = $(SCRIPT_DIR)/ortho-norm/$(E_LANG).pl
 ORTHONORM_F = $(SCRIPT_DIR)/ortho-norm/$(F_LANG).pl
+STEM_F = $(SCRIPT_DIR)/stemmers/$(F_LANG).pl
+STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl
+
 CLASSIFY = $(SUPPORT_DIR)/classify.pl
 MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl
 MODEL1 = $(TRAINING_DIR)/model1
 MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl
 
-orthonorm-dict.e: corpus.e
-	$(EXTRACT_VOCAB) corpus.e > e.voc
+e.voc: corpus.e
+	$(EXTRACT_VOCAB) < corpus.e > $@
+
+f.voc: corpus.f
+	$(EXTRACT_VOCAB) < corpus.f > $@
+
+orthonorm-dict.e: corpus.e e.voc
 	$(ORTHONORM_E) < e.voc > e.ortho-voc
 	$(MERGE_CORPUS) e.voc e.ortho-voc > $@
 
-orthonorm-dict.f: corpus.f
-	$(EXTRACT_VOCAB) corpus.f > f.voc
+orthonorm-dict.f: corpus.f f.voc
 	$(ORTHONORM_F) < f.voc > f.ortho-voc
 	$(MERGE_CORPUS) f.voc f.ortho-voc > $@
 
+# this is just a "stem" map
+estem.map: e.voc
+	$(STEM_E) --vocab < e.voc > $@
+
+fstem.map: f.voc
+	$(STEM_F) --vocab < f.voc > $@
+
+# corpus.stemmed.f can use context to do "stemming"
+corpus.stemmed.f: corpus.f
+	$(STEM_F) < corpus.f > $@
+
+corpus.stemmed.e: corpus.e
+	$(STEM_E) < corpus.e > $@
+
 voc2class.e: corpus.e $(MKCLS)
 	$(MKCLS) -c$(NCLASSES) -n10 -pcorpus.e -Vvoc2class.e opt
 
diff --git a/word-aligner/stemmers/en.pl b/word-aligner/stemmers/en.pl
new file mode 100755
index 00000000..2c85dd06
--- /dev/null
+++ b/word-aligner/stemmers/en.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 4;
+      if ($w =~ /^(ac|be|un|ad|al|re|co|de|in|im|ab|pr)/) { $el++; }
+      if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+
diff --git a/word-aligner/stemmers/fr.pl b/word-aligner/stemmers/fr.pl
new file mode 100755
index 00000000..fdf9ff6f
--- /dev/null
+++ b/word-aligner/stemmers/fr.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 4;
+      if ($w =~ /^(ac|un|ad|al|ré|co|dé|de|in|im|en|em|ab|pr)/) { $el++; }
+      if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+
diff --git a/word-aligner/support/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl
index fb9d0214..3e243125 100755
--- a/word-aligner/support/make_lex_grammar.pl
+++ b/word-aligner/support/make_lex_grammar.pl
@@ -99,7 +99,6 @@ $oe_dict{'<eps>'} = '<eps>';
 
 my $MIN_FEATURE_COUNT = 0;
 my $ADD_PREFIX_ID = 1;
-my $ADD_CLASS_CLASS = 1;
 my $ADD_LEN = 1;
 my $ADD_SIM = 1;
 my $ADD_DICE = 1;
@@ -203,11 +202,6 @@ for my $f (sort keys %fdict) {
         push @feats, "Dice=$dice";
       }
     }
-    if ($ADD_CLASS_CLASS) {
-      my $ce = $eclass{$e} or die "E- no class for: '$e'";
-      my $cf = $fclass{$f} or die "F- no class for: '$f'";
-      push @feats, "C${cf}_${ce}=1";
-    }
     my $is_null = undef;
     if ($ADD_NULL && $f eq '<eps>') {
       $is_null = 1;