summaryrefslogtreecommitdiff
path: root/word-aligner
diff options
context:
space:
mode:
authorredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-10-28 00:22:42 +0000
committerredpony <redpony@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-10-28 00:22:42 +0000
commit22e05bf807ad59bfad38fcdf35bb51524034e23b (patch)
treece537739d375d5216a9558414dd9d8dce065f6fc /word-aligner
parentecde8cb600b24c31b062f8f53d57641e3fa23379 (diff)
change stem handling
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@693 ec762483-ff6d-05da-a07a-a48fb63a330f
Diffstat (limited to 'word-aligner')
-rwxr-xr-xword-aligner/aligner.pl4
-rw-r--r--word-aligner/makefiles/makefile.grammars34
-rwxr-xr-xword-aligner/stemmers/en.pl39
-rwxr-xr-xword-aligner/stemmers/fr.pl39
-rwxr-xr-xword-aligner/support/make_lex_grammar.pl6
5 files changed, 109 insertions, 13 deletions
diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl
index e23c2beb..508dbd8d 100755
--- a/word-aligner/aligner.pl
+++ b/word-aligner/aligner.pl
@@ -105,7 +105,9 @@ sub make_stage {
my ($direction) = @_;
my $stage_dir = "$align_dir/model-$direction";
my $first = $direction;
+ my $second = $direction;
$first =~ s/^(.+)-.*$/$1/;
+ $second =~ s/^.+-(.+)$/$1/;
mkdir $stage_dir;
open CDEC, ">$stage_dir/cdec.ini" or die "Can't write $stage_dir/cdec.ini: $!";
print CDEC <<EOT;
@@ -113,6 +115,8 @@ formalism=lextrans
intersection_strategy=full
grammar=$align_dir/grammars/corpus.$direction.lex-grammar.gz
feature_function=LexicalPairIdentity
+feature_function=LexicalPairIdentity C $align_dir/grammars/corpus.class.$first $align_dir/grammars/voc2class.$second
+feature_function=LexicalPairIdentity S $align_dir/grammars/corpus.stemmed.$first $align_dir/grammars/${second}stem.map
feature_function=InputIdentity
feature_function=OutputIdentity
feature_function=RelativeSentencePosition $align_dir/grammars/corpus.class.$first
diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars
index f4b956bc..8a10cb19 100644
--- a/word-aligner/makefiles/makefile.grammars
+++ b/word-aligner/makefiles/makefile.grammars
@@ -1,8 +1,7 @@
-all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f
+all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f corpus.stemmed.f fstem.map corpus.stemmed.e estem.map
clean:
- $(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar*
-
+ $(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* *stem*
SUPPORT_DIR = $(SCRIPT_DIR)/support
GZIP = /usr/bin/gzip
ZCAT = zcat
@@ -12,21 +11,42 @@ SUPPLEMENT_WEIGHTS = $(SUPPORT_DIR)/supplement_weights_file.pl
EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl
ORTHONORM_E = $(SCRIPT_DIR)/ortho-norm/$(E_LANG).pl
ORTHONORM_F = $(SCRIPT_DIR)/ortho-norm/$(F_LANG).pl
+STEM_F = $(SCRIPT_DIR)/stemmers/$(F_LANG).pl
+STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl
+
CLASSIFY = $(SUPPORT_DIR)/classify.pl
MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl
MODEL1 = $(TRAINING_DIR)/model1
MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl
-orthonorm-dict.e: corpus.e
- $(EXTRACT_VOCAB) corpus.e > e.voc
+e.voc: corpus.e
+ $(EXTRACT_VOCAB) < corpus.e > $@
+
+f.voc: corpus.f
+ $(EXTRACT_VOCAB) < corpus.f > $@
+
+orthonorm-dict.e: corpus.e e.voc
$(ORTHONORM_E) < e.voc > e.ortho-voc
$(MERGE_CORPUS) e.voc e.ortho-voc > $@
-orthonorm-dict.f: corpus.f
- $(EXTRACT_VOCAB) corpus.f > f.voc
+orthonorm-dict.f: corpus.f f.voc
$(ORTHONORM_F) < f.voc > f.ortho-voc
$(MERGE_CORPUS) f.voc f.ortho-voc > $@
+# this is just a "stem" map
+estem.map: e.voc
+ $(STEM_E) --vocab < e.voc > $@
+
+fstem.map: f.voc
+ $(STEM_F) --vocab < f.voc > $@
+
+# corpus.stemmed.f can use context to do "stemming"
+corpus.stemmed.f: corpus.f
+ $(STEM_F) < corpus.f > $@
+
+corpus.stemmed.e: corpus.e
+ $(STEM_E) < corpus.e > $@
+
voc2class.e: corpus.e $(MKCLS)
$(MKCLS) -c$(NCLASSES) -n10 -pcorpus.e -Vvoc2class.e opt
diff --git a/word-aligner/stemmers/en.pl b/word-aligner/stemmers/en.pl
new file mode 100755
index 00000000..2c85dd06
--- /dev/null
+++ b/word-aligner/stemmers/en.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+ die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+ $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+ chomp;
+ my @words = split /\s+/;
+ my @out = ();
+ for my $w (@words) {
+ my $tw = $dict{$w};
+ if (!defined $tw) {
+ my $el = 4;
+ if ($w =~ /^(ac|be|un|ad|al|re|co|de|in|im|ab|pr)/) { $el++; }
+ if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+ if ($el > length($w)) { $el = length($w); }
+ $tw = substr $w, 0, $el;
+ $dict{$w} = $tw;
+ }
+ push @out, $tw;
+ }
+ if ($vocab) {
+ die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+ print "$_ @out\n";
+ } else {
+ print "@out\n";
+ }
+}
+
diff --git a/word-aligner/stemmers/fr.pl b/word-aligner/stemmers/fr.pl
new file mode 100755
index 00000000..fdf9ff6f
--- /dev/null
+++ b/word-aligner/stemmers/fr.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+ die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+ $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+ chomp;
+ my @words = split /\s+/;
+ my @out = ();
+ for my $w (@words) {
+ my $tw = $dict{$w};
+ if (!defined $tw) {
+ my $el = 4;
+ if ($w =~ /^(ac|un|ad|al|ré|co|dé|de|in|im|en|em|ab|pr)/) { $el++; }
+ if ($w =~ /^(trans|inter|under|over)/) { $el+=2; }
+ if ($el > length($w)) { $el = length($w); }
+ $tw = substr $w, 0, $el;
+ $dict{$w} = $tw;
+ }
+ push @out, $tw;
+ }
+ if ($vocab) {
+ die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+ print "$_ @out\n";
+ } else {
+ print "@out\n";
+ }
+}
+
diff --git a/word-aligner/support/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl
index fb9d0214..3e243125 100755
--- a/word-aligner/support/make_lex_grammar.pl
+++ b/word-aligner/support/make_lex_grammar.pl
@@ -99,7 +99,6 @@ $oe_dict{'<eps>'} = '<eps>';
my $MIN_FEATURE_COUNT = 0;
my $ADD_PREFIX_ID = 1;
-my $ADD_CLASS_CLASS = 1;
my $ADD_LEN = 1;
my $ADD_SIM = 1;
my $ADD_DICE = 1;
@@ -203,11 +202,6 @@ for my $f (sort keys %fdict) {
push @feats, "Dice=$dice";
}
}
- if ($ADD_CLASS_CLASS) {
- my $ce = $eclass{$e} or die "E- no class for: '$e'";
- my $cf = $fclass{$f} or die "F- no class for: '$f'";
- push @feats, "C${cf}_${ce}=1";
- }
my $is_null = undef;
if ($ADD_NULL && $f eq '<eps>') {
$is_null = 1;