diff options
Diffstat (limited to 'word-aligner')
-rwxr-xr-x | word-aligner/aligner.pl | 4 | ||||
-rw-r--r-- | word-aligner/makefiles/makefile.grammars | 34 | ||||
-rwxr-xr-x | word-aligner/stemmers/en.pl | 39 | ||||
-rwxr-xr-x | word-aligner/stemmers/fr.pl | 39 | ||||
-rwxr-xr-x | word-aligner/support/make_lex_grammar.pl | 6 |
5 files changed, 109 insertions, 13 deletions
diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl index e23c2beb..508dbd8d 100755 --- a/word-aligner/aligner.pl +++ b/word-aligner/aligner.pl @@ -105,7 +105,9 @@ sub make_stage { my ($direction) = @_; my $stage_dir = "$align_dir/model-$direction"; my $first = $direction; + my $second = $direction; $first =~ s/^(.+)-.*$/$1/; + $second =~ s/^.+-(.+)$/$1/; mkdir $stage_dir; open CDEC, ">$stage_dir/cdec.ini" or die "Can't write $stage_dir/cdec.ini: $!"; print CDEC <<EOT; @@ -113,6 +115,8 @@ formalism=lextrans intersection_strategy=full grammar=$align_dir/grammars/corpus.$direction.lex-grammar.gz feature_function=LexicalPairIdentity +feature_function=LexicalPairIdentity C $align_dir/grammars/corpus.class.$first $align_dir/grammars/voc2class.$second +feature_function=LexicalPairIdentity S $align_dir/grammars/corpus.stemmed.$first $align_dir/grammars/${second}stem.map feature_function=InputIdentity feature_function=OutputIdentity feature_function=RelativeSentencePosition $align_dir/grammars/corpus.class.$first diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars index f4b956bc..8a10cb19 100644 --- a/word-aligner/makefiles/makefile.grammars +++ b/word-aligner/makefiles/makefile.grammars @@ -1,8 +1,7 @@ -all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f +all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.class.f corpus.stemmed.f fstem.map corpus.stemmed.e estem.map clean: - $(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* - + $(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* *stem* SUPPORT_DIR = $(SCRIPT_DIR)/support GZIP = /usr/bin/gzip ZCAT = zcat @@ -12,21 +11,42 @@ SUPPLEMENT_WEIGHTS = $(SUPPORT_DIR)/supplement_weights_file.pl EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl ORTHONORM_E = $(SCRIPT_DIR)/ortho-norm/$(E_LANG).pl ORTHONORM_F = $(SCRIPT_DIR)/ortho-norm/$(F_LANG).pl +STEM_F = $(SCRIPT_DIR)/stemmers/$(F_LANG).pl +STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl + CLASSIFY = $(SUPPORT_DIR)/classify.pl MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl MODEL1 = $(TRAINING_DIR)/model1 MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl -orthonorm-dict.e: corpus.e - $(EXTRACT_VOCAB) corpus.e > e.voc +e.voc: corpus.e + $(EXTRACT_VOCAB) < corpus.e > $@ + +f.voc: corpus.f + $(EXTRACT_VOCAB) < corpus.f > $@ + +orthonorm-dict.e: corpus.e e.voc $(ORTHONORM_E) < e.voc > e.ortho-voc $(MERGE_CORPUS) e.voc e.ortho-voc > $@ -orthonorm-dict.f: corpus.f - $(EXTRACT_VOCAB) corpus.f > f.voc +orthonorm-dict.f: corpus.f f.voc $(ORTHONORM_F) < f.voc > f.ortho-voc $(MERGE_CORPUS) f.voc f.ortho-voc > $@ +# this is just a "stem" map +estem.map: e.voc + $(STEM_E) --vocab < e.voc > $@ + +fstem.map: f.voc + $(STEM_F) --vocab < f.voc > $@ + +# corpus.stemmed.f can use context to do "stemming" +corpus.stemmed.f: corpus.f + $(STEM_F) < corpus.f > $@ + +corpus.stemmed.e: corpus.e + $(STEM_E) < corpus.e > $@ + voc2class.e: corpus.e $(MKCLS) $(MKCLS) -c$(NCLASSES) -n10 -pcorpus.e -Vvoc2class.e opt diff --git a/word-aligner/stemmers/en.pl b/word-aligner/stemmers/en.pl new file mode 100755 index 00000000..2c85dd06 --- /dev/null +++ b/word-aligner/stemmers/en.pl @@ -0,0 +1,39 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while(<STDIN>) { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 4; + if ($w =~ /^(ac|be|un|ad|al|re|co|de|in|im|ab|pr)/) { $el++; } + if ($w =~ /^(trans|inter|under|over)/) { $el+=2; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + diff --git a/word-aligner/stemmers/fr.pl b/word-aligner/stemmers/fr.pl new file mode 100755 index 00000000..fdf9ff6f --- /dev/null +++ b/word-aligner/stemmers/fr.pl @@ -0,0 +1,39 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while(<STDIN>) { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 4; + if ($w =~ /^(ac|un|ad|al|ré|co|dé|de|in|im|en|em|ab|pr)/) { $el++; } + if ($w =~ /^(trans|inter|under|over)/) { $el+=2; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + diff --git a/word-aligner/support/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl index fb9d0214..3e243125 100755 --- a/word-aligner/support/make_lex_grammar.pl +++ b/word-aligner/support/make_lex_grammar.pl @@ -99,7 +99,6 @@ $oe_dict{'<eps>'} = '<eps>'; my $MIN_FEATURE_COUNT = 0; my $ADD_PREFIX_ID = 1; -my $ADD_CLASS_CLASS = 1; my $ADD_LEN = 1; my $ADD_SIM = 1; my $ADD_DICE = 1; @@ -203,11 +202,6 @@ for my $f (sort keys %fdict) { push @feats, "Dice=$dice"; } } - if ($ADD_CLASS_CLASS) { - my $ce = $eclass{$e} or die "E- no class for: '$e'"; - my $cf = $fclass{$f} or die "F- no class for: '$f'"; - push @feats, "C${cf}_${ce}=1"; - } my $is_null = undef; if ($ADD_NULL && $f eq '<eps>') { $is_null = 1; |