From a769e4964db2443ce165043095e18dfc0d788910 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 31 Oct 2011 14:03:22 -0400 Subject: lbfgs + time-series minibatch optimization --- word-aligner/aligner.pl | 8 +++++--- word-aligner/ortho-norm/mg.pl | 13 +++++++++++++ word-aligner/ortho-norm/rw.pl | 13 +++++++++++++ word-aligner/stemmers/mg.pl | 39 +++++++++++++++++++++++++++++++++++++++ word-aligner/stemmers/rw.pl | 38 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 3 deletions(-) create mode 100755 word-aligner/ortho-norm/mg.pl create mode 100755 word-aligner/ortho-norm/rw.pl create mode 100755 word-aligner/stemmers/mg.pl create mode 100755 word-aligner/stemmers/rw.pl (limited to 'word-aligner') diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl index 3a385a88..c5078645 100755 --- a/word-aligner/aligner.pl +++ b/word-aligner/aligner.pl @@ -27,11 +27,13 @@ die "Expected format corpus.l1-l2 where l1 & l2 are two-letter abbreviations\nfo my $f_lang = $1; my $e_lang = $2; +print STDERR " Using mkcls in: $mkcls\n\n"; print STDERR "Source language: $f_lang\n"; print STDERR "Target language: $e_lang\n"; -print STDERR " Using mkcls in: $mkcls\n\n"; -die "Don't have an orthographic normalizer for $f_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl"; -die "Don't have an orthographic normalizer for $e_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl"; +die "Don't have an stemmer for $f_lang: please create $SCRIPT_DIR/stemmers/$f_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$f_lang.pl"; +die "Don't have an stemmer for $e_lang: please create $SCRIPT_DIR/stemmers/$e_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$e_lang.pl"; +die "Don't have an orthographic normalizer for $f_lang: please create $SCRIPT_DIR/ortho-norm/$f_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl"; +die "Don't have an orthographic normalizer for $e_lang: please create $SCRIPT_DIR/ortho-norm/$e_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl"; my @directions = qw(f-e); diff --git a/word-aligner/ortho-norm/mg.pl b/word-aligner/ortho-norm/mg.pl new file mode 100755 index 00000000..4cb0e8e7 --- /dev/null +++ b/word-aligner/ortho-norm/mg.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -w +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +while() { + $_ = lc $_; + s/([a-z])'( |$)/$1$2/g; + print; +} + diff --git a/word-aligner/ortho-norm/rw.pl b/word-aligner/ortho-norm/rw.pl new file mode 100755 index 00000000..4cb0e8e7 --- /dev/null +++ b/word-aligner/ortho-norm/rw.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -w +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT, ":utf8"); + +while() { + $_ = lc $_; + s/([a-z])'( |$)/$1$2/g; + print; +} + diff --git a/word-aligner/stemmers/mg.pl b/word-aligner/stemmers/mg.pl new file mode 100755 index 00000000..2f79a94e --- /dev/null +++ b/word-aligner/stemmers/mg.pl @@ -0,0 +1,39 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while() { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 5; + if ($w =~ /(ndz|ndr|nts|ntr)/) { $el++; } + if ($w =~ /^(mp|mb|nd)/) { $el++; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + diff --git a/word-aligner/stemmers/rw.pl b/word-aligner/stemmers/rw.pl new file mode 100755 index 00000000..6d873b40 --- /dev/null +++ b/word-aligner/stemmers/rw.pl @@ -0,0 +1,38 @@ +#!/usr/bin/perl -w + +use strict; +use utf8; + +binmode(STDIN, ":utf8"); +binmode(STDOUT,":utf8"); + +my $vocab = undef; +if (scalar @ARGV > 0) { + die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1); + $vocab = 1; +} + +my %dict; +while() { + chomp; + my @words = split /\s+/; + my @out = (); + for my $w (@words) { + my $tw = $dict{$w}; + if (!defined $tw) { + my $el = 5; + if ($w =~ /(ny|jy|nk|nt|sh|cy)/) { $el++; } + if ($el > length($w)) { $el = length($w); } + $tw = substr $w, 0, $el; + $dict{$w} = $tw; + } + push @out, $tw; + } + if ($vocab) { + die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1; + print "$_ @out\n"; + } else { + print "@out\n"; + } +} + -- cgit v1.2.3