lbfgs + time-series minibatch optimization

5 files changed, 108 insertions, 3 deletions

diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl
index 3a385a88..c5078645 100755
--- a/word-aligner/aligner.pl
+++ b/word-aligner/aligner.pl
@@ -27,11 +27,13 @@ die "Expected format corpus.l1-l2 where l1 & l2 are two-letter abbreviations\nfo
 my $f_lang = $1;
 my $e_lang = $2;
 
+print STDERR " Using mkcls in: $mkcls\n\n";
 print STDERR "Source language: $f_lang\n";
 print STDERR "Target language: $e_lang\n";
-print STDERR " Using mkcls in: $mkcls\n\n";
-die "Don't have an orthographic normalizer for $f_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl";
-die "Don't have an orthographic normalizer for $e_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl";
+die "Don't have an stemmer for $f_lang: please create $SCRIPT_DIR/stemmers/$f_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$f_lang.pl";
+die "Don't have an stemmer for $e_lang: please create $SCRIPT_DIR/stemmers/$e_lang.pl\n" unless -f "$SCRIPT_DIR/stemmers/$e_lang.pl";
+die "Don't have an orthographic normalizer for $f_lang: please create $SCRIPT_DIR/ortho-norm/$f_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl";
+die "Don't have an orthographic normalizer for $e_lang: please create $SCRIPT_DIR/ortho-norm/$e_lang.pl\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl";
 
 my @directions = qw(f-e);
 
diff --git a/word-aligner/ortho-norm/mg.pl b/word-aligner/ortho-norm/mg.pl
new file mode 100755
index 00000000..4cb0e8e7
--- /dev/null
+++ b/word-aligner/ortho-norm/mg.pl
@@ -0,0 +1,13 @@
+#!/usr/bin/perl -w
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+
+while(<STDIN>) {
+  $_ = lc $_;
+  s/([a-z])'( |$)/$1$2/g;
+  print;
+}
+
diff --git a/word-aligner/ortho-norm/rw.pl b/word-aligner/ortho-norm/rw.pl
new file mode 100755
index 00000000..4cb0e8e7
--- /dev/null
+++ b/word-aligner/ortho-norm/rw.pl
@@ -0,0 +1,13 @@
+#!/usr/bin/perl -w
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+
+while(<STDIN>) {
+  $_ = lc $_;
+  s/([a-z])'( |$)/$1$2/g;
+  print;
+}
+
diff --git a/word-aligner/stemmers/mg.pl b/word-aligner/stemmers/mg.pl
new file mode 100755
index 00000000..2f79a94e
--- /dev/null
+++ b/word-aligner/stemmers/mg.pl
@@ -0,0 +1,39 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 5;
+      if ($w =~ /(ndz|ndr|nts|ntr)/) { $el++; }
+      if ($w =~ /^(mp|mb|nd)/) { $el++; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+
diff --git a/word-aligner/stemmers/rw.pl b/word-aligner/stemmers/rw.pl
new file mode 100755
index 00000000..6d873b40
--- /dev/null
+++ b/word-aligner/stemmers/rw.pl
@@ -0,0 +1,38 @@
+#!/usr/bin/perl -w
+
+use strict;
+use utf8;
+
+binmode(STDIN, ":utf8");
+binmode(STDOUT,":utf8");
+
+my $vocab = undef;
+if (scalar @ARGV > 0) {
+  die "Only allow --vocab" unless ($ARGV[0] eq '--vocab' && scalar @ARGV == 1);
+  $vocab = 1;
+}
+
+my %dict;
+while(<STDIN>) {
+  chomp;
+  my @words = split /\s+/;
+  my @out = ();
+  for my $w (@words) {
+    my $tw = $dict{$w};
+    if (!defined $tw) {
+      my $el = 5;
+      if ($w =~ /(ny|jy|nk|nt|sh|cy)/) { $el++; }
+      if ($el > length($w)) { $el = length($w); }
+      $tw = substr $w, 0, $el;
+      $dict{$w} = $tw;
+    }
+    push @out, $tw;
+  }
+  if ($vocab) {
+    die "Expected exactly one word per line with --vocab: $_" unless scalar @out == 1;
+    print "$_ @out\n";
+  } else {
+    print "@out\n";
+  }
+}
+