summaryrefslogtreecommitdiff
path: root/word-aligner
diff options
context:
space:
mode:
authorAvneesh Saluja <asaluja@gmail.com>2013-03-28 18:28:16 -0700
committerAvneesh Saluja <asaluja@gmail.com>2013-03-28 18:28:16 -0700
commit3d8d656fa7911524e0e6885647173474524e0784 (patch)
tree81b1ee2fcb67980376d03f0aa48e42e53abff222 /word-aligner
parentbe7f57fdd484e063775d7abf083b9fa4c403b610 (diff)
parent96fedabebafe7a38a6d5928be8fff767e411d705 (diff)
fixed conflicts
Diffstat (limited to 'word-aligner')
-rw-r--r--word-aligner/Makefile.am8
-rwxr-xr-xword-aligner/aligner.pl19
-rw-r--r--word-aligner/da.h79
-rw-r--r--word-aligner/fast_align.cc310
-rw-r--r--word-aligner/makefiles/makefile.grammars4
-rw-r--r--word-aligner/makefiles/makefile.model.f-e14
-rwxr-xr-xword-aligner/ortho-norm/ru.pl44
-rwxr-xr-xword-aligner/paste-parallel-files.pl35
-rw-r--r--word-aligner/ttables.cc31
-rw-r--r--word-aligner/ttables.h101
10 files changed, 607 insertions, 38 deletions
diff --git a/word-aligner/Makefile.am b/word-aligner/Makefile.am
new file mode 100644
index 00000000..1f7f78ae
--- /dev/null
+++ b/word-aligner/Makefile.am
@@ -0,0 +1,8 @@
+bin_PROGRAMS = fast_align
+
+fast_align_SOURCES = fast_align.cc ttables.cc da.h ttables.h
+fast_align_LDADD = ../utils/libutils.a
+
+EXTRA_DIST = aligner.pl ortho-norm support makefiles stemmers
+
+AM_CPPFLAGS = -W -Wall -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/training
diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl
index c5078645..cbccb94a 100755
--- a/word-aligner/aligner.pl
+++ b/word-aligner/aligner.pl
@@ -51,6 +51,8 @@ while(<IN>) {
chomp;
my ($f, $e) = split / \|\|\| /;
die "Bad format, excepted ||| separated line" unless defined $f && defined $e;
+ $f =~ s/\[/(/g;
+ $e =~ s/\]/)/g;
print F "$f\n";
print E "$e\n";
}
@@ -80,6 +82,11 @@ NCLASSES = $num_classes
TARGETS = @targets
PTRAIN = \$(TRAINING_DIR)/cluster-ptrain.pl --restart_if_necessary
PTRAIN_PARAMS = --gaussian_prior --sigma_squared 1.0 --max_iteration 15
+#MPIJOBS = 4
+#MPIRUN = mpirun -np $(MPIJOBS)
+MPIRUN=
+
+WALLTIME=90
export
@@ -99,7 +106,15 @@ clean:
EOT
close TOPLEVEL;
-print STDERR "Created alignment task. chdir to talign/ then type make.\n\n";
+print STDERR <<EOT;
+Created alignment task. To start, run:
+cd talign/
+make
+
+To specify the walltime *in minutes* used by the optimizer, use
+make WALLTIME=120
+
+EOT
exit 0;
sub make_stage {
@@ -142,6 +157,8 @@ EOT
open AGENDA, ">$stage_dir/agenda.txt" or die "Can't write $stage_dir/agenda.txt: $!";
print AGENDA "cdec.ini $TRAINING_ITERATIONS\n";
close AGENDA;
+ `cp $SCRIPT_DIR/makefiles/makefile.model.$direction $stage_dir/Makefile`;
+ die unless $? == 0;
}
sub usage {
diff --git a/word-aligner/da.h b/word-aligner/da.h
new file mode 100644
index 00000000..c979b641
--- /dev/null
+++ b/word-aligner/da.h
@@ -0,0 +1,79 @@
+#ifndef _DA_H_
+#define _DA_H_
+
+#include <cmath>
+#include <cassert>
+
+// m = trg len
+// n = src len
+// i = trg index
+// j = src index
+struct DiagonalAlignment {
+
+ static double UnnormalizedProb(const unsigned i, const unsigned j, const unsigned m, const unsigned n, const double alpha) {
+#if 0
+ assert(i > 0);
+ assert(n > 0);
+ assert(m >= i);
+ assert(n >= j);
+#endif
+ return exp(Feature(i, j, m, n) * alpha);
+ }
+
+ static double ComputeZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) {
+#if 0
+ assert(i > 0);
+ assert(n > 0);
+ assert(m >= i);
+#endif
+ const double split = double(i) * n / m;
+ const unsigned floor = split;
+ unsigned ceil = floor + 1;
+ const double ratio = exp(-alpha / n);
+ const unsigned num_top = n - floor;
+ double ezt = 0;
+ double ezb = 0;
+ if (num_top)
+ ezt = UnnormalizedProb(i, ceil, m, n, alpha) * (1.0 - pow(ratio, num_top)) / (1.0 - ratio);
+ if (floor)
+ ezb = UnnormalizedProb(i, floor, m, n, alpha) * (1.0 - pow(ratio, floor)) / (1.0 - ratio);
+ return ezb + ezt;
+ }
+
+ static double ComputeDLogZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) {
+ const double z = ComputeZ(i, n, m, alpha);
+ const double split = double(i) * n / m;
+ const unsigned floor = split;
+ const unsigned ceil = floor + 1;
+ const double ratio = exp(-alpha / n);
+ const double d = -1.0 / n;
+ const unsigned num_top = n - floor;
+ double pct = 0;
+ double pcb = 0;
+ if (num_top) {
+ pct = arithmetico_geometric_series(Feature(i, ceil, m, n), UnnormalizedProb(i, ceil, m, n, alpha), ratio, d, num_top);
+ //cerr << "PCT = " << pct << endl;
+ }
+ if (floor) {
+ pcb = arithmetico_geometric_series(Feature(i, floor, m, n), UnnormalizedProb(i, floor, m, n, alpha), ratio, d, floor);
+ //cerr << "PCB = " << pcb << endl;
+ }
+ return (pct + pcb) / z;
+ }
+
+ inline static double Feature(const unsigned i, const unsigned j, const unsigned m, const unsigned n) {
+ return -fabs(double(j) / n - double(i) / m);
+ }
+
+ private:
+ inline static double arithmetico_geometric_series(const double a_1, const double g_1, const double r, const double d, const unsigned n) {
+ const double g_np1 = g_1 * pow(r, n);
+ const double a_n = d * (n - 1) + a_1;
+ const double x_1 = a_1 * g_1;
+ const double g_2 = g_1 * r;
+ const double rm1 = r - 1;
+ return (a_n * g_np1 - x_1) / rm1 - d*(g_np1 - g_2) / (rm1 * rm1);
+ }
+};
+
+#endif
diff --git a/word-aligner/fast_align.cc b/word-aligner/fast_align.cc
new file mode 100644
index 00000000..9eb1dbc6
--- /dev/null
+++ b/word-aligner/fast_align.cc
@@ -0,0 +1,310 @@
+#include <iostream>
+#include <cmath>
+#include <utility>
+#include <tr1/unordered_map>
+
+#include <boost/functional/hash.hpp>
+#include <boost/program_options.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include "m.h"
+#include "corpus_tools.h"
+#include "stringlib.h"
+#include "filelib.h"
+#include "ttables.h"
+#include "tdict.h"
+#include "da.h"
+
+namespace po = boost::program_options;
+using namespace std;
+using namespace std::tr1;
+
+bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
+ po::options_description opts("Configuration options");
+ opts.add_options()
+ ("input,i",po::value<string>(),"Parallel corpus input file")
+ ("reverse,r","Reverse estimation (swap source and target during training)")
+ ("iterations,I",po::value<unsigned>()->default_value(5),"Number of iterations of EM training")
+ //("bidir,b", "Run bidirectional alignment")
+ ("favor_diagonal,d", "Use a static alignment distribution that assigns higher probabilities to alignments near the diagonal")
+ ("prob_align_null", po::value<double>()->default_value(0.08), "When --favor_diagonal is set, what's the probability of a null alignment?")
+ ("diagonal_tension,T", po::value<double>()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)")
+ ("optimize_tension,o", "Optimize diagonal tension during EM")
+ ("variational_bayes,v","Infer VB estimate of parameters under a symmetric Dirichlet prior")
+ ("alpha,a", po::value<double>()->default_value(0.01), "Hyperparameter for optional Dirichlet prior")
+ ("no_null_word,N","Do not generate from a null token")
+ ("output_parameters,p", "Write model parameters instead of alignments")
+ ("beam_threshold,t",po::value<double>()->default_value(-4),"When writing parameters, log_10 of beam threshold for writing parameter (-10000 to include everything, 0 max parameter only)")
+ ("hide_training_alignments,H", "Hide training alignments (only useful if you want to use -x option and just compute testset statistics)")
+ ("testset,x", po::value<string>(), "After training completes, compute the log likelihood of this set of sentence pairs under the learned model")
+ ("no_add_viterbi,V","When writing model parameters, do not add Viterbi alignment points (may generate a grammar where some training sentence pairs are unreachable)");
+ po::options_description clo("Command line options");
+ clo.add_options()
+ ("config", po::value<string>(), "Configuration file")
+ ("help,h", "Print this help message and exit");
+ po::options_description dconfig_options, dcmdline_options;
+ dconfig_options.add(opts);
+ dcmdline_options.add(opts).add(clo);
+
+ po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
+ if (conf->count("config")) {
+ ifstream config((*conf)["config"].as<string>().c_str());
+ po::store(po::parse_config_file(config, dconfig_options), *conf);
+ }
+ po::notify(*conf);
+
+ if (conf->count("help") || conf->count("input") == 0) {
+ cerr << "Usage " << argv[0] << " [OPTIONS] -i corpus.fr-en\n";
+ cerr << dcmdline_options << endl;
+ return false;
+ }
+ return true;
+}
+
+int main(int argc, char** argv) {
+ po::variables_map conf;
+ if (!InitCommandLine(argc, argv, &conf)) return 1;
+ const string fname = conf["input"].as<string>();
+ const bool reverse = conf.count("reverse") > 0;
+ const int ITERATIONS = conf["iterations"].as<unsigned>();
+ const double BEAM_THRESHOLD = pow(10.0, conf["beam_threshold"].as<double>());
+ const bool use_null = (conf.count("no_null_word") == 0);
+ const WordID kNULL = TD::Convert("<eps>");
+ const bool add_viterbi = (conf.count("no_add_viterbi") == 0);
+ const bool variational_bayes = (conf.count("variational_bayes") > 0);
+ const bool write_alignments = (conf.count("output_parameters") == 0);
+ double diagonal_tension = conf["diagonal_tension"].as<double>();
+ bool optimize_tension = conf.count("optimize_tension");
+ const bool hide_training_alignments = (conf.count("hide_training_alignments") > 0);
+ string testset;
+ if (conf.count("testset")) testset = conf["testset"].as<string>();
+ double prob_align_null = conf["prob_align_null"].as<double>();
+ double prob_align_not_null = 1.0 - prob_align_null;
+ const double alpha = conf["alpha"].as<double>();
+ const bool favor_diagonal = conf.count("favor_diagonal");
+ if (variational_bayes && alpha <= 0.0) {
+ cerr << "--alpha must be > 0\n";
+ return 1;
+ }
+
+ TTable s2t, t2s;
+ TTable::Word2Word2Double s2t_viterbi;
+ unordered_map<pair<short, short>, unsigned, boost::hash<pair<short, short> > > size_counts;
+ double tot_len_ratio = 0;
+ double mean_srclen_multiplier = 0;
+ vector<double> probs;
+ for (int iter = 0; iter < ITERATIONS; ++iter) {
+ const bool final_iteration = (iter == (ITERATIONS - 1));
+ cerr << "ITERATION " << (iter + 1) << (final_iteration ? " (FINAL)" : "") << endl;
+ ReadFile rf(fname);
+ istream& in = *rf.stream();
+ double likelihood = 0;
+ double denom = 0.0;
+ int lc = 0;
+ bool flag = false;
+ string line;
+ string ssrc, strg;
+ vector<WordID> src, trg;
+ double c0 = 0;
+ double emp_feat = 0;
+ double toks = 0;
+ while(true) {
+ getline(in, line);
+ if (!in) break;
+ ++lc;
+ if (lc % 1000 == 0) { cerr << '.'; flag = true; }
+ if (lc %50000 == 0) { cerr << " [" << lc << "]\n" << flush; flag = false; }
+ src.clear(); trg.clear();
+ CorpusTools::ReadLine(line, &src, &trg);
+ if (reverse) swap(src, trg);
+ if (src.size() == 0 || trg.size() == 0) {
+ cerr << "Error: " << lc << "\n" << line << endl;
+ return 1;
+ }
+ if (iter == 0)
+ tot_len_ratio += static_cast<double>(trg.size()) / static_cast<double>(src.size());
+ denom += trg.size();
+ probs.resize(src.size() + 1);
+ if (iter == 0)
+ ++size_counts[make_pair<short,short>(trg.size(), src.size())];
+ bool first_al = true; // used for write_alignments
+ toks += trg.size();
+ for (unsigned j = 0; j < trg.size(); ++j) {
+ const WordID& f_j = trg[j];
+ double sum = 0;
+ double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1)
+ if (use_null) {
+ if (favor_diagonal) prob_a_i = prob_align_null;
+ probs[0] = s2t.prob(kNULL, f_j) * prob_a_i;
+ sum += probs[0];
+ }
+ double az = 0;
+ if (favor_diagonal)
+ az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null;
+ for (unsigned i = 1; i <= src.size(); ++i) {
+ if (favor_diagonal)
+ prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az;
+ probs[i] = s2t.prob(src[i-1], f_j) * prob_a_i;
+ sum += probs[i];
+ }
+ if (final_iteration) {
+ if (add_viterbi || write_alignments) {
+ WordID max_i = 0;
+ double max_p = -1;
+ int max_index = -1;
+ if (use_null) {
+ max_i = kNULL;
+ max_index = 0;
+ max_p = probs[0];
+ }
+ for (unsigned i = 1; i <= src.size(); ++i) {
+ if (probs[i] > max_p) {
+ max_index = i;
+ max_p = probs[i];
+ max_i = src[i-1];
+ }
+ }
+ if (!hide_training_alignments && write_alignments) {
+ if (max_index > 0) {
+ if (first_al) first_al = false; else cout << ' ';
+ if (reverse)
+ cout << j << '-' << (max_index - 1);
+ else
+ cout << (max_index - 1) << '-' << j;
+ }
+ }
+ s2t_viterbi[max_i][f_j] = 1.0;
+ }
+ } else {
+ if (use_null) {
+ double count = probs[0] / sum;
+ c0 += count;
+ s2t.Increment(kNULL, f_j, count);
+ }
+ for (unsigned i = 1; i <= src.size(); ++i) {
+ const double p = probs[i] / sum;
+ s2t.Increment(src[i-1], f_j, p);
+ emp_feat += DiagonalAlignment::Feature(j, i, trg.size(), src.size()) * p;
+ }
+ }
+ likelihood += log(sum);
+ }
+ if (write_alignments && final_iteration && !hide_training_alignments) cout << endl;
+ }
+
+ // log(e) = 1.0
+ double base2_likelihood = likelihood / log(2);
+
+ if (flag) { cerr << endl; }
+ if (iter == 0) {
+ mean_srclen_multiplier = tot_len_ratio / lc;
+ cerr << "expected target length = source length * " << mean_srclen_multiplier << endl;
+ }
+ emp_feat /= toks;
+ cerr << " log_e likelihood: " << likelihood << endl;
+ cerr << " log_2 likelihood: " << base2_likelihood << endl;
+ cerr << " cross entropy: " << (-base2_likelihood / denom) << endl;
+ cerr << " perplexity: " << pow(2.0, -base2_likelihood / denom) << endl;
+ cerr << " posterior p0: " << c0 / toks << endl;
+ cerr << " posterior al-feat: " << emp_feat << endl;
+ //cerr << " model tension: " << mod_feat / toks << endl;
+ cerr << " size counts: " << size_counts.size() << endl;
+ if (!final_iteration) {
+ if (favor_diagonal && optimize_tension && iter > 0) {
+ for (int ii = 0; ii < 8; ++ii) {
+ double mod_feat = 0;
+ unordered_map<pair<short,short>,unsigned,boost::hash<pair<short, short> > >::iterator it = size_counts.begin();
+ for(; it != size_counts.end(); ++it) {
+ const pair<short,short>& p = it->first;
+ for (short j = 1; j <= p.first; ++j)
+ mod_feat += it->second * DiagonalAlignment::ComputeDLogZ(j, p.first, p.second, diagonal_tension);
+ }
+ mod_feat /= toks;
+ cerr << " " << ii + 1 << " model al-feat: " << mod_feat << " (tension=" << diagonal_tension << ")\n";
+ diagonal_tension += (emp_feat - mod_feat) * 20.0;
+ if (diagonal_tension <= 0.1) diagonal_tension = 0.1;
+ if (diagonal_tension > 14) diagonal_tension = 14;
+ }
+ cerr << " final tension: " << diagonal_tension << endl;
+ }
+ if (variational_bayes)
+ s2t.NormalizeVB(alpha);
+ else
+ s2t.Normalize();
+ //prob_align_null *= 0.8; // XXX
+ //prob_align_null += (c0 / toks) * 0.2;
+ prob_align_not_null = 1.0 - prob_align_null;
+ }
+ }
+ if (testset.size()) {
+ ReadFile rf(testset);
+ istream& in = *rf.stream();
+ int lc = 0;
+ double tlp = 0;
+ string line;
+ while (getline(in, line)) {
+ ++lc;
+ vector<WordID> src, trg;
+ CorpusTools::ReadLine(line, &src, &trg);
+ cout << TD::GetString(src) << " ||| " << TD::GetString(trg) << " |||";
+ if (reverse) swap(src, trg);
+ double log_prob = Md::log_poisson(trg.size(), 0.05 + src.size() * mean_srclen_multiplier);
+
+ // compute likelihood
+ for (unsigned j = 0; j < trg.size(); ++j) {
+ const WordID& f_j = trg[j];
+ double sum = 0;
+ int a_j = 0;
+ double max_pat = 0;
+ double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1)
+ if (use_null) {
+ if (favor_diagonal) prob_a_i = prob_align_null;
+ max_pat = s2t.prob(kNULL, f_j) * prob_a_i;
+ sum += max_pat;
+ }
+ double az = 0;
+ if (favor_diagonal)
+ az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null;
+ for (unsigned i = 1; i <= src.size(); ++i) {
+ if (favor_diagonal)
+ prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az;
+ double pat = s2t.prob(src[i-1], f_j) * prob_a_i;
+ if (pat > max_pat) { max_pat = pat; a_j = i; }
+ sum += pat;
+ }
+ log_prob += log(sum);
+ if (write_alignments) {
+ if (a_j > 0) {
+ cout << ' ';
+ if (reverse)
+ cout << j << '-' << (a_j - 1);
+ else
+ cout << (a_j - 1) << '-' << j;
+ }
+ }
+ }
+ tlp += log_prob;
+ cout << " ||| " << log_prob << endl << flush;
+ } // loop over test set sentences
+ cerr << "TOTAL LOG PROB " << tlp << endl;
+ }
+
+ if (write_alignments) return 0;
+
+ for (TTable::Word2Word2Double::iterator ei = s2t.ttable.begin(); ei != s2t.ttable.end(); ++ei) {
+ const TTable::Word2Double& cpd = ei->second;
+ const TTable::Word2Double& vit = s2t_viterbi[ei->first];
+ const string& esym = TD::Convert(ei->first);
+ double max_p = -1;
+ for (TTable::Word2Double::const_iterator fi = cpd.begin(); fi != cpd.end(); ++fi)
+ if (fi->second > max_p) max_p = fi->second;
+ const double threshold = max_p * BEAM_THRESHOLD;
+ for (TTable::Word2Double::const_iterator fi = cpd.begin(); fi != cpd.end(); ++fi) {
+ if (fi->second > threshold || (vit.find(fi->first) != vit.end())) {
+ cout << esym << ' ' << TD::Convert(fi->first) << ' ' << log(fi->second) << endl;
+ }
+ }
+ }
+ return 0;
+}
+
diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars
index 08ff33e1..8d3ea8cb 100644
--- a/word-aligner/makefiles/makefile.grammars
+++ b/word-aligner/makefiles/makefile.grammars
@@ -4,7 +4,7 @@ clean:
$(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* *stem* freq* wordpairs*
SUPPORT_DIR = $(SCRIPT_DIR)/support
-GZIP = /usr/bin/gzip
+GZIP = gzip
ZCAT = zcat
EXTRACT_GRAMMAR = $(SUPPORT_DIR)/extract_grammar.pl
EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl
@@ -16,7 +16,7 @@ STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl
CLASSIFY = $(SUPPORT_DIR)/classify.pl
MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl
-MODEL1 = $(TRAINING_DIR)/fast_align
+MODEL1 = $(SCRIPT_DIR)/fast_align
MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl
e.voc: corpus.e
diff --git a/word-aligner/makefiles/makefile.model.f-e b/word-aligner/makefiles/makefile.model.f-e
new file mode 100644
index 00000000..404f5b30
--- /dev/null
+++ b/word-aligner/makefiles/makefile.model.f-e
@@ -0,0 +1,14 @@
+all: output.f-e.aligned
+
+clean:
+ $(RM) output.f-e.a weights.cur.gz
+
+CDEC = $(SCRIPT_DIR)/../decoder/cdec
+OPTIMIZE = $(SCRIPT_DIR)/../training/crf/mpi_online_optimize
+
+weights.cur.gz: ../grammars/wordpairs.f-e.features.gz
+ $(MPIRUN) $(OPTIMIZE) -a agenda.txt -1 -C 1.0 -t ../grammars/corpus.f-e --max_walltime 90
+
+output.f-e.aligned: weights.cur.gz
+ $(CDEC) -c cdec.ini -w $< --lextrans_align_only -i ../grammars/corpus.f-e -a > $@
+
diff --git a/word-aligner/ortho-norm/ru.pl b/word-aligner/ortho-norm/ru.pl
new file mode 100755
index 00000000..34452d06
--- /dev/null
+++ b/word-aligner/ortho-norm/ru.pl
@@ -0,0 +1,44 @@
+#!/usr/bin/perl -w
+use strict;
+use utf8;
+binmode(STDIN,":utf8");
+binmode(STDOUT,":utf8");
+while(<STDIN>) {
+ $_ = uc $_;
+ s/А/a/g;
+ s/І/i/g;
+ s/Б/b/g;
+ s/В/v/g;
+ s/Г/g/g;
+ s/Д/d/g;
+ s/Е/e/g;
+ s/Ж/zh/g;
+ s/З/z/g;
+ s/И/i/g;
+ s/Й/i/g;
+ s/К/k/g;
+ s/Л/l/g;
+ s/М/m/g;
+ s/Н/n/g;
+ s/О/o/g;
+ s/П/p/g;
+ s/Р/r/g;
+ s/С/s/g;
+ s/Т/t/g;
+ s/У/u/g;
+ s/Ф/f/g;
+ s/Х/kh/g;
+ s/Ц/c/g;
+ s/Ч/ch/g;
+ s/Ш/sh/g;
+ s/Щ/shch/g;
+ s/Ъ//g;
+ s/Ы//g;
+ s/Ь//g;
+ s/Э/e/g;
+ s/Ю/yo/g;
+ s/Я/ya/g;
+ $_ = lc $_;
+ print;
+}
+
diff --git a/word-aligner/paste-parallel-files.pl b/word-aligner/paste-parallel-files.pl
deleted file mode 100755
index ce53b325..00000000
--- a/word-aligner/paste-parallel-files.pl
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/perl -w
-use strict;
-
-my @fs = ();
-for my $file (@ARGV) {
- my $fh;
- open $fh, "<$file" or die "Can't open $file for reading: $!";
- push @fs, $fh;
-}
-my $num = scalar @fs;
-die "Usage: $0 file1.txt file2.txt [...]\n" unless $num > 1;
-
-my $first = $fs[0];
-while(<$first>) {
- chomp;
- my @out = ();
- push @out, $_;
- for (my $i=1; $i < $num; $i++) {
- my $f = $fs[$i];
- my $line = <$f>;
- die "Mismatched number of lines!" unless defined $line;
- chomp $line;
- push @out, $line;
- }
- print join(' ||| ', @out) . "\n";
-}
-
-for my $fh (@fs) {
- my $x=<$fh>;
- die "Mismatched number of lines!" if defined $x;
- close $fh;
-}
-
-exit 0;
-
diff --git a/word-aligner/ttables.cc b/word-aligner/ttables.cc
new file mode 100644
index 00000000..45bf14c5
--- /dev/null
+++ b/word-aligner/ttables.cc
@@ -0,0 +1,31 @@
+#include "ttables.h"
+
+#include <cassert>
+
+#include "dict.h"
+
+using namespace std;
+using namespace std::tr1;
+
+void TTable::DeserializeProbsFromText(std::istream* in) {
+ int c = 0;
+ while(*in) {
+ string e;
+ string f;
+ double p;
+ (*in) >> e >> f >> p;
+ if (e.empty()) break;
+ ++c;
+ ttable[TD::Convert(e)][TD::Convert(f)] = p;
+ }
+ cerr << "Loaded " << c << " translation parameters.\n";
+}
+
+void TTable::SerializeHelper(string* out, const Word2Word2Double& o) {
+ assert(!"not implemented");
+}
+
+void TTable::DeserializeHelper(const string& in, Word2Word2Double* o) {
+ assert(!"not implemented");
+}
+
diff --git a/word-aligner/ttables.h b/word-aligner/ttables.h
new file mode 100644
index 00000000..9baa13ca
--- /dev/null
+++ b/word-aligner/ttables.h
@@ -0,0 +1,101 @@
+#ifndef _TTABLES_H_
+#define _TTABLES_H_
+
+#include <iostream>
+#include <tr1/unordered_map>
+
+#include "sparse_vector.h"
+#include "m.h"
+#include "wordid.h"
+#include "tdict.h"
+
+class TTable {
+ public:
+ TTable() {}
+ typedef std::tr1::unordered_map<WordID, double> Word2Double;
+ typedef std::tr1::unordered_map<WordID, Word2Double> Word2Word2Double;
+ inline double prob(const int& e, const int& f) const {
+ const Word2Word2Double::const_iterator cit = ttable.find(e);
+ if (cit != ttable.end()) {
+ const Word2Double& cpd = cit->second;
+ const Word2Double::const_iterator it = cpd.find(f);
+ if (it == cpd.end()) return 1e-9;
+ return it->second;
+ } else {
+ return 1e-9;
+ }
+ }
+ inline void Increment(const int& e, const int& f) {
+ counts[e][f] += 1.0;
+ }
+ inline void Increment(const int& e, const int& f, double x) {
+ counts[e][f] += x;
+ }
+ void NormalizeVB(const double alpha) {
+ ttable.swap(counts);
+ for (Word2Word2Double::iterator cit = ttable.begin();
+ cit != ttable.end(); ++cit) {
+ double tot = 0;
+ Word2Double& cpd = cit->second;
+ for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+ tot += it->second + alpha;
+ for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+ it->second = exp(Md::digamma(it->second + alpha) - Md::digamma(tot));
+ }
+ counts.clear();
+ }
+ void Normalize() {
+ ttable.swap(counts);
+ for (Word2Word2Double::iterator cit = ttable.begin();
+ cit != ttable.end(); ++cit) {
+ double tot = 0;
+ Word2Double& cpd = cit->second;
+ for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+ tot += it->second;
+ for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+ it->second /= tot;
+ }
+ counts.clear();
+ }
+ // adds counts from another TTable - probabilities remain unchanged
+ TTable& operator+=(const TTable& rhs) {
+ for (Word2Word2Double::const_iterator it = rhs.counts.begin();
+ it != rhs.counts.end(); ++it) {
+ const Word2Double& cpd = it->second;
+ Word2Double& tgt = counts[it->first];
+ for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) {
+ tgt[j->first] += j->second;
+ }
+ }
+ return *this;
+ }
+ void ShowTTable() const {
+ for (Word2Word2Double::const_iterator it = ttable.begin(); it != ttable.end(); ++it) {
+ const Word2Double& cpd = it->second;
+ for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) {
+ std::cerr << "P(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl;
+ }
+ }
+ }
+ void ShowCounts() const {
+ for (Word2Word2Double::const_iterator it = counts.begin(); it != counts.end(); ++it) {
+ const Word2Double& cpd = it->second;
+ for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) {
+ std::cerr << "c(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl;
+ }
+ }
+ }
+ void DeserializeProbsFromText(std::istream* in);
+ void SerializeCounts(std::string* out) const { SerializeHelper(out, counts); }
+ void DeserializeCounts(const std::string& in) { DeserializeHelper(in, &counts); }
+ void SerializeProbs(std::string* out) const { SerializeHelper(out, ttable); }
+ void DeserializeProbs(const std::string& in) { DeserializeHelper(in, &ttable); }
+ private:
+ static void SerializeHelper(std::string*, const Word2Word2Double& o);
+ static void DeserializeHelper(const std::string&, Word2Word2Double* o);
+ public:
+ Word2Word2Double ttable;
+ Word2Word2Double counts;
+};
+
+#endif