diff options
Diffstat (limited to 'word-aligner')
| -rw-r--r-- | word-aligner/Makefile.am | 8 | ||||
| -rwxr-xr-x | word-aligner/aligner.pl | 19 | ||||
| -rw-r--r-- | word-aligner/da.h | 79 | ||||
| -rw-r--r-- | word-aligner/fast_align.cc | 310 | ||||
| -rw-r--r-- | word-aligner/makefiles/makefile.grammars | 4 | ||||
| -rw-r--r-- | word-aligner/makefiles/makefile.model.f-e | 14 | ||||
| -rwxr-xr-x | word-aligner/ortho-norm/ru.pl | 44 | ||||
| -rwxr-xr-x | word-aligner/paste-parallel-files.pl | 35 | ||||
| -rw-r--r-- | word-aligner/ttables.cc | 31 | ||||
| -rw-r--r-- | word-aligner/ttables.h | 101 |
10 files changed, 607 insertions, 38 deletions
diff --git a/word-aligner/Makefile.am b/word-aligner/Makefile.am new file mode 100644 index 00000000..1f7f78ae --- /dev/null +++ b/word-aligner/Makefile.am @@ -0,0 +1,8 @@ +bin_PROGRAMS = fast_align + +fast_align_SOURCES = fast_align.cc ttables.cc da.h ttables.h +fast_align_LDADD = ../utils/libutils.a + +EXTRA_DIST = aligner.pl ortho-norm support makefiles stemmers + +AM_CPPFLAGS = -W -Wall -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/training diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl index c5078645..cbccb94a 100755 --- a/word-aligner/aligner.pl +++ b/word-aligner/aligner.pl @@ -51,6 +51,8 @@ while(<IN>) { chomp; my ($f, $e) = split / \|\|\| /; die "Bad format, excepted ||| separated line" unless defined $f && defined $e; + $f =~ s/\[/(/g; + $e =~ s/\]/)/g; print F "$f\n"; print E "$e\n"; } @@ -80,6 +82,11 @@ NCLASSES = $num_classes TARGETS = @targets PTRAIN = \$(TRAINING_DIR)/cluster-ptrain.pl --restart_if_necessary PTRAIN_PARAMS = --gaussian_prior --sigma_squared 1.0 --max_iteration 15 +#MPIJOBS = 4 +#MPIRUN = mpirun -np $(MPIJOBS) +MPIRUN= + +WALLTIME=90 export @@ -99,7 +106,15 @@ clean: EOT close TOPLEVEL; -print STDERR "Created alignment task. chdir to talign/ then type make.\n\n"; +print STDERR <<EOT; +Created alignment task. To start, run: +cd talign/ +make + +To specify the walltime *in minutes* used by the optimizer, use +make WALLTIME=120 + +EOT exit 0; sub make_stage { @@ -142,6 +157,8 @@ EOT open AGENDA, ">$stage_dir/agenda.txt" or die "Can't write $stage_dir/agenda.txt: $!"; print AGENDA "cdec.ini $TRAINING_ITERATIONS\n"; close AGENDA; + `cp $SCRIPT_DIR/makefiles/makefile.model.$direction $stage_dir/Makefile`; + die unless $? == 0; } sub usage { diff --git a/word-aligner/da.h b/word-aligner/da.h new file mode 100644 index 00000000..c979b641 --- /dev/null +++ b/word-aligner/da.h @@ -0,0 +1,79 @@ +#ifndef _DA_H_ +#define _DA_H_ + +#include <cmath> +#include <cassert> + +// m = trg len +// n = src len +// i = trg index +// j = src index +struct DiagonalAlignment { + + static double UnnormalizedProb(const unsigned i, const unsigned j, const unsigned m, const unsigned n, const double alpha) { +#if 0 + assert(i > 0); + assert(n > 0); + assert(m >= i); + assert(n >= j); +#endif + return exp(Feature(i, j, m, n) * alpha); + } + + static double ComputeZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) { +#if 0 + assert(i > 0); + assert(n > 0); + assert(m >= i); +#endif + const double split = double(i) * n / m; + const unsigned floor = split; + unsigned ceil = floor + 1; + const double ratio = exp(-alpha / n); + const unsigned num_top = n - floor; + double ezt = 0; + double ezb = 0; + if (num_top) + ezt = UnnormalizedProb(i, ceil, m, n, alpha) * (1.0 - pow(ratio, num_top)) / (1.0 - ratio); + if (floor) + ezb = UnnormalizedProb(i, floor, m, n, alpha) * (1.0 - pow(ratio, floor)) / (1.0 - ratio); + return ezb + ezt; + } + + static double ComputeDLogZ(const unsigned i, const unsigned m, const unsigned n, const double alpha) { + const double z = ComputeZ(i, n, m, alpha); + const double split = double(i) * n / m; + const unsigned floor = split; + const unsigned ceil = floor + 1; + const double ratio = exp(-alpha / n); + const double d = -1.0 / n; + const unsigned num_top = n - floor; + double pct = 0; + double pcb = 0; + if (num_top) { + pct = arithmetico_geometric_series(Feature(i, ceil, m, n), UnnormalizedProb(i, ceil, m, n, alpha), ratio, d, num_top); + //cerr << "PCT = " << pct << endl; + } + if (floor) { + pcb = arithmetico_geometric_series(Feature(i, floor, m, n), UnnormalizedProb(i, floor, m, n, alpha), ratio, d, floor); + //cerr << "PCB = " << pcb << endl; + } + return (pct + pcb) / z; + } + + inline static double Feature(const unsigned i, const unsigned j, const unsigned m, const unsigned n) { + return -fabs(double(j) / n - double(i) / m); + } + + private: + inline static double arithmetico_geometric_series(const double a_1, const double g_1, const double r, const double d, const unsigned n) { + const double g_np1 = g_1 * pow(r, n); + const double a_n = d * (n - 1) + a_1; + const double x_1 = a_1 * g_1; + const double g_2 = g_1 * r; + const double rm1 = r - 1; + return (a_n * g_np1 - x_1) / rm1 - d*(g_np1 - g_2) / (rm1 * rm1); + } +}; + +#endif diff --git a/word-aligner/fast_align.cc b/word-aligner/fast_align.cc new file mode 100644 index 00000000..9eb1dbc6 --- /dev/null +++ b/word-aligner/fast_align.cc @@ -0,0 +1,310 @@ +#include <iostream> +#include <cmath> +#include <utility> +#include <tr1/unordered_map> + +#include <boost/functional/hash.hpp> +#include <boost/program_options.hpp> +#include <boost/program_options/variables_map.hpp> + +#include "m.h" +#include "corpus_tools.h" +#include "stringlib.h" +#include "filelib.h" +#include "ttables.h" +#include "tdict.h" +#include "da.h" + +namespace po = boost::program_options; +using namespace std; +using namespace std::tr1; + +bool InitCommandLine(int argc, char** argv, po::variables_map* conf) { + po::options_description opts("Configuration options"); + opts.add_options() + ("input,i",po::value<string>(),"Parallel corpus input file") + ("reverse,r","Reverse estimation (swap source and target during training)") + ("iterations,I",po::value<unsigned>()->default_value(5),"Number of iterations of EM training") + //("bidir,b", "Run bidirectional alignment") + ("favor_diagonal,d", "Use a static alignment distribution that assigns higher probabilities to alignments near the diagonal") + ("prob_align_null", po::value<double>()->default_value(0.08), "When --favor_diagonal is set, what's the probability of a null alignment?") + ("diagonal_tension,T", po::value<double>()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)") + ("optimize_tension,o", "Optimize diagonal tension during EM") + ("variational_bayes,v","Infer VB estimate of parameters under a symmetric Dirichlet prior") + ("alpha,a", po::value<double>()->default_value(0.01), "Hyperparameter for optional Dirichlet prior") + ("no_null_word,N","Do not generate from a null token") + ("output_parameters,p", "Write model parameters instead of alignments") + ("beam_threshold,t",po::value<double>()->default_value(-4),"When writing parameters, log_10 of beam threshold for writing parameter (-10000 to include everything, 0 max parameter only)") + ("hide_training_alignments,H", "Hide training alignments (only useful if you want to use -x option and just compute testset statistics)") + ("testset,x", po::value<string>(), "After training completes, compute the log likelihood of this set of sentence pairs under the learned model") + ("no_add_viterbi,V","When writing model parameters, do not add Viterbi alignment points (may generate a grammar where some training sentence pairs are unreachable)"); + po::options_description clo("Command line options"); + clo.add_options() + ("config", po::value<string>(), "Configuration file") + ("help,h", "Print this help message and exit"); + po::options_description dconfig_options, dcmdline_options; + dconfig_options.add(opts); + dcmdline_options.add(opts).add(clo); + + po::store(parse_command_line(argc, argv, dcmdline_options), *conf); + if (conf->count("config")) { + ifstream config((*conf)["config"].as<string>().c_str()); + po::store(po::parse_config_file(config, dconfig_options), *conf); + } + po::notify(*conf); + + if (conf->count("help") || conf->count("input") == 0) { + cerr << "Usage " << argv[0] << " [OPTIONS] -i corpus.fr-en\n"; + cerr << dcmdline_options << endl; + return false; + } + return true; +} + +int main(int argc, char** argv) { + po::variables_map conf; + if (!InitCommandLine(argc, argv, &conf)) return 1; + const string fname = conf["input"].as<string>(); + const bool reverse = conf.count("reverse") > 0; + const int ITERATIONS = conf["iterations"].as<unsigned>(); + const double BEAM_THRESHOLD = pow(10.0, conf["beam_threshold"].as<double>()); + const bool use_null = (conf.count("no_null_word") == 0); + const WordID kNULL = TD::Convert("<eps>"); + const bool add_viterbi = (conf.count("no_add_viterbi") == 0); + const bool variational_bayes = (conf.count("variational_bayes") > 0); + const bool write_alignments = (conf.count("output_parameters") == 0); + double diagonal_tension = conf["diagonal_tension"].as<double>(); + bool optimize_tension = conf.count("optimize_tension"); + const bool hide_training_alignments = (conf.count("hide_training_alignments") > 0); + string testset; + if (conf.count("testset")) testset = conf["testset"].as<string>(); + double prob_align_null = conf["prob_align_null"].as<double>(); + double prob_align_not_null = 1.0 - prob_align_null; + const double alpha = conf["alpha"].as<double>(); + const bool favor_diagonal = conf.count("favor_diagonal"); + if (variational_bayes && alpha <= 0.0) { + cerr << "--alpha must be > 0\n"; + return 1; + } + + TTable s2t, t2s; + TTable::Word2Word2Double s2t_viterbi; + unordered_map<pair<short, short>, unsigned, boost::hash<pair<short, short> > > size_counts; + double tot_len_ratio = 0; + double mean_srclen_multiplier = 0; + vector<double> probs; + for (int iter = 0; iter < ITERATIONS; ++iter) { + const bool final_iteration = (iter == (ITERATIONS - 1)); + cerr << "ITERATION " << (iter + 1) << (final_iteration ? " (FINAL)" : "") << endl; + ReadFile rf(fname); + istream& in = *rf.stream(); + double likelihood = 0; + double denom = 0.0; + int lc = 0; + bool flag = false; + string line; + string ssrc, strg; + vector<WordID> src, trg; + double c0 = 0; + double emp_feat = 0; + double toks = 0; + while(true) { + getline(in, line); + if (!in) break; + ++lc; + if (lc % 1000 == 0) { cerr << '.'; flag = true; } + if (lc %50000 == 0) { cerr << " [" << lc << "]\n" << flush; flag = false; } + src.clear(); trg.clear(); + CorpusTools::ReadLine(line, &src, &trg); + if (reverse) swap(src, trg); + if (src.size() == 0 || trg.size() == 0) { + cerr << "Error: " << lc << "\n" << line << endl; + return 1; + } + if (iter == 0) + tot_len_ratio += static_cast<double>(trg.size()) / static_cast<double>(src.size()); + denom += trg.size(); + probs.resize(src.size() + 1); + if (iter == 0) + ++size_counts[make_pair<short,short>(trg.size(), src.size())]; + bool first_al = true; // used for write_alignments + toks += trg.size(); + for (unsigned j = 0; j < trg.size(); ++j) { + const WordID& f_j = trg[j]; + double sum = 0; + double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1) + if (use_null) { + if (favor_diagonal) prob_a_i = prob_align_null; + probs[0] = s2t.prob(kNULL, f_j) * prob_a_i; + sum += probs[0]; + } + double az = 0; + if (favor_diagonal) + az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null; + for (unsigned i = 1; i <= src.size(); ++i) { + if (favor_diagonal) + prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az; + probs[i] = s2t.prob(src[i-1], f_j) * prob_a_i; + sum += probs[i]; + } + if (final_iteration) { + if (add_viterbi || write_alignments) { + WordID max_i = 0; + double max_p = -1; + int max_index = -1; + if (use_null) { + max_i = kNULL; + max_index = 0; + max_p = probs[0]; + } + for (unsigned i = 1; i <= src.size(); ++i) { + if (probs[i] > max_p) { + max_index = i; + max_p = probs[i]; + max_i = src[i-1]; + } + } + if (!hide_training_alignments && write_alignments) { + if (max_index > 0) { + if (first_al) first_al = false; else cout << ' '; + if (reverse) + cout << j << '-' << (max_index - 1); + else + cout << (max_index - 1) << '-' << j; + } + } + s2t_viterbi[max_i][f_j] = 1.0; + } + } else { + if (use_null) { + double count = probs[0] / sum; + c0 += count; + s2t.Increment(kNULL, f_j, count); + } + for (unsigned i = 1; i <= src.size(); ++i) { + const double p = probs[i] / sum; + s2t.Increment(src[i-1], f_j, p); + emp_feat += DiagonalAlignment::Feature(j, i, trg.size(), src.size()) * p; + } + } + likelihood += log(sum); + } + if (write_alignments && final_iteration && !hide_training_alignments) cout << endl; + } + + // log(e) = 1.0 + double base2_likelihood = likelihood / log(2); + + if (flag) { cerr << endl; } + if (iter == 0) { + mean_srclen_multiplier = tot_len_ratio / lc; + cerr << "expected target length = source length * " << mean_srclen_multiplier << endl; + } + emp_feat /= toks; + cerr << " log_e likelihood: " << likelihood << endl; + cerr << " log_2 likelihood: " << base2_likelihood << endl; + cerr << " cross entropy: " << (-base2_likelihood / denom) << endl; + cerr << " perplexity: " << pow(2.0, -base2_likelihood / denom) << endl; + cerr << " posterior p0: " << c0 / toks << endl; + cerr << " posterior al-feat: " << emp_feat << endl; + //cerr << " model tension: " << mod_feat / toks << endl; + cerr << " size counts: " << size_counts.size() << endl; + if (!final_iteration) { + if (favor_diagonal && optimize_tension && iter > 0) { + for (int ii = 0; ii < 8; ++ii) { + double mod_feat = 0; + unordered_map<pair<short,short>,unsigned,boost::hash<pair<short, short> > >::iterator it = size_counts.begin(); + for(; it != size_counts.end(); ++it) { + const pair<short,short>& p = it->first; + for (short j = 1; j <= p.first; ++j) + mod_feat += it->second * DiagonalAlignment::ComputeDLogZ(j, p.first, p.second, diagonal_tension); + } + mod_feat /= toks; + cerr << " " << ii + 1 << " model al-feat: " << mod_feat << " (tension=" << diagonal_tension << ")\n"; + diagonal_tension += (emp_feat - mod_feat) * 20.0; + if (diagonal_tension <= 0.1) diagonal_tension = 0.1; + if (diagonal_tension > 14) diagonal_tension = 14; + } + cerr << " final tension: " << diagonal_tension << endl; + } + if (variational_bayes) + s2t.NormalizeVB(alpha); + else + s2t.Normalize(); + //prob_align_null *= 0.8; // XXX + //prob_align_null += (c0 / toks) * 0.2; + prob_align_not_null = 1.0 - prob_align_null; + } + } + if (testset.size()) { + ReadFile rf(testset); + istream& in = *rf.stream(); + int lc = 0; + double tlp = 0; + string line; + while (getline(in, line)) { + ++lc; + vector<WordID> src, trg; + CorpusTools::ReadLine(line, &src, &trg); + cout << TD::GetString(src) << " ||| " << TD::GetString(trg) << " |||"; + if (reverse) swap(src, trg); + double log_prob = Md::log_poisson(trg.size(), 0.05 + src.size() * mean_srclen_multiplier); + + // compute likelihood + for (unsigned j = 0; j < trg.size(); ++j) { + const WordID& f_j = trg[j]; + double sum = 0; + int a_j = 0; + double max_pat = 0; + double prob_a_i = 1.0 / (src.size() + use_null); // uniform (model 1) + if (use_null) { + if (favor_diagonal) prob_a_i = prob_align_null; + max_pat = s2t.prob(kNULL, f_j) * prob_a_i; + sum += max_pat; + } + double az = 0; + if (favor_diagonal) + az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null; + for (unsigned i = 1; i <= src.size(); ++i) { + if (favor_diagonal) + prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az; + double pat = s2t.prob(src[i-1], f_j) * prob_a_i; + if (pat > max_pat) { max_pat = pat; a_j = i; } + sum += pat; + } + log_prob += log(sum); + if (write_alignments) { + if (a_j > 0) { + cout << ' '; + if (reverse) + cout << j << '-' << (a_j - 1); + else + cout << (a_j - 1) << '-' << j; + } + } + } + tlp += log_prob; + cout << " ||| " << log_prob << endl << flush; + } // loop over test set sentences + cerr << "TOTAL LOG PROB " << tlp << endl; + } + + if (write_alignments) return 0; + + for (TTable::Word2Word2Double::iterator ei = s2t.ttable.begin(); ei != s2t.ttable.end(); ++ei) { + const TTable::Word2Double& cpd = ei->second; + const TTable::Word2Double& vit = s2t_viterbi[ei->first]; + const string& esym = TD::Convert(ei->first); + double max_p = -1; + for (TTable::Word2Double::const_iterator fi = cpd.begin(); fi != cpd.end(); ++fi) + if (fi->second > max_p) max_p = fi->second; + const double threshold = max_p * BEAM_THRESHOLD; + for (TTable::Word2Double::const_iterator fi = cpd.begin(); fi != cpd.end(); ++fi) { + if (fi->second > threshold || (vit.find(fi->first) != vit.end())) { + cout << esym << ' ' << TD::Convert(fi->first) << ' ' << log(fi->second) << endl; + } + } + } + return 0; +} + diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars index 08ff33e1..8d3ea8cb 100644 --- a/word-aligner/makefiles/makefile.grammars +++ b/word-aligner/makefiles/makefile.grammars @@ -4,7 +4,7 @@ clean: $(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar* *stem* freq* wordpairs* SUPPORT_DIR = $(SCRIPT_DIR)/support -GZIP = /usr/bin/gzip +GZIP = gzip ZCAT = zcat EXTRACT_GRAMMAR = $(SUPPORT_DIR)/extract_grammar.pl EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl @@ -16,7 +16,7 @@ STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl CLASSIFY = $(SUPPORT_DIR)/classify.pl MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl -MODEL1 = $(TRAINING_DIR)/fast_align +MODEL1 = $(SCRIPT_DIR)/fast_align MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl e.voc: corpus.e diff --git a/word-aligner/makefiles/makefile.model.f-e b/word-aligner/makefiles/makefile.model.f-e new file mode 100644 index 00000000..404f5b30 --- /dev/null +++ b/word-aligner/makefiles/makefile.model.f-e @@ -0,0 +1,14 @@ +all: output.f-e.aligned + +clean: + $(RM) output.f-e.a weights.cur.gz + +CDEC = $(SCRIPT_DIR)/../decoder/cdec +OPTIMIZE = $(SCRIPT_DIR)/../training/crf/mpi_online_optimize + +weights.cur.gz: ../grammars/wordpairs.f-e.features.gz + $(MPIRUN) $(OPTIMIZE) -a agenda.txt -1 -C 1.0 -t ../grammars/corpus.f-e --max_walltime 90 + +output.f-e.aligned: weights.cur.gz + $(CDEC) -c cdec.ini -w $< --lextrans_align_only -i ../grammars/corpus.f-e -a > $@ + diff --git a/word-aligner/ortho-norm/ru.pl b/word-aligner/ortho-norm/ru.pl new file mode 100755 index 00000000..34452d06 --- /dev/null +++ b/word-aligner/ortho-norm/ru.pl @@ -0,0 +1,44 @@ +#!/usr/bin/perl -w +use strict; +use utf8; +binmode(STDIN,":utf8"); +binmode(STDOUT,":utf8"); +while(<STDIN>) { + $_ = uc $_; + s/А/a/g; + s/І/i/g; + s/Б/b/g; + s/В/v/g; + s/Г/g/g; + s/Д/d/g; + s/Е/e/g; + s/Ж/zh/g; + s/З/z/g; + s/И/i/g; + s/Й/i/g; + s/К/k/g; + s/Л/l/g; + s/М/m/g; + s/Н/n/g; + s/О/o/g; + s/П/p/g; + s/Р/r/g; + s/С/s/g; + s/Т/t/g; + s/У/u/g; + s/Ф/f/g; + s/Х/kh/g; + s/Ц/c/g; + s/Ч/ch/g; + s/Ш/sh/g; + s/Щ/shch/g; + s/Ъ//g; + s/Ы//g; + s/Ь//g; + s/Э/e/g; + s/Ю/yo/g; + s/Я/ya/g; + $_ = lc $_; + print; +} + diff --git a/word-aligner/paste-parallel-files.pl b/word-aligner/paste-parallel-files.pl deleted file mode 100755 index ce53b325..00000000 --- a/word-aligner/paste-parallel-files.pl +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my @fs = (); -for my $file (@ARGV) { - my $fh; - open $fh, "<$file" or die "Can't open $file for reading: $!"; - push @fs, $fh; -} -my $num = scalar @fs; -die "Usage: $0 file1.txt file2.txt [...]\n" unless $num > 1; - -my $first = $fs[0]; -while(<$first>) { - chomp; - my @out = (); - push @out, $_; - for (my $i=1; $i < $num; $i++) { - my $f = $fs[$i]; - my $line = <$f>; - die "Mismatched number of lines!" unless defined $line; - chomp $line; - push @out, $line; - } - print join(' ||| ', @out) . "\n"; -} - -for my $fh (@fs) { - my $x=<$fh>; - die "Mismatched number of lines!" if defined $x; - close $fh; -} - -exit 0; - diff --git a/word-aligner/ttables.cc b/word-aligner/ttables.cc new file mode 100644 index 00000000..45bf14c5 --- /dev/null +++ b/word-aligner/ttables.cc @@ -0,0 +1,31 @@ +#include "ttables.h" + +#include <cassert> + +#include "dict.h" + +using namespace std; +using namespace std::tr1; + +void TTable::DeserializeProbsFromText(std::istream* in) { + int c = 0; + while(*in) { + string e; + string f; + double p; + (*in) >> e >> f >> p; + if (e.empty()) break; + ++c; + ttable[TD::Convert(e)][TD::Convert(f)] = p; + } + cerr << "Loaded " << c << " translation parameters.\n"; +} + +void TTable::SerializeHelper(string* out, const Word2Word2Double& o) { + assert(!"not implemented"); +} + +void TTable::DeserializeHelper(const string& in, Word2Word2Double* o) { + assert(!"not implemented"); +} + diff --git a/word-aligner/ttables.h b/word-aligner/ttables.h new file mode 100644 index 00000000..9baa13ca --- /dev/null +++ b/word-aligner/ttables.h @@ -0,0 +1,101 @@ +#ifndef _TTABLES_H_ +#define _TTABLES_H_ + +#include <iostream> +#include <tr1/unordered_map> + +#include "sparse_vector.h" +#include "m.h" +#include "wordid.h" +#include "tdict.h" + +class TTable { + public: + TTable() {} + typedef std::tr1::unordered_map<WordID, double> Word2Double; + typedef std::tr1::unordered_map<WordID, Word2Double> Word2Word2Double; + inline double prob(const int& e, const int& f) const { + const Word2Word2Double::const_iterator cit = ttable.find(e); + if (cit != ttable.end()) { + const Word2Double& cpd = cit->second; + const Word2Double::const_iterator it = cpd.find(f); + if (it == cpd.end()) return 1e-9; + return it->second; + } else { + return 1e-9; + } + } + inline void Increment(const int& e, const int& f) { + counts[e][f] += 1.0; + } + inline void Increment(const int& e, const int& f, double x) { + counts[e][f] += x; + } + void NormalizeVB(const double alpha) { + ttable.swap(counts); + for (Word2Word2Double::iterator cit = ttable.begin(); + cit != ttable.end(); ++cit) { + double tot = 0; + Word2Double& cpd = cit->second; + for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) + tot += it->second + alpha; + for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) + it->second = exp(Md::digamma(it->second + alpha) - Md::digamma(tot)); + } + counts.clear(); + } + void Normalize() { + ttable.swap(counts); + for (Word2Word2Double::iterator cit = ttable.begin(); + cit != ttable.end(); ++cit) { + double tot = 0; + Word2Double& cpd = cit->second; + for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) + tot += it->second; + for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it) + it->second /= tot; + } + counts.clear(); + } + // adds counts from another TTable - probabilities remain unchanged + TTable& operator+=(const TTable& rhs) { + for (Word2Word2Double::const_iterator it = rhs.counts.begin(); + it != rhs.counts.end(); ++it) { + const Word2Double& cpd = it->second; + Word2Double& tgt = counts[it->first]; + for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) { + tgt[j->first] += j->second; + } + } + return *this; + } + void ShowTTable() const { + for (Word2Word2Double::const_iterator it = ttable.begin(); it != ttable.end(); ++it) { + const Word2Double& cpd = it->second; + for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) { + std::cerr << "P(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl; + } + } + } + void ShowCounts() const { + for (Word2Word2Double::const_iterator it = counts.begin(); it != counts.end(); ++it) { + const Word2Double& cpd = it->second; + for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) { + std::cerr << "c(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl; + } + } + } + void DeserializeProbsFromText(std::istream* in); + void SerializeCounts(std::string* out) const { SerializeHelper(out, counts); } + void DeserializeCounts(const std::string& in) { DeserializeHelper(in, &counts); } + void SerializeProbs(std::string* out) const { SerializeHelper(out, ttable); } + void DeserializeProbs(const std::string& in) { DeserializeHelper(in, &ttable); } + private: + static void SerializeHelper(std::string*, const Word2Word2Double& o); + static void DeserializeHelper(const std::string&, Word2Word2Double* o); + public: + Word2Word2Double ttable; + Word2Word2Double counts; +}; + +#endif |