summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Dyer <cdyer@cs.cmu.edu>2012-09-20 21:51:11 -0400
committerChris Dyer <cdyer@cs.cmu.edu>2012-09-20 21:51:11 -0400
commit78518f1f417616633b300a361cd5e0c1bcb1ff24 (patch)
tree5298351747288698d1564192349599f2ad4c1f35
parent90ea67a64e94d2e7464bcd9c5b908c09e2271fdc (diff)
rename model1 to fast_aligner, since it does way more than model1 but is mostly just fast
-rw-r--r--training/Makefile.am6
-rw-r--r--training/fast_align.cc (renamed from training/model1.cc)39
-rw-r--r--word-aligner/makefiles/makefile.grammars10
3 files changed, 31 insertions, 24 deletions
diff --git a/training/Makefile.am b/training/Makefile.am
index 4cef0d5b..5254333a 100644
--- a/training/Makefile.am
+++ b/training/Makefile.am
@@ -1,5 +1,5 @@
bin_PROGRAMS = \
- model1 \
+ fast_align \
lbl_model \
test_ngram \
mr_em_map_adapter \
@@ -55,8 +55,8 @@ augment_grammar_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/lib
test_ngram_SOURCES = test_ngram.cc
test_ngram_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a ../klm/lm/libklm.a ../klm/util/libklm_util.a -lz
-model1_SOURCES = model1.cc ttables.cc
-model1_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz
+fast_align_SOURCES = fast_align.cc ttables.cc
+fast_align_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz
lbl_model_SOURCES = lbl_model.cc
lbl_model_LDADD = libtraining.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz
diff --git a/training/model1.cc b/training/fast_align.cc
index 19692b9a..0d7b0202 100644
--- a/training/model1.cc
+++ b/training/fast_align.cc
@@ -17,18 +17,20 @@ using namespace std;
bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
po::options_description opts("Configuration options");
opts.add_options()
- ("iterations,i",po::value<unsigned>()->default_value(5),"Number of iterations of EM training")
- ("beam_threshold,t",po::value<double>()->default_value(-4),"log_10 of beam threshold (-10000 to include everything, 0 max)")
- ("bidir,b", "Run bidirectional alignment")
- ("no_null_word,N","Do not generate from the null token")
- ("write_alignments,A", "Write alignments instead of parameters")
+ ("input,i",po::value<string>(),"Parallel corpus input file")
+ ("reverse,r","Reverse estimation (swap source and target during training)")
+ ("iterations,I",po::value<unsigned>()->default_value(5),"Number of iterations of EM training")
+ //("bidir,b", "Run bidirectional alignment")
("favor_diagonal,d", "Use a static alignment distribution that assigns higher probabilities to alignments near the diagonal")
- ("diagonal_tension,T", po::value<double>()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)")
("prob_align_null", po::value<double>()->default_value(0.08), "When --favor_diagonal is set, what's the probability of a null alignment?")
- ("variational_bayes,v","Add a symmetric Dirichlet prior and infer VB estimate of weights")
- ("testset,x", po::value<string>(), "After training completes, compute the log likelihood of this set of sentence pairs under the learned model")
+ ("diagonal_tension,T", po::value<double>()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)")
+ ("variational_bayes,v","Infer VB estimate of parameters under a symmetric Dirichlet prior")
("alpha,a", po::value<double>()->default_value(0.01), "Hyperparameter for optional Dirichlet prior")
- ("no_add_viterbi,V","Do not add Viterbi alignment points (may generate a grammar where some training sentence pairs are unreachable)");
+ ("no_null_word,N","Do not generate from a null token")
+ ("output_parameters,p", "Write model parameters instead of alignments")
+ ("beam_threshold,t",po::value<double>()->default_value(-4),"When writing parameters, log_10 of beam threshold for writing parameter (-10000 to include everything, 0 max parameter only)")
+ ("testset,x", po::value<string>(), "After training completes, compute the log likelihood of this set of sentence pairs under the learned model")
+ ("no_add_viterbi,V","When writing model parameters, do not add Viterbi alignment points (may generate a grammar where some training sentence pairs are unreachable)");
po::options_description clo("Command line options");
clo.add_options()
("config", po::value<string>(), "Configuration file")
@@ -44,15 +46,14 @@ bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
}
po::notify(*conf);
- if (argc < 2 || conf->count("help")) {
- cerr << "Usage " << argv[0] << " [OPTIONS] corpus.fr-en\n";
+ if (conf->count("help") || conf->count("input") == 0) {
+ cerr << "Usage " << argv[0] << " [OPTIONS] -i corpus.fr-en\n";
cerr << dcmdline_options << endl;
return false;
}
return true;
}
-// src and trg are source and target strings, respectively (not really lattices)
double PosteriorInference(const vector<WordID>& src, const vector<WordID>& trg) {
double llh = 0;
static vector<double> unnormed_a_i;
@@ -64,14 +65,15 @@ double PosteriorInference(const vector<WordID>& src, const vector<WordID>& trg)
int main(int argc, char** argv) {
po::variables_map conf;
if (!InitCommandLine(argc, argv, &conf)) return 1;
- const string fname = argv[argc - 1];
+ const string fname = conf["input"].as<string>();
+ const bool reverse = conf.count("reverse") > 0;
const int ITERATIONS = conf["iterations"].as<unsigned>();
const double BEAM_THRESHOLD = pow(10.0, conf["beam_threshold"].as<double>());
const bool use_null = (conf.count("no_null_word") == 0);
const WordID kNULL = TD::Convert("<eps>");
const bool add_viterbi = (conf.count("no_add_viterbi") == 0);
const bool variational_bayes = (conf.count("variational_bayes") > 0);
- const bool write_alignments = (conf.count("write_alignments") > 0);
+ const bool write_alignments = (conf.count("output_parameters") == 0);
const double diagonal_tension = conf["diagonal_tension"].as<double>();
const double prob_align_null = conf["prob_align_null"].as<double>();
string testset;
@@ -100,14 +102,16 @@ int main(int argc, char** argv) {
bool flag = false;
string line;
string ssrc, strg;
+ vector<WordID> src, trg;
while(true) {
getline(in, line);
if (!in) break;
++lc;
if (lc % 1000 == 0) { cerr << '.'; flag = true; }
if (lc %50000 == 0) { cerr << " [" << lc << "]\n" << flush; flag = false; }
- vector<WordID> src, trg;
+ src.clear(); trg.clear();
CorpusTools::ReadLine(line, &src, &trg);
+ if (reverse) swap(src, trg);
if (src.size() == 0 || trg.size() == 0) {
cerr << "Error: " << lc << "\n" << line << endl;
return 1;
@@ -163,7 +167,10 @@ int main(int argc, char** argv) {
if (write_alignments) {
if (max_index > 0) {
if (first_al) first_al = false; else cout << ' ';
- cout << (max_index - 1) << "-" << j;
+ if (reverse)
+ cout << j << '-' << (max_index - 1);
+ else
+ cout << (max_index - 1) << '-' << j;
}
}
s2t_viterbi[max_i][f_j] = 1.0;
diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars
index 1a069abf..08ff33e1 100644
--- a/word-aligner/makefiles/makefile.grammars
+++ b/word-aligner/makefiles/makefile.grammars
@@ -16,7 +16,7 @@ STEM_E = $(SCRIPT_DIR)/stemmers/$(E_LANG).pl
CLASSIFY = $(SUPPORT_DIR)/classify.pl
MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl
-MODEL1 = $(TRAINING_DIR)/model1
+MODEL1 = $(TRAINING_DIR)/fast_align
MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl
e.voc: corpus.e
@@ -66,16 +66,16 @@ corpus.e-f: corpus.f corpus.e
$(MERGE_CORPUS) corpus.e corpus.f > $@
corpus.f-e.model1: corpus.f-e
- $(MODEL1) -v corpus.f-e > $@
+ $(MODEL1) -p -v -i corpus.f-e > $@
corpus.e-f.model1: corpus.e-f
- $(MODEL1) -v -V corpus.e-f > $@
+ $(MODEL1) -p -v -V -i corpus.e-f > $@
corpus.f-e.full-model1: corpus.f-e
- $(MODEL1) -t -999999 -v -V corpus.f-e > $@
+ $(MODEL1) -p -t -999999 -v -V -i corpus.f-e > $@
corpus.e-f.full-model1: corpus.e-f
- $(MODEL1) -t -999999 -v -V corpus.e-f > $@
+ $(MODEL1) -p -t -999999 -v -V -i corpus.e-f > $@
corpus.f-e.lex-grammar.gz: corpus.f-e corpus.f-e.model1 corpus.e-f.model1
$(MAKE_LEX_GRAMMAR) corpus.f-e corpus.f-e.model1 corpus.e-f.model1 | $(GZIP) -9 > $@