summaryrefslogtreecommitdiff
path: root/decoder
diff options
context:
space:
mode:
Diffstat (limited to 'decoder')
-rw-r--r--decoder/apply_models.cc21
-rw-r--r--decoder/bottom_up_parser.cc11
-rw-r--r--decoder/decoder.cc52
-rw-r--r--decoder/decoder.h2
-rw-r--r--decoder/hg_intersect.cc3
-rw-r--r--decoder/inside_outside.h4
-rw-r--r--decoder/scfg_translator.cc9
-rw-r--r--decoder/sentence_metadata.h2
8 files changed, 60 insertions, 44 deletions
diff --git a/decoder/apply_models.cc b/decoder/apply_models.cc
index 60b6f498..aec31a4f 100644
--- a/decoder/apply_models.cc
+++ b/decoder/apply_models.cc
@@ -13,6 +13,7 @@
#include <boost/functional/hash.hpp>
+#include "verbose.h"
#include "hg.h"
#include "ff.h"
@@ -170,7 +171,7 @@ public:
out(*o),
D(in.nodes_.size()),
pop_limit_(pop_limit) {
- cerr << " Applying feature functions (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
+ if (!SILENT) cerr << " Applying feature functions (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
node_states_.reserve(kRESERVE_NUM_NODES);
}
@@ -181,14 +182,16 @@ public:
int every = 1;
if (num_nodes > 100) every = 10;
assert(in.nodes_[pregoal].out_edges_.size() == 1);
- cerr << " ";
+ if (!SILENT) cerr << " ";
for (int i = 0; i < in.nodes_.size(); ++i) {
- if (i % every == 0) cerr << '.';
+ if (!SILENT && i % every == 0) cerr << '.';
KBest(i, i == goal_id);
}
- cerr << endl;
- cerr << " Best path: " << log(D[goal_id].front()->vit_prob_)
- << "\t" << log(D[goal_id].front()->est_prob_) << endl;
+ if (!SILENT) {
+ cerr << endl;
+ cerr << " Best path: " << log(D[goal_id].front()->vit_prob_)
+ << "\t" << log(D[goal_id].front()->est_prob_) << endl;
+ }
out.PruneUnreachable(D[goal_id].front()->node_index_);
FreeAll();
}
@@ -379,12 +382,12 @@ struct NoPruningRescorer {
int every = 1;
if (num_nodes > 100) every = 10;
assert(in.nodes_[pregoal].out_edges_.size() == 1);
- cerr << " ";
+ if (!SILENT) cerr << " ";
for (int i = 0; i < in.nodes_.size(); ++i) {
- if (i % every == 0) cerr << '.';
+ if (!SILENT && i % every == 0) cerr << '.';
ProcessOneNode(i, i == goal_id);
}
- cerr << endl;
+ if (!SILENT) cerr << endl;
}
private:
diff --git a/decoder/bottom_up_parser.cc b/decoder/bottom_up_parser.cc
index 88514b82..2d945222 100644
--- a/decoder/bottom_up_parser.cc
+++ b/decoder/bottom_up_parser.cc
@@ -10,6 +10,7 @@
#include "hg.h"
#include "array2d.h"
#include "tdict.h"
+#include "verbose.h"
using namespace std;
@@ -17,7 +18,7 @@ struct ParserStats {
ParserStats() : active_items(), passive_items() {}
void Reset() { active_items=0; passive_items=0; }
void Report() {
- cerr << " ACTIVE ITEMS: " << active_items << "\tPASSIVE ITEMS: " << passive_items << endl;
+ if (!SILENT) cerr << " ACTIVE ITEMS: " << active_items << "\tPASSIVE ITEMS: " << passive_items << endl;
}
int active_items;
int passive_items;
@@ -172,7 +173,7 @@ PassiveChart::PassiveChart(const string& goal,
for (int i = 0; i < grammars_.size(); ++i)
act_chart_[i] = new ActiveChart(forest, *this);
if (!kGOAL) kGOAL = TD::Convert("Goal") * -1;
- cerr << " Goal category: [" << goal << ']' << endl;
+ if (!SILENT) cerr << " Goal category: [" << goal << ']' << endl;
}
void PassiveChart::ApplyRule(const int i,
@@ -241,9 +242,9 @@ bool PassiveChart::Parse() {
for (int gi = 0; gi < grammars_.size(); ++gi)
act_chart_[gi]->SeedActiveChart(*grammars_[gi]);
- cerr << " ";
+ if (!SILENT) cerr << " ";
for (int l=1; l<input_.size()+1; ++l) {
- cerr << '.';
+ if (!SILENT) cerr << '.';
for (int i=0; i<input_.size() + 1 - l; ++i) {
int j = i + l;
for (int gi = 0; gi < grammars_.size(); ++gi) {
@@ -278,7 +279,7 @@ bool PassiveChart::Parse() {
}
}
}
- cerr << endl;
+ if (!SILENT) cerr << endl;
if (GoalFound())
forest_->PruneUnreachable(forest_->nodes_.size() - 1);
diff --git a/decoder/decoder.cc b/decoder/decoder.cc
index 2c53ff62..e0967e40 100644
--- a/decoder/decoder.cc
+++ b/decoder/decoder.cc
@@ -10,6 +10,7 @@
#include "filelib.h"
#include "fdict.h"
#include "timing_stats.h"
+#include "verbose.h"
#include "translator.h"
#include "phrasebased_translator.h"
@@ -51,6 +52,8 @@ static bool verbose_feature_functions=true;
namespace Hack { void MaxTrans(const Hypergraph& in, int beam_size); }
namespace NgramCache { void Clear(); }
+DecoderObserver::~DecoderObserver() {}
+void DecoderObserver::NotifyDecodingStart(const SentenceMetadata& smeta) {}
void DecoderObserver::NotifySourceParseFailure(const SentenceMetadata&) {}
void DecoderObserver::NotifyTranslationForest(const SentenceMetadata&, Hypergraph*) {}
void DecoderObserver::NotifyAlignmentFailure(const SentenceMetadata&) {}
@@ -639,15 +642,17 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
if (sgml.find("id") != sgml.end())
sent_id = atoi(sgml["id"].c_str());
- cerr << "\nINPUT: ";
- if (buf.size() < 100)
- cerr << buf << endl;
- else {
- size_t x = buf.rfind(" ", 100);
- if (x == string::npos) x = 100;
- cerr << buf.substr(0, x) << " ..." << endl;
+ if (!SILENT) {
+ cerr << "\nINPUT: ";
+ if (buf.size() < 100)
+ cerr << buf << endl;
+ else {
+ size_t x = buf.rfind(" ", 100);
+ if (x == string::npos) x = 100;
+ cerr << buf.substr(0, x) << " ..." << endl;
+ }
+ cerr << " id = " << sent_id << endl;
}
- cerr << " id = " << sent_id << endl;
string to_translate;
Lattice ref;
ParseTranslatorInputLattice(buf, &to_translate, &ref);
@@ -655,6 +660,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
//FIXME: should get the avg. or max source length of the input lattice (like Lattice::dist_(start,end)); but this is only used to scale beam parameters (optionally) anyway so fidelity isn't important.
const bool has_ref = ref.size() > 0;
SentenceMetadata smeta(sent_id, ref);
+ o->NotifyDecodingStart(smeta);
Hypergraph forest; // -LM forest
translator->ProcessMarkupHints(sgml);
Timer t("Translation");
@@ -664,7 +670,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
translator->SentenceComplete();
if (!translation_successful) {
- cerr << " NO PARSE FOUND.\n";
+ if (!SILENT) cerr << " NO PARSE FOUND.\n";
o->NotifySourceParseFailure(smeta);
o->NotifyDecodingComplete(smeta);
return false;
@@ -672,7 +678,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
const bool show_tree_structure=conf.count("show_tree_structure");
const bool show_features=conf.count("show_features");
- forest_stats(forest," -LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+ if (!SILENT) forest_stats(forest," -LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
if (conf.count("show_expected_length")) {
const PRPair<double, double> res =
Inside<PRPair<double, double>,
@@ -714,7 +720,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
&lm_forest);
forest.swap(lm_forest);
forest.Reweight(feature_weights);
- forest_stats(forest," +LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+ if (!SILENT) forest_stats(forest," +LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
}
maybe_prune(forest,conf,"beam_prune","density_prune","+LM",srclen);
@@ -729,22 +735,22 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
Hypergraph fsa_forest;
assert(fsa_ffs.size()==1);
ApplyFsaBy cfg(str("apply_fsa_by",conf),pop_limit);
- cerr << "FSA rescoring with "<<cfg<<" "<<fsa_ffs[0]->describe()<<endl;
+ if (!SILENT) cerr << "FSA rescoring with "<<cfg<<" "<<fsa_ffs[0]->describe()<<endl;
ApplyFsaModels(hgcfg,smeta,*fsa_ffs[0],feature_weights,cfg,&fsa_forest);
forest.swap(fsa_forest);
forest.Reweight(feature_weights);
- forest_stats(forest," +FSA forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+ if (!SILENT) forest_stats(forest," +FSA forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
}
// Oracle Rescoring
if(get_oracle_forest) {
Oracle oc=oracle.ComputeOracle(smeta,&forest,FeatureVector(feature_weights),10,conf["forest_output"].as<std::string>());
- cerr << " +Oracle BLEU forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
- cerr << " +Oracle BLEU (paths): " << forest.NumberOfPaths() << endl;
+ if (!SILENT) cerr << " +Oracle BLEU forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+ if (!SILENT) cerr << " +Oracle BLEU (paths): " << forest.NumberOfPaths() << endl;
oc.hope.Print(cerr," +Oracle BLEU");
oc.fear.Print(cerr," -Oracle BLEU");
//Add 1-best translation (trans) to psuedo-doc vectors
- oracle.IncludeLastScore(&cerr);
+ if (!SILENT) oracle.IncludeLastScore(&cerr);
}
o->NotifyTranslationForest(smeta, &forest);
@@ -752,7 +758,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
if (conf.count("forest_output") && !has_ref) {
ForestWriter writer(str("forest_output",conf), sent_id);
if (FileExists(writer.fname_)) {
- cerr << " Unioning...\n";
+ if (!SILENT) cerr << " Unioning...\n";
Hypergraph new_hg;
{
ReadFile rf(writer.fname_);
@@ -812,15 +818,15 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
HypergraphIO::WriteAsCFG(forest);
if (has_ref) {
if (HG::Intersect(ref, &forest)) {
- cerr << " Constr. forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
- cerr << " Constr. forest (paths): " << forest.NumberOfPaths() << endl;
+ if (!SILENT) cerr << " Constr. forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+ if (!SILENT) cerr << " Constr. forest (paths): " << forest.NumberOfPaths() << endl;
if (crf_uniform_empirical) {
- cerr << " USING UNIFORM WEIGHTS\n";
+ if (!SILENT) cerr << " USING UNIFORM WEIGHTS\n";
for (int i = 0; i < forest.edges_.size(); ++i)
forest.edges_[i].edge_prob_=prob_t::One();
} else {
forest.Reweight(feature_weights);
- cerr << " Constr. VitTree: " << ViterbiFTree(forest) << endl;
+ if (!SILENT) cerr << " Constr. VitTree: " << ViterbiFTree(forest) << endl;
}
if (conf.count("show_partition")) {
const prob_t z = Inside<prob_t, EdgeProb>(forest);
@@ -830,7 +836,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
if (conf.count("forest_output")) {
ForestWriter writer(str("forest_output",conf), sent_id);
if (FileExists(writer.fname_)) {
- cerr << " Unioning...\n";
+ if (!SILENT) cerr << " Unioning...\n";
Hypergraph new_hg;
{
ReadFile rf(writer.fname_);
@@ -893,7 +899,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
if (conf.count("graphviz")) forest.PrintGraphviz();
} else {
o->NotifyAlignmentFailure(smeta);
- cerr << " REFERENCE UNREACHABLE.\n";
+ if (!SILENT) cerr << " REFERENCE UNREACHABLE.\n";
if (write_gradient) {
cout << endl << flush;
}
diff --git a/decoder/decoder.h b/decoder/decoder.h
index 5dd6e1aa..2a1a43ce 100644
--- a/decoder/decoder.h
+++ b/decoder/decoder.h
@@ -12,6 +12,8 @@ struct Hypergraph;
struct DecoderImpl;
struct DecoderObserver {
+ virtual ~DecoderObserver();
+ virtual void NotifyDecodingStart(const SentenceMetadata& smeta);
virtual void NotifySourceParseFailure(const SentenceMetadata& smeta);
virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg);
virtual void NotifyAlignmentFailure(const SentenceMetadata& semta);
diff --git a/decoder/hg_intersect.cc b/decoder/hg_intersect.cc
index a3a2cbd9..81350cb0 100644
--- a/decoder/hg_intersect.cc
+++ b/decoder/hg_intersect.cc
@@ -5,6 +5,7 @@
#include "fast_lexical_cast.hpp"
#include <boost/functional/hash.hpp>
+#include "verbose.h"
#include "tdict.h"
#include "hg.h"
#include "trule.h"
@@ -50,7 +51,7 @@ struct RuleFilter {
};
static bool FastLinearIntersect(const Lattice& target, Hypergraph* hg) {
- cerr << " Fast linear-chain intersection...\n";
+ if (!SILENT) cerr << " Fast linear-chain intersection...\n";
vector<bool> prune(hg->edges_.size(), false);
set<int> cov;
map<const TRule*, TRulePtr> inverted_rules;
diff --git a/decoder/inside_outside.h b/decoder/inside_outside.h
index e6289176..dc96f1a9 100644
--- a/decoder/inside_outside.h
+++ b/decoder/inside_outside.h
@@ -1,5 +1,5 @@
-#ifndef _INSIDE_H_
-#define _INSIDE_H_
+#ifndef _INSIDE_OUTSIDE_H_
+#define _INSIDE_OUTSIDE_H_
#include <vector>
#include <algorithm>
diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc
index 08276c71..4e6cc226 100644
--- a/decoder/scfg_translator.cc
+++ b/decoder/scfg_translator.cc
@@ -14,6 +14,7 @@
#include "sentence_metadata.h"
#include "tdict.h"
#include "viterbi.h"
+#include "verbose.h"
#define foreach BOOST_FOREACH
#define reverse_foreach BOOST_REVERSE_FOREACH
@@ -100,7 +101,7 @@ struct SCFGTranslatorImpl {
LatticeTools::ConvertTextOrPLF(input, &lattice);
smeta->SetSourceLength(lattice.size());
if (add_pass_through_rules){
- cerr << "Adding pass through grammar" << endl;
+ if (!SILENT) cerr << "Adding pass through grammar" << endl;
PassThroughGrammar* g = new PassThroughGrammar(lattice, default_nt, ctf_iterations_);
g->SetGrammarName("PassThrough");
glist.push_back(GrammarPtr(g));
@@ -109,13 +110,13 @@ struct SCFGTranslatorImpl {
if(printGrammarsUsed)
cerr << "Using grammar::" << glist[gi]->GetGrammarName() << endl;
}
- cerr << "First pass parse... " << endl;
+ if (!SILENT) cerr << "First pass parse... " << endl;
ExhaustiveBottomUpParser parser(goal, glist);
if (!parser.Parse(lattice, forest)){
- cerr << "parse failed." << endl;
+ if (!SILENT) cerr << "parse failed." << endl;
return false;
} else {
- cerr << "parse succeeded." << endl;
+ if (!SILENT) cerr << "parse succeeded." << endl;
}
forest->Reweight(weights);
if (use_ctf_) {
diff --git a/decoder/sentence_metadata.h b/decoder/sentence_metadata.h
index 593019c8..c9a78dd3 100644
--- a/decoder/sentence_metadata.h
+++ b/decoder/sentence_metadata.h
@@ -13,6 +13,8 @@ struct SentenceMetadata {
trg_len_(ref.size()),
ref_(has_reference_ ? &ref : NULL) {}
+ int GetSentenceId() const { return sent_id_; }
+
// this should be called by the Translator object after
// it has parsed the source
void SetSourceLength(int sl) { src_len_ = sl; }
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-11 04:27:33 +0000