summaryrefslogtreecommitdiff
path: root/mteval
diff options
context:
space:
mode:
authorPaul Baltescu <pauldb89@gmail.com>2013-11-23 17:33:47 +0000
committerPaul Baltescu <pauldb89@gmail.com>2013-11-23 17:33:47 +0000
commitcc6313b23cac25eb05976b6cf64f96faf1ed4163 (patch)
tree3dc28060ad25b43773e875bea7388ab1cefcd927 /mteval
parent7990c750829af93f0a1e0fc14534582f52ee9e8c (diff)
parentf2fb69b10a897e8beb4e6e6d6cbb4327096235ef (diff)
Merge branch 'master' of https://github.com/redpony/cdec
Diffstat (limited to 'mteval')
-rw-r--r--mteval/mbr_kbest.cc8
-rw-r--r--mteval/ns.cc2
-rw-r--r--mteval/ns_ter.cc8
-rw-r--r--mteval/scorer.cc25
-rw-r--r--mteval/ter.cc8
5 files changed, 37 insertions, 14 deletions
diff --git a/mteval/mbr_kbest.cc b/mteval/mbr_kbest.cc
index 2519bc01..76d2c7fc 100644
--- a/mteval/mbr_kbest.cc
+++ b/mteval/mbr_kbest.cc
@@ -1,9 +1,14 @@
#include <iostream>
#include <vector>
-#include <tr1/unordered_map>
#include <boost/program_options.hpp>
#include <boost/functional/hash.hpp>
+#ifndef HAVE_OLD_CPP
+# include <unordered_map>
+#else
+# include <tr1/unordered_map>
+namespace std { using std::tr1::unordered_map; }
+#endif
#include "prob.h"
#include "tdict.h"
@@ -12,7 +17,6 @@
#include "stringlib.h"
using namespace std;
-using namespace std::tr1;
namespace po = boost::program_options;
diff --git a/mteval/ns.cc b/mteval/ns.cc
index b64d4798..c1ea238b 100644
--- a/mteval/ns.cc
+++ b/mteval/ns.cc
@@ -106,7 +106,7 @@ struct BleuSegmentEvaluator : public SegmentEvaluator {
}
struct NGramCompare {
- int operator() (const vector<WordID>& a, const vector<WordID>& b) {
+ int operator() (const vector<WordID>& a, const vector<WordID>& b) const {
const size_t as = a.size();
const size_t bs = b.size();
const size_t s = (as < bs ? as : bs);
diff --git a/mteval/ns_ter.cc b/mteval/ns_ter.cc
index 0e1008db..680fb7b4 100644
--- a/mteval/ns_ter.cc
+++ b/mteval/ns_ter.cc
@@ -4,7 +4,12 @@
#include <cassert>
#include <iostream>
#include <limits>
-#include <tr1/unordered_map>
+#ifndef HAVE_OLD_CPP
+# include <unordered_map>
+#else
+# include <tr1/unordered_map>
+namespace std { using std::tr1::unordered_map; }
+#endif
#include <set>
#include <boost/functional/hash.hpp>
#include "tdict.h"
@@ -20,7 +25,6 @@ static const unsigned kREF_WORDCOUNT = 4;
static const unsigned kDUMMY_LAST_ENTRY = 5;
using namespace std;
-using namespace std::tr1;
bool TERMetric::IsErrorMetric() const {
return true;
diff --git a/mteval/scorer.cc b/mteval/scorer.cc
index de84e076..fedbca91 100644
--- a/mteval/scorer.cc
+++ b/mteval/scorer.cc
@@ -198,7 +198,7 @@ class BLEUScorerBase : public SentenceScorer {
virtual float ComputeRefLength(const vector<WordID>& hyp) const = 0;
private:
struct NGramCompare {
- int operator() (const vector<WordID>& a, const vector<WordID>& b) {
+ int operator() (const vector<WordID>& a, const vector<WordID>& b) const {
size_t as = a.size();
size_t bs = b.size();
const size_t s = (as < bs ? as : bs);
@@ -594,26 +594,29 @@ void DocScorer::Init(
const ScoreType type,
const vector<string>& ref_files,
const string& src_file, bool verbose) {
+ cerr << "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+ "!!! This code is using the deprecated DocScorer interface, please fix !!!\n"
+ "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n";
scorers_.clear();
- // TODO stop using valarray, start using ReadFile
+ static const WordID kDIV = TD::Convert("|||");
cerr << "Loading references (" << ref_files.size() << " files)\n";
ReadFile srcrf;
if (type == AER && src_file.size() > 0) {
cerr << " (source=" << src_file << ")\n";
srcrf.Init(src_file);
}
+ std::vector<WordID> tmp;
std::vector<ReadFile> ifs(ref_files.begin(),ref_files.end());
for (int i=0; i < ref_files.size(); ++i) ifs[i].Init(ref_files[i]);
char buf[64000];
bool expect_eof = false;
int line=0;
while (ifs[0].get()) {
- vector<vector<WordID> > refs(ref_files.size());
+ vector<vector<WordID> > refs;
for (int i=0; i < ref_files.size(); ++i) {
istream &in=ifs[i].get();
if (in.eof()) break;
in.getline(buf, 64000);
- refs[i].clear();
if (strlen(buf) == 0) {
if (in.eof()) {
if (!expect_eof) {
@@ -622,9 +625,17 @@ void DocScorer::Init(
}
break;
}
- } else {
- TD::ConvertSentence(buf, &refs[i]);
- assert(!refs[i].empty());
+ } else { // process reference
+ tmp.clear();
+ TD::ConvertSentence(buf, &tmp);
+ unsigned last = 0;
+ for (unsigned j = 0; j < tmp.size(); ++j) {
+ if (tmp[j] == kDIV) {
+ refs.push_back(vector<WordID>(tmp.begin() + last, tmp.begin() + j));
+ last = j + 1;
+ }
+ }
+ refs.push_back(vector<WordID>(tmp.begin() + last, tmp.end()));
}
assert(!expect_eof);
}
diff --git a/mteval/ter.cc b/mteval/ter.cc
index cacc5b00..19b90bbc 100644
--- a/mteval/ter.cc
+++ b/mteval/ter.cc
@@ -5,7 +5,12 @@
#include <iostream>
#include <limits>
#include <sstream>
-#include <tr1/unordered_map>
+#ifndef HAVE_OLD_CPP
+# include <unordered_map>
+#else
+# include <tr1/unordered_map>
+namespace std { using std::tr1::unordered_map; }
+#endif
#include <set>
#include <valarray>
#include <boost/functional/hash.hpp>
@@ -16,7 +21,6 @@ const bool ter_use_average_ref_len = true;
const int ter_short_circuit_long_sentences = -1;
using namespace std;
-using namespace std::tr1;
struct COSTS {
static const float substitution;