new doc scorer

2 files changed, 91 insertions, 0 deletions

diff --git a/mteval/ns_docscorer.cc b/mteval/ns_docscorer.cc
new file mode 100644
index 00000000..28a2fd09
--- /dev/null
+++ b/mteval/ns_docscorer.cc
@@ -0,0 +1,60 @@
+#include "ns_docscorer.h"
+
+#include <iostream>
+#include <cstring>
+
+#include "tdict.h"
+#include "filelib.h"
+#include "ns.h"
+
+using namespace std;
+
+DocumentScorer::~DocumentScorer() {}
+
+void DocumentScorer::Init(const EvaluationMetric* metric,
+            const vector<string>& ref_files,
+            const string& src_file,
+            bool verbose) {
+  scorers_.clear();
+  cerr << "Loading references (" << ref_files.size() << " files)\n";
+  assert(src_file.empty());
+  std::vector<ReadFile> ifs(ref_files.begin(),ref_files.end());
+  for (int i=0; i < ref_files.size(); ++i) ifs[i].Init(ref_files[i]);
+  char buf[64000];
+  bool expect_eof = false;
+  int line=0;
+  while (ifs[0].get()) {
+    vector<vector<WordID> > refs(ref_files.size());
+    for (int i=0; i < ref_files.size(); ++i) {
+      istream &in=ifs[i].get();
+      if (in.eof()) break;
+      in.getline(buf, 64000);
+      refs[i].clear();
+      if (strlen(buf) == 0) {
+        if (in.eof()) {
+          if (!expect_eof) {
+            assert(i == 0);
+            expect_eof = true;
+          }
+          break;
+        }
+      } else {
+        TD::ConvertSentence(buf, &refs[i]);
+        assert(!refs[i].empty());
+      }
+      assert(!expect_eof);
+    }
+    if (!expect_eof) {
+      string src_line;
+      //if (srcrf) {
+      //  getline(srcrf.get(), src_line);
+      //  map<string,string> dummy;
+      //  ProcessAndStripSGML(&src_line, &dummy);
+      //}
+      scorers_.push_back(metric->CreateSegmentEvaluator(refs));
+      ++line;
+    }
+  }
+  cerr << "Loaded reference translations for " << scorers_.size() << " sentences.\n";
+}
+
diff --git a/mteval/ns_docscorer.h b/mteval/ns_docscorer.h
new file mode 100644
index 00000000..170ac627
--- /dev/null
+++ b/mteval/ns_docscorer.h
@@ -0,0 +1,31 @@
+#ifndef _NS_DOC_SCORER_H_
+#define _NS_DOC_SCORER_H_
+
+#include <vector>
+#include <string>
+#include <boost/shared_ptr.hpp>
+
+struct EvaluationMetric;
+struct SegmentEvaluator;
+class DocumentScorer {
+ public:
+  ~DocumentScorer();
+  DocumentScorer() {  }
+  DocumentScorer(const EvaluationMetric* metric,
+                 const std::vector<std::string>& ref_files,
+                 const std::string& src_file = "",
+                 bool verbose=false) {
+    Init(metric,ref_files,src_file,verbose);
+  }
+  void Init(const EvaluationMetric* metric,
+            const std::vector<std::string>& ref_files,
+            const std::string& src_file = "",
+            bool verbose=false);
+
+  int size() const { return scorers_.size(); }
+  const SegmentEvaluator* operator[](size_t i) const { return scorers_[i].get(); }
+ private:
+  std::vector<boost::shared_ptr<SegmentEvaluator> > scorers_;
+};
+
+#endif