Merge with upstream

1 files changed, 11 insertions, 346 deletions

diff --git a/decoder/hg_io.cc b/decoder/hg_io.cc
index eb0be3d4..626b2954 100644
--- a/decoder/hg_io.cc
+++ b/decoder/hg_io.cc
@@ -6,362 +6,27 @@
 #include <sstream>
 #include <iostream>
 
+#include <boost/archive/binary_iarchive.hpp>
+#include <boost/archive/binary_oarchive.hpp>
+#include <boost/serialization/shared_ptr.hpp>
+
 #include "fast_lexical_cast.hpp"
 
 #include "tdict.h"
-#include "json_parse.h"
 #include "hg.h"
 
 using namespace std;
 
-struct HGReader : public JSONParser {
-  HGReader(Hypergraph* g) : rp("[X] ||| "), state(-1), hg(*g), nodes_needed(true), edges_needed(true) { nodes = 0; edges = 0; }
-
-  void CreateNode(const string& cat, const string& shash, const vector<int>& in_edges) {
-    WordID c = TD::Convert("X") * -1;
-    if (!cat.empty()) c = TD::Convert(cat) * -1;
-    Hypergraph::Node* node = hg.AddNode(c);
-    char* dend;
-    if (shash.size())
-      node->node_hash = strtoull(shash.c_str(), &dend, 16);
-    else
-      node->node_hash = 0;
-    for (int i = 0; i < in_edges.size(); ++i) {
-      if (in_edges[i] >= hg.edges_.size()) {
-        cerr << "JSONParser: in_edges[" << i << "]=" << in_edges[i]
-             << ", but hg only has " << hg.edges_.size() << " edges!\n";
-        abort();
-      }
-      hg.ConnectEdgeToHeadNode(&hg.edges_[in_edges[i]], node);
-    }
-  }
-  void CreateEdge(const TRulePtr& rule, SparseVector<double>* feats, const SmallVectorUnsigned& tail) {
-    Hypergraph::Edge* edge = hg.AddEdge(rule, tail);
-    feats->swap(edge->feature_values_);
-    edge->i_ = spans[0];
-    edge->j_ = spans[1];
-    edge->prev_i_ = spans[2];
-    edge->prev_j_ = spans[3];
-  }
-
-  bool HandleJSONEvent(int type, const JSON_value* value) {
-    switch(state) {
-    case -1:
-      assert(type == JSON_T_OBJECT_BEGIN);
-      state = 0;
-      break;
-    case 0:
-      if (type == JSON_T_OBJECT_END) {
-        //cerr << "HG created\n";  // TODO, signal some kind of callback
-      } else if (type == JSON_T_KEY) {
-        string val = value->vu.str.value;
-        if (val == "features") { assert(fdict.empty()); state = 1; }
-        else if (val == "is_sorted") { state = 3; }
-        else if (val == "rules") { assert(rules.empty()); state = 4; }
-        else if (val == "node") { state = 8; }
-        else if (val == "edges") { state = 13; }
-        else { cerr << "Unexpected key: " << val << endl; return false; }
-      }
-      break;
-
-    // features
-    case 1:
-      if(type == JSON_T_NULL) { state = 0; break; }
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 2;
-      break;
-    case 2:
-      if(type == JSON_T_ARRAY_END) { state = 0; break; }
-      assert(type == JSON_T_STRING);
-      fdict.push_back(FD::Convert(value->vu.str.value));
-      assert(fdict.back() > 0);
-      break;
-
-    // is_sorted
-    case 3:
-      assert(type == JSON_T_TRUE || type == JSON_T_FALSE);
-      is_sorted = (type == JSON_T_TRUE);
-      if (!is_sorted) { cerr << "[WARNING] is_sorted flag is ignored\n"; }
-      state = 0;
-      break;
-
-    // rules
-    case 4:
-      if(type == JSON_T_NULL) { state = 0; break; }
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 5;
-      break;
-    case 5:
-      if(type == JSON_T_ARRAY_END) { state = 0; break; }
-      assert(type == JSON_T_INTEGER);
-      state = 6;
-      rule_id = value->vu.integer_value;
-      break;
-    case 6:
-      assert(type == JSON_T_STRING);
-      rules[rule_id] = TRulePtr(new TRule(value->vu.str.value));
-      state = 5;
-      break;
-
-    // Nodes
-    case 8:
-      assert(type == JSON_T_OBJECT_BEGIN);
-      ++nodes;
-      in_edges.clear();
-      cat.clear();
-      shash.clear();
-      state = 9; break;
-    case 9:
-      if (type == JSON_T_OBJECT_END) {
-        //cerr << "Creating NODE\n";
-        CreateNode(cat, shash, in_edges);
-        state = 0; break;
-      }
-      assert(type == JSON_T_KEY);
-      cur_key = value->vu.str.value;
-      if (cur_key == "cat") { assert(cat.empty()); state = 10; break; }
-      if (cur_key == "in_edges") { assert(in_edges.empty()); state = 11; break; }
-      if (cur_key == "node_hash") { assert(shash.empty()); state = 24; break; }
-      cerr << "Syntax error: unexpected key " << cur_key << " in node specification.\n";
-      return false;
-    case 10:
-      assert(type == JSON_T_STRING || type == JSON_T_NULL);
-      cat = value->vu.str.value;
-      state = 9; break;
-    case 11:
-      if (type == JSON_T_NULL) { state = 9; break; }
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 12; break;
-    case 12:
-      if (type == JSON_T_ARRAY_END) { state = 9; break; }
-      assert(type == JSON_T_INTEGER);
-      //cerr << "in_edges: " << value->vu.integer_value << endl;
-      in_edges.push_back(value->vu.integer_value);
-      break;
-
-    //   "edges": [ { "tail": null, "feats" : [0,1.63,1,-0.54], "rule": 12},
-    //         { "tail": null, "feats" : [0,0.87,1,0.02], "spans":[1,2,3,4], "rule": 17},
-    //         { "tail": [0], "feats" : [1,2.3,2,15.3,"ExtraFeature",1.2], "rule": 13}]
-    case 13:
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 14;
-      break;
-    case 14:
-      if (type == JSON_T_ARRAY_END) { state = 0; break; }
-      assert(type == JSON_T_OBJECT_BEGIN);
-      //cerr << "New edge\n";
-      ++edges;
-      cur_rule.reset(); feats.clear(); tail.clear();
-      state = 15; break;
-    case 15:
-      if (type == JSON_T_OBJECT_END) {
-        CreateEdge(cur_rule, &feats, tail);
-        state = 14; break;
-      }
-      assert(type == JSON_T_KEY);
-      cur_key = value->vu.str.value;
-      //cerr << "edge key " << cur_key << endl;
-      if (cur_key == "rule") { assert(!cur_rule); state = 16; break; }
-      if (cur_key == "spans") { assert(!cur_rule); state = 22; break; }
-      if (cur_key == "feats") { assert(feats.empty()); state = 17; break; }
-      if (cur_key == "tail") { assert(tail.empty()); state = 20; break; }
-      cerr << "Unexpected key " << cur_key << " in edge specification\n";
-      return false;
-    case 16:    // edge.rule
-      if (type == JSON_T_INTEGER) {
-        int rule_id = value->vu.integer_value;
-        if (rules.find(rule_id) == rules.end()) {
-          // rules list must come before the edge definitions!
-          cerr << "Rule_id " << rule_id << " given but only loaded " << rules.size() << " rules\n";
-          return false;
-        }
-        cur_rule = rules[rule_id];
-      } else if (type == JSON_T_STRING) {
-        cur_rule.reset(new TRule(value->vu.str.value));
-      } else {
-        cerr << "Rule must be either a rule id or a rule string" << endl;
-        return false;
-      }
-      // cerr << "Edge: rule=" << cur_rule->AsString() << endl;
-      state = 15;
-      break;
-    case 17:      // edge.feats
-      if (type == JSON_T_NULL) { state = 15; break; }
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 18; break;
-    case 18:
-      if (type == JSON_T_ARRAY_END) { state = 15; break; }
-      if (type != JSON_T_INTEGER && type != JSON_T_STRING) {
-        cerr << "Unexpected feature id type\n"; return false;
-      }
-      if (type == JSON_T_INTEGER) {
-        fid = value->vu.integer_value;
-        assert(fid < fdict.size());
-        fid = fdict[fid];
-      } else if (JSON_T_STRING) {
-        fid = FD::Convert(value->vu.str.value);
-      } else { abort(); }
-      state = 19;
-      break;
-    case 19:
-      {
-        assert(type == JSON_T_INTEGER || type == JSON_T_FLOAT);
-        double val = (type == JSON_T_INTEGER ? static_cast<double>(value->vu.integer_value) :
-	                                       strtod(value->vu.str.value, NULL));
-        feats.set_value(fid, val);
-        state = 18;
-        break;
-      }
-    case 20:     // edge.tail
-      if (type == JSON_T_NULL) { state = 15; break; }
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 21; break;
-    case 21:
-      if (type == JSON_T_ARRAY_END) { state = 15; break; }
-      assert(type == JSON_T_INTEGER);
-      tail.push_back(value->vu.integer_value);
-      break;
-    case 22:     // edge.spans
-      assert(type == JSON_T_ARRAY_BEGIN);
-      state = 23;
-      spans[0] = spans[1] = spans[2] = spans[3] = -1;
-      spanc = 0;
-      break;
-    case 23:
-      if (type == JSON_T_ARRAY_END) { state = 15; break; }
-      assert(type == JSON_T_INTEGER);
-      assert(spanc < 4);
-      spans[spanc] = value->vu.integer_value;
-      ++spanc;
-      break;
-    case 24:  // read node hash
-      assert(type == JSON_T_STRING);
-      shash = value->vu.str.value;
-      state = 9;
-      break;
-    }
-    return true;
-  }
-  string rp;
-  string cat;
-  SmallVectorUnsigned tail;
-  vector<int> in_edges;
-  string shash;
-  TRulePtr cur_rule;
-  map<int, TRulePtr> rules;
-  vector<int> fdict;
-  SparseVector<double> feats;
-  int state;
-  int fid;
-  int nodes;
-  int edges;
-  int spans[4];
-  int spanc;
-  string cur_key;
-  Hypergraph& hg;
-  int rule_id;
-  bool nodes_needed;
-  bool edges_needed;
-  bool is_sorted;
-};
-
-bool HypergraphIO::ReadFromJSON(istream* in, Hypergraph* hg) {
+bool HypergraphIO::ReadFromBinary(istream* in, Hypergraph* hg) {
+  boost::archive::binary_iarchive oa(*in);
   hg->clear();
-  HGReader reader(hg);
-  return reader.Parse(in);
-}
-
-static void WriteRule(const TRule& r, ostream* out) {
-  if (!r.lhs_) { (*out) << "[X] ||| "; }
-  JSONParser::WriteEscapedString(r.AsString(), out);
+  oa >> *hg;
+  return true;
 }
 
-bool HypergraphIO::WriteToJSON(const Hypergraph& hg, bool remove_rules, ostream* out) {
-  if (hg.empty()) { *out << "{}\n"; return true; }
-  map<const TRule*, int> rid;
-  ostream& o = *out;
-  rid[NULL] = 0;
-  o << '{';
-  if (!remove_rules) {
-    o << "\"rules\":[";
-    for (int i = 0; i < hg.edges_.size(); ++i) {
-      const TRule* r = hg.edges_[i].rule_.get();
-      int &id = rid[r];
-      if (!id) {
-        id=rid.size() - 1;
-        if (id > 1) o << ',';
-        o << id << ',';
-        WriteRule(*r, &o);
-      };
-    }
-    o << "],";
-  }
-  const bool use_fdict = FD::NumFeats() < 1000;
-  if (use_fdict) {
-    o << "\"features\":[";
-    for (int i = 1; i < FD::NumFeats(); ++i) {
-      o << (i==1 ? "":",");
-      JSONParser::WriteEscapedString(FD::Convert(i), &o);
-    }
-    o << "],";
-  }
-  vector<int> edgemap(hg.edges_.size(), -1);  // edges may be in non-topo order
-  int edge_count = 0;
-  for (int i = 0; i < hg.nodes_.size(); ++i) {
-    const Hypergraph::Node& node = hg.nodes_[i];
-    if (i > 0) { o << ","; }
-    o << "\"edges\":[";
-    for (int j = 0; j < node.in_edges_.size(); ++j) {
-      const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]];
-      edgemap[edge.id_] = edge_count;
-      ++edge_count;
-      o << (j == 0 ? "" : ",") << "{";
-
-      o << "\"tail\":[";
-      for (int k = 0; k < edge.tail_nodes_.size(); ++k) {
-        o << (k > 0 ? "," : "") << edge.tail_nodes_[k];
-      }
-      o << "],";
-
-      o << "\"spans\":[" << edge.i_ << "," << edge.j_ << "," << edge.prev_i_ << "," << edge.prev_j_ << "],";
-
-      o << "\"feats\":[";
-      bool first = true;
-      for (SparseVector<double>::const_iterator it = edge.feature_values_.begin(); it != edge.feature_values_.end(); ++it) {
-        if (!it->second) continue;   // don't write features that have a zero value
-        if (!it->first) continue;    // if the feature set was frozen this might happen
-        if (!first) o << ',';
-        if (use_fdict)
-          o << (it->first - 1);
-        else {
-	  JSONParser::WriteEscapedString(FD::Convert(it->first), &o);
-        }
-	o << ',' << it->second;
-        first = false;
-      }
-      o << "]";
-      if (!remove_rules) { o << ",\"rule\":" << rid[edge.rule_.get()]; }
-      o << "}";
-    }
-    o << "],";
-
-    o << "\"node\":{\"in_edges\":[";
-    for (int j = 0; j < node.in_edges_.size(); ++j) {
-      int mapped_edge = edgemap[node.in_edges_[j]];
-      assert(mapped_edge >= 0);
-      o << (j == 0 ? "" : ",") << mapped_edge;
-    }
-    o << "]";
-    if (node.cat_ < 0) {
-       o << ",\"cat\":";
-       JSONParser::WriteEscapedString(TD::Convert(node.cat_ * -1), &o);
-    }
-    char buf[48];
-    sprintf(buf, "%016lX", node.node_hash);
-    o << ",\"node_hash\":\"" << buf << "\"";
-    o << "}";
-  }
-  o << "}\n";
+bool HypergraphIO::WriteToBinary(const Hypergraph& hg, ostream* out) {
+  boost::archive::binary_oarchive oa(*out);
+  oa << hg;
   return true;
 }