few small fixes of alignment tools, add new orthographic similarity feature for word aligner, final naming of directories, libraries in cdec

1 files changed, 37 insertions, 0 deletions

diff --git a/decoder/hg_io.h b/decoder/hg_io.h
new file mode 100644
index 00000000..69a516c1
--- /dev/null
+++ b/decoder/hg_io.h
@@ -0,0 +1,37 @@
+#ifndef _HG_IO_H_
+#define _HG_IO_H_
+
+#include <iostream>
+
+#include "lattice.h"
+class Hypergraph;
+
+struct HypergraphIO {
+
+  // the format is basically a list of nodes and edges in topological order
+  // any edge you read, you must have already read its tail nodes
+  // any node you read, you must have already read its incoming edges
+  // this may make writing a bit more challenging if your forest is not
+  // topologically sorted (but that probably doesn't happen very often),
+  // but it makes reading much more memory efficient.
+  // see test_data/small.json.gz for an email encoding
+  static bool ReadFromJSON(std::istream* in, Hypergraph* out);
+
+  // if remove_rules is used, the hypergraph is serialized without rule information
+  // (so it only contains structure and feature information)
+  static bool WriteToJSON(const Hypergraph& hg, bool remove_rules, std::ostream* out);
+
+  // serialization utils
+  static void ReadFromPLF(const std::string& in, Hypergraph* out, int line = 0);
+  // return PLF string representation (undefined behavior on non-lattices)
+  static std::string AsPLF(const Hypergraph& hg, bool include_global_parentheses = true);
+  static void PLFtoLattice(const std::string& plf, Lattice* pl);
+  static std::string Escape(const std::string& s);  // PLF helper
+};
+
+namespace B64 {
+  bool b64decode(const unsigned char* data, const size_t insize, char* out, const size_t outsize);
+  void b64encode(const char* data, const size_t size, std::ostream* out);
+}
+
+#endif