From 6ba464f6d78e38970d5467b10ce1114f4d7feaa4 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 7 Aug 2012 23:22:44 -0400 Subject: sample trees from hypergraphs --- decoder/hg_sampler.cc | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ decoder/hg_sampler.h | 7 +++++++ 2 files changed, 62 insertions(+) (limited to 'decoder') diff --git a/decoder/hg_sampler.cc b/decoder/hg_sampler.cc index cdf0ec3c..c4d3dede 100644 --- a/decoder/hg_sampler.cc +++ b/decoder/hg_sampler.cc @@ -71,3 +71,58 @@ void HypergraphSampler::sample_hypotheses(const Hypergraph& hg, Viterbi(hg, &hyp.words, ESentenceTraversal(), SampledDerivationWeightFunction(sampled_edges)); } } + +void HypergraphSampler::sample_trees(const Hypergraph& hg, + unsigned n, + MT19937* rng, + vector* trees) { + trees->clear(); + trees->resize(n); + + // compute inside probabilities + vector node_probs; + Inside(hg, &node_probs, EdgeProb()); + + vector sampled_edges(hg.edges_.size()); + queue q; + SampleSet ss; + for (unsigned i = 0; i < n; ++i) { + fill(sampled_edges.begin(), sampled_edges.end(), false); + // sample derivation top down + assert(q.empty()); + q.push(hg.nodes_.size() - 1); + prob_t model_score = prob_t::One(); + while(!q.empty()) { + unsigned cur_node_id = q.front(); + q.pop(); + const Hypergraph::Node& node = hg.nodes_[cur_node_id]; + const unsigned num_in_edges = node.in_edges_.size(); + unsigned sampled_edge_idx = 0; + if (num_in_edges == 1) { + sampled_edge_idx = node.in_edges_[0]; + } else { + assert(num_in_edges > 1); + ss.clear(); + for (unsigned j = 0; j < num_in_edges; ++j) { + const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; + prob_t p = edge.edge_prob_; // edge weight + for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) + p *= node_probs[edge.tail_nodes_[k]]; // tail node inside weight + ss.add(p); + } + sampled_edge_idx = node.in_edges_[rng->SelectSample(ss)]; + } + sampled_edges[sampled_edge_idx] = true; + const Hypergraph::Edge& sampled_edge = hg.edges_[sampled_edge_idx]; + model_score *= sampled_edge.edge_prob_; + //sampled_deriv->push_back(sampled_edge_idx); + for (unsigned j = 0; j < sampled_edge.tail_nodes_.size(); ++j) { + q.push(sampled_edge.tail_nodes_[j]); + } + } + vector tmp; + Viterbi(hg, &tmp, ETreeTraversal(), SampledDerivationWeightFunction(sampled_edges)); + (*trees)[n] = TD::GetString(tmp); + } +} + diff --git a/decoder/hg_sampler.h b/decoder/hg_sampler.h index bf4e1eb0..6ac39a20 100644 --- a/decoder/hg_sampler.h +++ b/decoder/hg_sampler.h @@ -3,6 +3,7 @@ #include +#include #include "sparse_vector.h" #include "sampler.h" #include "wordid.h" @@ -22,6 +23,12 @@ struct HypergraphSampler { unsigned n, // how many samples to draw MT19937* rng, std::vector* hypos); + + static void + sample_trees(const Hypergraph& hg, + unsigned n, + MT19937* rng, + std::vector* trees); }; #endif -- cgit v1.2.3 From 89df59e9ad1260b6ff36e4aa0e730ad84faec211 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 8 Aug 2012 00:40:08 -0400 Subject: random sampling trees fix --- decoder/hg_sampler.cc | 2 +- python/src/_cdec.cpp | 4643 +++++++++++++++++++++++-------------------------- 2 files changed, 2180 insertions(+), 2465 deletions(-) (limited to 'decoder') diff --git a/decoder/hg_sampler.cc b/decoder/hg_sampler.cc index c4d3dede..8e520871 100644 --- a/decoder/hg_sampler.cc +++ b/decoder/hg_sampler.cc @@ -122,7 +122,7 @@ void HypergraphSampler::sample_trees(const Hypergraph& hg, } vector tmp; Viterbi(hg, &tmp, ETreeTraversal(), SampledDerivationWeightFunction(sampled_edges)); - (*trees)[n] = TD::GetString(tmp); + (*trees)[i] = TD::GetString(tmp); } } diff --git a/python/src/_cdec.cpp b/python/src/_cdec.cpp index dcd33448..4b02edd5 100644 --- a/python/src/_cdec.cpp +++ b/python/src/_cdec.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.17.beta1 on Sun Jul 29 23:00:27 2012 */ +/* Generated by Cython 0.16 on Tue Aug 7 23:28:20 2012 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -47,6 +47,12 @@ #define CYTHON_COMPILING_IN_CPYTHON 1 #endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyCFunction_Call PyObject_Call +#else + #define __Pyx_PyCFunction_Call PyCFunction_Call +#endif + #if PY_VERSION_HEX < 0x02050000 typedef int Py_ssize_t; #define PY_SSIZE_T_MAX INT_MAX @@ -54,11 +60,8 @@ #define PY_FORMAT_SIZE_T "" #define PyInt_FromSsize_t(z) PyInt_FromLong(z) #define PyInt_AsSsize_t(o) __Pyx_PyInt_AsInt(o) - #define PyNumber_Index(o) ((PyNumber_Check(o) && !PyFloat_Check(o)) ? PyNumber_Int(o) : \ - (PyErr_Format(PyExc_TypeError, \ - "expected index value, got %.200s", Py_TYPE(o)->tp_name), \ - (PyObject*)0)) - #define PyIndex_Check(o) (PyNumber_Check(o) && !PyFloat_Check(o) && !PyComplex_Check(o)) + #define PyNumber_Index(o) PyNumber_Int(o) + #define PyIndex_Check(o) PyNumber_Check(o) #define PyErr_WarnEx(category, message, stacklevel) PyErr_Warn(category, message) #define __PYX_BUILD_PY_SSIZE_T "i" #else @@ -127,20 +130,14 @@ #endif -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_GET_LENGTH) #define CYTHON_PEP393_ENABLED 1 - #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ - 0 : _PyUnicode_Ready((PyObject *)(op))) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) - #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #else #define CYTHON_PEP393_ENABLED 0 - #define __Pyx_PyUnicode_READY(op) (0) - #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - - #define __Pyx_PyUnicode_READ(k, d, i) ((k=k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #endif #if PY_MAJOR_VERSION >= 3 @@ -273,7 +270,6 @@ #include #define __PYX_HAVE___cdec #define __PYX_HAVE_API___cdec -#include "string.h" #include #include #include @@ -361,11 +357,7 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*); -#if CYTHON_COMPILING_IN_CPYTHON #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) -#else -#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) -#endif #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) #ifdef __GNUC__ @@ -399,19 +391,18 @@ static const char *__pyx_f[] = { "hypergraph.pxi", "lattice.pxi", "mteval.pxi", - "stringsource", "cdec.sa._sa.pxd", }; /*--- Type declarations ---*/ struct __pyx_obj_5_cdec_Scorer; struct __pyx_obj_5_cdec_NTRef; -struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__; -struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr; struct __pyx_obj_4cdec_2sa_3_sa_Phrase; +struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__; struct __pyx_obj_5_cdec_Grammar; +struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines; +struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees; struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__; -struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot; struct __pyx_obj_5_cdec___pyx_scope_struct_2__phrase; struct __pyx_obj_5_cdec_CandidateSet; struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__; @@ -419,18 +410,21 @@ struct __pyx_obj_5_cdec_TRule; struct __pyx_obj_4cdec_2sa_3_sa_Rule; struct __pyx_obj_5_cdec_MRule; struct __pyx_obj_5_cdec_SegmentEvaluator; -struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__; +struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr; +struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot; struct __pyx_obj_5_cdec_Candidate; struct __pyx_obj_5_cdec___pyx_scope_struct_6_genexpr; struct __pyx_obj_5_cdec_NT; +struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__; struct __pyx_obj_5_cdec_HypergraphEdge; struct __pyx_obj_5_cdec___pyx_scope_struct_1___iter__; struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__; struct __pyx_obj_5_cdec___pyx_scope_struct_4___get__; +struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__; struct __pyx_obj_5_cdec_Decoder; struct __pyx_obj_5_cdec_HypergraphNode; struct __pyx_obj_5_cdec_SparseVector; -struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__; +struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__; struct __pyx_obj_5_cdec___pyx_scope_struct____iter__; struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__; struct __pyx_obj_5_cdec_DenseVector; @@ -446,10 +440,8 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_9_kbest_trees; struct __pyx_obj_5_cdec_Hypergraph; struct __pyx_obj_5_cdec_Lattice; struct __pyx_obj_5_cdec___pyx_scope_struct_11_sample; -struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__; -struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config; +struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config; struct __pyx_obj_5_cdec_TextGrammar; -struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines; struct __pyx_opt_args_5_cdec_as_str; /* "_cdec.pyx":6 @@ -464,7 +456,7 @@ struct __pyx_opt_args_5_cdec_as_str { char *error_msg; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":117 +/* "/home/cdyer/cdec/python/src/mteval.pxi":117 * return CandidateSet(self) * * cdef class Scorer: # <<<<<<<<<<<<<< @@ -478,7 +470,7 @@ struct __pyx_obj_5_cdec_Scorer { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":20 +/* "/home/cdyer/cdec/python/src/grammar.pxi":20 * return '[%s]' % self.cat * * cdef class NTRef: # <<<<<<<<<<<<<< @@ -491,37 +483,7 @@ struct __pyx_obj_5_cdec_NTRef { }; -/* "_cdec.pyx":43 - * cdef DenseVector weights - * - * def __cinit__(self, config_str=None, **config): # <<<<<<<<<<<<<< - * """ Configuration can be given as a string: - * Decoder('formalism = scfg') - */ -struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ { - PyObject_HEAD - PyObject *__pyx_v_config; -}; - - -/* "_cdec.pyx":54 - * 'csplit', 'tagger', 'lexalign'): - * raise InvalidConfig('formalism "%s" unknown' % formalism) - * config_str = '\n'.join('%s = %s' % kv for kv in _make_config(config)) # <<<<<<<<<<<<<< - * cdef istringstream* config_stream = new istringstream(config_str) - * self.dec = new decoder.Decoder(config_stream) - */ -struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr { - PyObject_HEAD - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *__pyx_outer_scope; - PyObject *__pyx_v_kv; - PyObject *__pyx_t_0; - Py_ssize_t __pyx_t_1; - PyObject *(*__pyx_t_2)(PyObject *); -}; - - -/* "/Users/vchahun/Sandbox/cdec/python/src/cdec.sa._sa.pxd":1 +/* "/home/cdyer/cdec/python/src/cdec.sa._sa.pxd":1 * cdef class Phrase: # <<<<<<<<<<<<<< * cdef int *syms * cdef int n, *varpos, n_vars @@ -536,7 +498,23 @@ struct __pyx_obj_4cdec_2sa_3_sa_Phrase { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":181 +/* "/home/cdyer/cdec/python/src/mteval.pxi":90 + * return candidate + * + * def __iter__(self): # <<<<<<<<<<<<<< + * cdef unsigned i + * for i in range(len(self)): + */ +struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ { + PyObject_HEAD + unsigned int __pyx_v_i; + struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self; + Py_ssize_t __pyx_t_0; + unsigned int __pyx_t_1; +}; + + +/* "/home/cdyer/cdec/python/src/grammar.pxi":181 * super(MRule, self).__init__(lhs, rhs, e, scores, a) * * cdef class Grammar: # <<<<<<<<<<<<<< @@ -549,36 +527,64 @@ struct __pyx_obj_5_cdec_Grammar { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":119 +/* "/home/cdyer/cdec/python/src/lattice.pxi":58 * - * property nodes: - * def __get__(self): # <<<<<<<<<<<<<< - * cdef unsigned i - * for i in range(self.hg.nodes_.size()): + * def todot(self): + * def lines(): # <<<<<<<<<<<<<< + * yield 'digraph lattice {' + * yield 'rankdir = LR;' */ -struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ { +struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines { PyObject_HEAD - unsigned int __pyx_v_i; + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *__pyx_outer_scope; + PyObject *__pyx_v_delta; + PyObject *__pyx_v_i; + PyObject *__pyx_v_label; + PyObject *__pyx_v_weight; + Py_ssize_t __pyx_t_0; + PyObject *__pyx_t_1; + PyObject *(*__pyx_t_2)(PyObject *); + PyObject *__pyx_t_3; + Py_ssize_t __pyx_t_4; + PyObject *(*__pyx_t_5)(PyObject *); +}; + + +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":88 + * del hypos + * + * def sample_trees(self, unsigned n): # <<<<<<<<<<<<<< + * cdef vector[string]* trees = new vector[string]() + * if self.rng == NULL: + */ +struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees { + PyObject_HEAD + unsigned int __pyx_v_k; + unsigned int __pyx_v_n; struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self; + std::vector *__pyx_v_trees; size_t __pyx_t_0; unsigned int __pyx_t_1; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":57 - * yield self[i] +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":125 * - * def todot(self): # <<<<<<<<<<<<<< - * def lines(): - * yield 'digraph lattice {' + * property edges: + * def __get__(self): # <<<<<<<<<<<<<< + * cdef unsigned i + * for i in range(self.hg.edges_.size()): */ -struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot { +struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ { PyObject_HEAD - struct __pyx_obj_5_cdec_Lattice *__pyx_v_self; + unsigned int __pyx_v_i; + struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self; + size_t __pyx_t_0; + unsigned int __pyx_t_1; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":5 +/* "/home/cdyer/cdec/python/src/grammar.pxi":5 * import cdec.sa._sa as _sa * * def _phrase(phrase): # <<<<<<<<<<<<<< @@ -591,7 +597,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_2__phrase { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":65 +/* "/home/cdyer/cdec/python/src/mteval.pxi":65 * return result * * cdef class CandidateSet: # <<<<<<<<<<<<<< @@ -606,23 +612,23 @@ struct __pyx_obj_5_cdec_CandidateSet { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":167 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":131 * - * property tail_nodes: + * property nodes: * def __get__(self): # <<<<<<<<<<<<<< * cdef unsigned i - * for i in range(self.edge.tail_nodes_.size()): + * for i in range(self.hg.nodes_.size()): */ struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ { PyObject_HEAD unsigned int __pyx_v_i; - struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self; - unsigned int __pyx_t_0; + struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self; + size_t __pyx_t_0; unsigned int __pyx_t_1; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":50 +/* "/home/cdyer/cdec/python/src/grammar.pxi":50 * return TRule(lhs, f, e, scores, a) * * cdef class TRule: # <<<<<<<<<<<<<< @@ -635,7 +641,7 @@ struct __pyx_obj_5_cdec_TRule { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/cdec.sa._sa.pxd":7 +/* "/home/cdyer/cdec/python/src/cdec.sa._sa.pxd":7 * cdef public int chunklen(self, int k) * * cdef class Rule: # <<<<<<<<<<<<<< @@ -653,7 +659,7 @@ struct __pyx_obj_4cdec_2sa_3_sa_Rule { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":169 +/* "/home/cdyer/cdec/python/src/grammar.pxi":169 * _phrase(self.f), _phrase(self.e), scores) * * cdef class MRule(TRule): # <<<<<<<<<<<<<< @@ -665,7 +671,7 @@ struct __pyx_obj_5_cdec_MRule { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":98 +/* "/home/cdyer/cdec/python/src/mteval.pxi":98 * self.cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get()) * * cdef class SegmentEvaluator: # <<<<<<<<<<<<<< @@ -679,24 +685,37 @@ struct __pyx_obj_5_cdec_SegmentEvaluator { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":44 - * return self.stats.size() - * - * def __iter__(self): # <<<<<<<<<<<<<< - * for i in range(len(self)): - * yield self[i] +/* "_cdec.pyx":54 + * 'csplit', 'tagger', 'lexalign'): + * raise InvalidConfig('formalism "%s" unknown' % formalism) + * config_str = '\n'.join('%s = %s' % kv for kv in _make_config(config)) # <<<<<<<<<<<<<< + * cdef istringstream* config_stream = new istringstream(config_str) + * self.dec = new decoder.Decoder(config_stream) */ -struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ { +struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr { PyObject_HEAD - PyObject *__pyx_v_i; - struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self; - Py_ssize_t __pyx_t_0; - PyObject *__pyx_t_1; + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *__pyx_outer_scope; + PyObject *__pyx_v_kv; + PyObject *__pyx_t_0; + Py_ssize_t __pyx_t_1; PyObject *(*__pyx_t_2)(PyObject *); }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":12 +/* "/home/cdyer/cdec/python/src/lattice.pxi":57 + * yield self[i] + * + * def todot(self): # <<<<<<<<<<<<<< + * def lines(): + * yield 'digraph lattice {' + */ +struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot { + PyObject_HEAD + struct __pyx_obj_5_cdec_Lattice *__pyx_v_self; +}; + + +/* "/home/cdyer/cdec/python/src/mteval.pxi":12 * return stats * * cdef class Candidate: # <<<<<<<<<<<<<< @@ -710,7 +729,7 @@ struct __pyx_obj_5_cdec_Candidate { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":165 +/* "/home/cdyer/cdec/python/src/grammar.pxi":165 * * def __str__(self): * scores = ' '.join('%s=%s' % feat for feat in self.scores) # <<<<<<<<<<<<<< @@ -727,7 +746,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_6_genexpr { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":8 +/* "/home/cdyer/cdec/python/src/grammar.pxi":8 * return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase) * * cdef class NT: # <<<<<<<<<<<<<< @@ -741,7 +760,20 @@ struct __pyx_obj_5_cdec_NT { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":147 +/* "_cdec.pyx":43 + * cdef DenseVector weights + * + * def __cinit__(self, config_str=None, **config): # <<<<<<<<<<<<<< + * """ Configuration can be given as a string: + * Decoder('formalism = scfg') + */ +struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ { + PyObject_HEAD + PyObject *__pyx_v_config; +}; + + +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":159 * return vector * * cdef class HypergraphEdge: # <<<<<<<<<<<<<< @@ -757,7 +789,7 @@ struct __pyx_obj_5_cdec_HypergraphEdge { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":67 +/* "/home/cdyer/cdec/python/src/vectors.pxi":67 * self.vector.set_value(fid, value) * * def __iter__(self): # <<<<<<<<<<<<<< @@ -774,12 +806,12 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_1___iter__ { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":209 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":215 * - * property out_edges: + * property in_edges: * def __get__(self): # <<<<<<<<<<<<<< * cdef unsigned i - * for i in range(self.node.out_edges_.size()): + * for i in range(self.node.in_edges_.size()): */ struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ { PyObject_HEAD @@ -790,7 +822,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":124 +/* "/home/cdyer/cdec/python/src/grammar.pxi":124 * * property a: * def __get__(self): # <<<<<<<<<<<<<< @@ -807,6 +839,22 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_4___get__ { }; +/* "/home/cdyer/cdec/python/src/lattice.pxi":52 + * return hypergraph.AsPLF(self.lattice[0], True).c_str() + * + * def __iter__(self): # <<<<<<<<<<<<<< + * cdef unsigned i + * for i in range(len(self)): + */ +struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ { + PyObject_HEAD + unsigned int __pyx_v_i; + struct __pyx_obj_5_cdec_Lattice *__pyx_v_self; + Py_ssize_t __pyx_t_0; + unsigned int __pyx_t_1; +}; + + /* "_cdec.pyx":39 * yield key, bytes(value) * @@ -821,7 +869,7 @@ struct __pyx_obj_5_cdec_Decoder { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":193 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":205 * raise NotImplemented('comparison not implemented for HypergraphEdge') * * cdef class HypergraphNode: # <<<<<<<<<<<<<< @@ -836,7 +884,7 @@ struct __pyx_obj_5_cdec_HypergraphNode { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":45 +/* "/home/cdyer/cdec/python/src/vectors.pxi":45 * return sparse * * cdef class SparseVector: # <<<<<<<<<<<<<< @@ -849,23 +897,23 @@ struct __pyx_obj_5_cdec_SparseVector { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":52 - * return hypergraph.AsPLF(self.lattice[0], True).c_str() +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":221 * - * def __iter__(self): # <<<<<<<<<<<<<< - * cdef unsigned i - * for i in range(len(self)): + * property out_edges: + * def __get__(self): # <<<<<<<<<<<<<< + * cdef unsigned i + * for i in range(self.node.out_edges_.size()): */ -struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ { +struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ { PyObject_HEAD unsigned int __pyx_v_i; - struct __pyx_obj_5_cdec_Lattice *__pyx_v_self; - Py_ssize_t __pyx_t_0; + struct __pyx_obj_5_cdec_HypergraphNode *__pyx_v_self; + size_t __pyx_t_0; unsigned int __pyx_t_1; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":31 +/* "/home/cdyer/cdec/python/src/vectors.pxi":31 * self.vector[0][fid] = value * * def __iter__(self): # <<<<<<<<<<<<<< @@ -881,23 +929,24 @@ struct __pyx_obj_5_cdec___pyx_scope_struct____iter__ { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90 - * return candidate +/* "/home/cdyer/cdec/python/src/mteval.pxi":44 + * return self.stats.size() * * def __iter__(self): # <<<<<<<<<<<<<< - * cdef unsigned i * for i in range(len(self)): + * yield self[i] */ struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ { PyObject_HEAD - unsigned int __pyx_v_i; - struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self; + PyObject *__pyx_v_i; + struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self; Py_ssize_t __pyx_t_0; - unsigned int __pyx_t_1; + PyObject *__pyx_t_1; + PyObject *(*__pyx_t_2)(PyObject *); }; -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":3 +/* "/home/cdyer/cdec/python/src/vectors.pxi":3 * from cython.operator cimport preincrement as pinc * * cdef class DenseVector: # <<<<<<<<<<<<<< @@ -911,7 +960,7 @@ struct __pyx_obj_5_cdec_DenseVector { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":187 +/* "/home/cdyer/cdec/python/src/grammar.pxi":187 * del self.grammar * * def __iter__(self): # <<<<<<<<<<<<<< @@ -930,7 +979,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":173 +/* "/home/cdyer/cdec/python/src/mteval.pxi":173 * out.fields[i] = ss[i] * * cdef class Metric: # <<<<<<<<<<<<<< @@ -943,7 +992,7 @@ struct __pyx_obj_5_cdec_Metric { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":26 +/* "/home/cdyer/cdec/python/src/mteval.pxi":26 * return fmap * * cdef class SufficientStats: # <<<<<<<<<<<<<< @@ -957,7 +1006,7 @@ struct __pyx_obj_5_cdec_SufficientStats { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":31 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":31 * return unicode(hypergraph.JoshuaVisualizationString(self.hg[0]).c_str(), 'utf8') * * def kbest(self, size): # <<<<<<<<<<<<<< @@ -976,7 +1025,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_8_kbest { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":61 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":61 * del e_derivations * * def kbest_features(self, size): # <<<<<<<<<<<<<< @@ -996,23 +1045,23 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_10_kbest_features { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":203 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":179 * - * property in_edges: + * property tail_nodes: * def __get__(self): # <<<<<<<<<<<<<< * cdef unsigned i - * for i in range(self.node.in_edges_.size()): + * for i in range(self.edge.tail_nodes_.size()): */ struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ { PyObject_HEAD unsigned int __pyx_v_i; - struct __pyx_obj_5_cdec_HypergraphNode *__pyx_v_self; - size_t __pyx_t_0; + struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self; + unsigned int __pyx_t_0; unsigned int __pyx_t_1; }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":164 +/* "/home/cdyer/cdec/python/src/grammar.pxi":164 * self.rule.get().lhs_ = -TDConvert(lhs.cat) * * def __str__(self): # <<<<<<<<<<<<<< @@ -1025,7 +1074,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_5___str__ { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":6 +/* "/home/cdyer/cdec/python/src/grammar.pxi":6 * * def _phrase(phrase): * return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase) # <<<<<<<<<<<<<< @@ -1042,7 +1091,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_3_genexpr { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":43 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":43 * del derivations * * def kbest_trees(self, size): # <<<<<<<<<<<<<< @@ -1065,7 +1114,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_9_kbest_trees { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":4 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":4 * cimport kbest * * cdef class Hypergraph: # <<<<<<<<<<<<<< @@ -1079,7 +1128,7 @@ struct __pyx_obj_5_cdec_Hypergraph { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":3 +/* "/home/cdyer/cdec/python/src/lattice.pxi":3 * cimport lattice * * cdef class Lattice: # <<<<<<<<<<<<<< @@ -1092,7 +1141,7 @@ struct __pyx_obj_5_cdec_Lattice { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":76 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":76 * del derivations * * def sample(self, unsigned n): # <<<<<<<<<<<<<< @@ -1110,22 +1159,6 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_11_sample { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":113 - * - * property edges: - * def __get__(self): # <<<<<<<<<<<<<< - * cdef unsigned i - * for i in range(self.hg.edges_.size()): - */ -struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ { - PyObject_HEAD - unsigned int __pyx_v_i; - struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self; - size_t __pyx_t_0; - unsigned int __pyx_t_1; -}; - - /* "_cdec.pyx":28 * class ParseFailed(Exception): pass * @@ -1133,7 +1166,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ { * for key, value in config.items(): * if isinstance(value, dict): */ -struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config { +struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config { PyObject_HEAD PyObject *__pyx_v_config; PyObject *__pyx_v_info; @@ -1149,7 +1182,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":204 +/* "/home/cdyer/cdec/python/src/grammar.pxi":204 * self.grammar.get().SetGrammarName(string(name)) * * cdef class TextGrammar(Grammar): # <<<<<<<<<<<<<< @@ -1161,31 +1194,8 @@ struct __pyx_obj_5_cdec_TextGrammar { }; -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":58 - * - * def todot(self): - * def lines(): # <<<<<<<<<<<<<< - * yield 'digraph lattice {' - * yield 'rankdir = LR;' - */ -struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines { - PyObject_HEAD - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *__pyx_outer_scope; - PyObject *__pyx_v_delta; - PyObject *__pyx_v_i; - PyObject *__pyx_v_label; - PyObject *__pyx_v_weight; - Py_ssize_t __pyx_t_0; - PyObject *__pyx_t_1; - PyObject *(*__pyx_t_2)(PyObject *); - PyObject *__pyx_t_3; - Py_ssize_t __pyx_t_4; - PyObject *(*__pyx_t_5)(PyObject *); -}; - - -/* "/Users/vchahun/Sandbox/cdec/python/src/cdec.sa._sa.pxd":1 +/* "/home/cdyer/cdec/python/src/cdec.sa._sa.pxd":1 * cdef class Phrase: # <<<<<<<<<<<<<< * cdef int *syms * cdef int n, *varpos, n_vars @@ -1198,7 +1208,7 @@ struct __pyx_vtabstruct_4cdec_2sa_3_sa_Phrase { static struct __pyx_vtabstruct_4cdec_2sa_3_sa_Phrase *__pyx_vtabptr_4cdec_2sa_3_sa_Phrase; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":193 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":205 * raise NotImplemented('comparison not implemented for HypergraphEdge') * * cdef class HypergraphNode: # <<<<<<<<<<<<<< @@ -1212,7 +1222,7 @@ struct __pyx_vtabstruct_5_cdec_HypergraphNode { static struct __pyx_vtabstruct_5_cdec_HypergraphNode *__pyx_vtabptr_5_cdec_HypergraphNode; -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":147 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":159 * return vector * * cdef class HypergraphEdge: # <<<<<<<<<<<<<< @@ -1308,28 +1318,13 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ const char* function_name); /*proto*/ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE int __Pyx_PyList_Append(PyObject* list, PyObject* x) { - PyListObject* L = (PyListObject*) list; - Py_ssize_t len = Py_SIZE(list); - if (likely(L->allocated > len)) { - Py_INCREF(x); - PyList_SET_ITEM(list, len, x); - Py_SIZE(list) = len+1; - return 0; - } - return PyList_Append(list, x); -} -#else -#define __Pyx_PyList_Append(L,x) PyList_Append(L,x) -#endif - static CYTHON_INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) { if (likely(PyList_CheckExact(L))) { - if (unlikely(PyList_Append(L, x) < 0)) return NULL; + if (PyList_Append(L, x) < 0) return NULL; Py_INCREF(Py_None); return Py_None; /* this is just to have an accurate signature */ - } else { + } + else { PyObject *r, *m; m = __Pyx_GetAttrString(L, "append"); if (!m) return NULL; @@ -1350,47 +1345,42 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j __Pyx_GetItemInt_List_Fast(o, i) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i) { -#if CYTHON_COMPILING_IN_CPYTHON - if (likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { - PyObject *r = PyList_GET_ITEM(o, i); - Py_INCREF(r); - return r; - } - else if ((-PyList_GET_SIZE(o) <= i) & (i < 0)) { - PyObject *r = PyList_GET_ITEM(o, PyList_GET_SIZE(o) + i); - Py_INCREF(r); - return r; + if (likely(o != Py_None)) { + if (likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + else if ((-PyList_GET_SIZE(o) <= i) & (i < 0)) { + PyObject *r = PyList_GET_ITEM(o, PyList_GET_SIZE(o) + i); + Py_INCREF(r); + return r; + } } return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif } #define __Pyx_GetItemInt_Tuple(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \ __Pyx_GetItemInt_Tuple_Fast(o, i) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i) { -#if CYTHON_COMPILING_IN_CPYTHON - if (likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { - PyObject *r = PyTuple_GET_ITEM(o, i); - Py_INCREF(r); - return r; - } - else if ((-PyTuple_GET_SIZE(o) <= i) & (i < 0)) { - PyObject *r = PyTuple_GET_ITEM(o, PyTuple_GET_SIZE(o) + i); - Py_INCREF(r); - return r; + if (likely(o != Py_None)) { + if (likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + else if ((-PyTuple_GET_SIZE(o) <= i) & (i < 0)) { + PyObject *r = PyTuple_GET_ITEM(o, PyTuple_GET_SIZE(o) + i); + Py_INCREF(r); + return r; + } } return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); -#else - return PySequence_GetItem(o, i); -#endif } #define __Pyx_GetItemInt(o, i, size, to_py_func) (((size) <= sizeof(Py_ssize_t)) ? \ __Pyx_GetItemInt_Fast(o, i) : \ __Pyx_GetItemInt_Generic(o, to_py_func(i))) static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i) { -#if CYTHON_COMPILING_IN_CPYTHON if (PyList_CheckExact(o)) { Py_ssize_t n = (likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); if (likely((n >= 0) & (n < PyList_GET_SIZE(o)))) { @@ -1406,30 +1396,19 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i) Py_INCREF(r); return r; } - } else { /* inlined PySequence_GetItem() */ + } + else if (likely(i >= 0)) { PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; if (likely(m && m->sq_item)) { - if (unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); - if (unlikely(l < 0)) return NULL; - i += l; - } return m->sq_item(o, i); } } -#else - if (PySequence_Check(o)) { - return PySequence_GetItem(o, i); - } -#endif return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); } -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); - static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index); -static CYTHON_INLINE int __Pyx_IterFinish(void); /*proto*/ +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected); static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected); /*proto*/ @@ -1440,20 +1419,13 @@ static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) { return unlikely(b < 0) ? NULL : __Pyx_PyBool_FromLong(b); } -#define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL) +#define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL); static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject *, PyObject *); /*proto*/ static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */ -#if CYTHON_COMPILING_IN_PYPY -#define __Pyx_PyObject_AsDouble(obj) \ -(likely(PyFloat_CheckExact(obj)) ? PyFloat_AS_DOUBLE(obj) : \ - likely(PyInt_CheckExact(obj)) ? \ - PyFloat_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) -#else #define __Pyx_PyObject_AsDouble(obj) \ ((likely(PyFloat_CheckExact(obj))) ? \ PyFloat_AS_DOUBLE(obj) : __Pyx__PyObject_AsDouble(obj)) -#endif static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/ @@ -1467,45 +1439,6 @@ static PyObject *__Pyx_FindPy2Metaclass(PyObject *bases); /*proto*/ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name, PyObject *modname); /*proto*/ -#ifndef __Pyx_CppExn2PyErr -#include -#include -#include -#include -static void __Pyx_CppExn2PyErr() { - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } catch (const std::bad_alloc& exn) { - PyErr_SetString(PyExc_MemoryError, exn.what()); - } catch (const std::bad_cast& exn) { - PyErr_SetString(PyExc_TypeError, exn.what()); - } catch (const std::domain_error& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::invalid_argument& exn) { - PyErr_SetString(PyExc_ValueError, exn.what()); - } catch (const std::ios_base::failure& exn) { - PyErr_SetString(PyExc_IOError, exn.what()); - } catch (const std::out_of_range& exn) { - PyErr_SetString(PyExc_IndexError, exn.what()); - } catch (const std::overflow_error& exn) { - PyErr_SetString(PyExc_OverflowError, exn.what()); - } catch (const std::range_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::underflow_error& exn) { - PyErr_SetString(PyExc_ArithmeticError, exn.what()); - } catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} -#endif - static CYTHON_INLINE WordID __Pyx_PyInt_from_py_WordID(PyObject *); static PyObject* __Pyx_Globals(void); /*proto*/ @@ -1603,7 +1536,6 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, #define __Pyx_Generator_USED #include -#include typedef PyObject *(*__pyx_generator_body_t)(PyObject *, PyObject *); typedef struct { PyObject_HEAD @@ -1616,17 +1548,10 @@ typedef struct { PyObject *exc_traceback; PyObject *gi_weakreflist; PyObject *classobj; - PyObject *yieldfrom; } __pyx_GeneratorObject; static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, PyObject *closure); static int __pyx_Generator_init(void); -static int __Pyx_Generator_clear(PyObject* self); -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); -#else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) -#endif static int __Pyx_check_binary_version(void); @@ -1668,8 +1593,6 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ -/* Module declarations from 'libc.string' */ - /* Module declarations from 'libcpp.string' */ /* Module declarations from 'libcpp.vector' */ @@ -1732,25 +1655,25 @@ static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_8_kbest = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_9_kbest_trees = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_10_kbest_features = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_11_sample = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_12___get__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_12_sample_trees = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_13___get__ = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_14___get__ = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_15___get__ = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_16___get__ = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_17___iter__ = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_18_todot = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_19_lines = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_20___iter__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_17___get__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_18___iter__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_19_todot = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_20_lines = 0; static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_21___iter__ = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_22__make_config = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_23___cinit__ = 0; -static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_24_genexpr = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_22___iter__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_23__make_config = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_24___cinit__ = 0; +static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_25_genexpr = 0; static char *__pyx_f_5_cdec_as_str(PyObject *, struct __pyx_opt_args_5_cdec_as_str *__pyx_optional_args); /*proto*/ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_obj_4cdec_2sa_3_sa_Rule *); /*proto*/ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject *, PyObject *); /*proto*/ static float __pyx_f_5_cdec__compute_score(void *, SufficientStats *); /*proto*/ static void __pyx_f_5_cdec__compute_sufficient_stats(void *, std::string *, std::vector *, SufficientStats *); /*proto*/ -static PyObject *__pyx_convert_string_to_py_(const std::string &); /*proto*/ #define __Pyx_MODULE_NAME "_cdec" int __pyx_module_is_main__cdec = 0; @@ -1840,15 +1763,16 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_10kbest(struct __pyx_obj_5_cdec_Hy static PyObject *__pyx_pf_5_cdec_10Hypergraph_13kbest_trees(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_16kbest_features(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_19sample(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n); /* proto */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat); /* proto */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs); /* proto */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_22sample_trees(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_25intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_27prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_29lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_31reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5edges___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5nodes___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4goal___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6npaths___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5_cdec_10Hypergraph_33inside_outside(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */ static Py_ssize_t __pyx_pf_5_cdec_14HypergraphEdge___len__(struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_9head_node___get__(struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_10tail_nodes___get__(struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self); /* proto */ @@ -1935,7 +1859,7 @@ static char __pyx_k_27[] = "digraph lattice {"; static char __pyx_k_32[] = "\\\""; static char __pyx_k_34[] = "%d [shape=doublecircle]"; static char __pyx_k_35[] = "}"; -static char __pyx_k_38[] = "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi"; +static char __pyx_k_38[] = "/home/cdyer/cdec/python/src/lattice.pxi"; static char __pyx_k_39[] = "\n"; static char __pyx_k_41[] = "sufficient stats vector index out of range"; static char __pyx_k_43[] = "candidate set index out of range"; @@ -1947,8 +1871,8 @@ static char __pyx_k_50[] = "#"; static char __pyx_k_53[] = "Cannot translate input type %s"; static char __pyx_k_54[] = "cdec.sa._sa"; static char __pyx_k_55[] = "*"; -static char __pyx_k_58[] = "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi"; -static char __pyx_k_64[] = "/Users/vchahun/Sandbox/cdec/python/src/_cdec.pyx"; +static char __pyx_k_58[] = "/home/cdyer/cdec/python/src/grammar.pxi"; +static char __pyx_k_64[] = "/home/cdyer/cdec/python/src/_cdec.pyx"; static char __pyx_k__a[] = "a"; static char __pyx_k__e[] = "e"; static char __pyx_k__f[] = "f"; @@ -2350,7 +2274,7 @@ static int __pyx_pw_5_cdec_11DenseVector_1__init__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":7 +/* "/home/cdyer/cdec/python/src/vectors.pxi":7 * cdef bint owned # if True, do not manage memory * * def __init__(self): # <<<<<<<<<<<<<< @@ -2363,7 +2287,7 @@ static int __pyx_pf_5_cdec_11DenseVector___init__(struct __pyx_obj_5_cdec_DenseV __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":8 + /* "/home/cdyer/cdec/python/src/vectors.pxi":8 * * def __init__(self): * self.vector = new vector[weight_t]() # <<<<<<<<<<<<<< @@ -2372,7 +2296,7 @@ static int __pyx_pf_5_cdec_11DenseVector___init__(struct __pyx_obj_5_cdec_DenseV */ __pyx_v_self->vector = new std::vector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":9 + /* "/home/cdyer/cdec/python/src/vectors.pxi":9 * def __init__(self): * self.vector = new vector[weight_t]() * self.owned = False # <<<<<<<<<<<<<< @@ -2395,7 +2319,7 @@ static void __pyx_pw_5_cdec_11DenseVector_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":11 +/* "/home/cdyer/cdec/python/src/vectors.pxi":11 * self.owned = False * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -2408,7 +2332,7 @@ static void __pyx_pf_5_cdec_11DenseVector_2__dealloc__(struct __pyx_obj_5_cdec_D int __pyx_t_1; __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":12 + /* "/home/cdyer/cdec/python/src/vectors.pxi":12 * * def __dealloc__(self): * if not self.owned: # <<<<<<<<<<<<<< @@ -2418,7 +2342,7 @@ static void __pyx_pf_5_cdec_11DenseVector_2__dealloc__(struct __pyx_obj_5_cdec_D __pyx_t_1 = (!__pyx_v_self->owned); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":13 + /* "/home/cdyer/cdec/python/src/vectors.pxi":13 * def __dealloc__(self): * if not self.owned: * del self.vector # <<<<<<<<<<<<<< @@ -2444,7 +2368,7 @@ static Py_ssize_t __pyx_pw_5_cdec_11DenseVector_5__len__(PyObject *__pyx_v_self) return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":15 +/* "/home/cdyer/cdec/python/src/vectors.pxi":15 * del self.vector * * def __len__(self): # <<<<<<<<<<<<<< @@ -2457,7 +2381,7 @@ static Py_ssize_t __pyx_pf_5_cdec_11DenseVector_4__len__(struct __pyx_obj_5_cdec __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":16 + /* "/home/cdyer/cdec/python/src/vectors.pxi":16 * * def __len__(self): * return self.vector.size() # <<<<<<<<<<<<<< @@ -2494,7 +2418,7 @@ static PyObject *__pyx_pw_5_cdec_11DenseVector_7__getitem__(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":18 +/* "/home/cdyer/cdec/python/src/vectors.pxi":18 * return self.vector.size() * * def __getitem__(self, char* fname): # <<<<<<<<<<<<<< @@ -2514,7 +2438,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_6__getitem__(struct __pyx_obj_5_c int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":19 + /* "/home/cdyer/cdec/python/src/vectors.pxi":19 * * def __getitem__(self, char* fname): * cdef int fid = FDConvert(fname) # <<<<<<<<<<<<<< @@ -2523,7 +2447,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_6__getitem__(struct __pyx_obj_5_c */ __pyx_v_fid = FD::Convert(__pyx_v_fname); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":20 + /* "/home/cdyer/cdec/python/src/vectors.pxi":20 * def __getitem__(self, char* fname): * cdef int fid = FDConvert(fname) * if 0 <= fid < self.vector.size(): # <<<<<<<<<<<<<< @@ -2536,7 +2460,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_6__getitem__(struct __pyx_obj_5_c } if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":21 + /* "/home/cdyer/cdec/python/src/vectors.pxi":21 * cdef int fid = FDConvert(fname) * if 0 <= fid < self.vector.size(): * return self.vector[0][fid] # <<<<<<<<<<<<<< @@ -2553,7 +2477,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_6__getitem__(struct __pyx_obj_5_c } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":22 + /* "/home/cdyer/cdec/python/src/vectors.pxi":22 * if 0 <= fid < self.vector.size(): * return self.vector[0][fid] * raise KeyError(fname) # <<<<<<<<<<<<<< @@ -2612,7 +2536,7 @@ static int __pyx_pw_5_cdec_11DenseVector_9__setitem__(PyObject *__pyx_v_self, Py return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":24 +/* "/home/cdyer/cdec/python/src/vectors.pxi":24 * raise KeyError(fname) * * def __setitem__(self, char* fname, float value): # <<<<<<<<<<<<<< @@ -2632,7 +2556,7 @@ static int __pyx_pf_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_5_cdec_De int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":25 + /* "/home/cdyer/cdec/python/src/vectors.pxi":25 * * def __setitem__(self, char* fname, float value): * cdef int fid = FDConvert(fname) # <<<<<<<<<<<<<< @@ -2641,7 +2565,7 @@ static int __pyx_pf_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_5_cdec_De */ __pyx_v_fid = FD::Convert(((char *)__pyx_v_fname)); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":26 + /* "/home/cdyer/cdec/python/src/vectors.pxi":26 * def __setitem__(self, char* fname, float value): * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) # <<<<<<<<<<<<<< @@ -2667,7 +2591,7 @@ static int __pyx_pf_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_5_cdec_De } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":27 + /* "/home/cdyer/cdec/python/src/vectors.pxi":27 * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) * if self.vector.size() <= fid: # <<<<<<<<<<<<<< @@ -2677,7 +2601,7 @@ static int __pyx_pf_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_5_cdec_De __pyx_t_1 = (__pyx_v_self->vector->size() <= __pyx_v_fid); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":28 + /* "/home/cdyer/cdec/python/src/vectors.pxi":28 * if fid < 0: raise KeyError(fname) * if self.vector.size() <= fid: * self.vector.resize(fid + 1) # <<<<<<<<<<<<<< @@ -2689,7 +2613,7 @@ static int __pyx_pf_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_5_cdec_De } __pyx_L4:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":29 + /* "/home/cdyer/cdec/python/src/vectors.pxi":29 * if self.vector.size() <= fid: * self.vector.resize(fid + 1) * self.vector[0][fid] = value # <<<<<<<<<<<<<< @@ -2722,7 +2646,7 @@ static PyObject *__pyx_pw_5_cdec_11DenseVector_11__iter__(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":31 +/* "/home/cdyer/cdec/python/src/vectors.pxi":31 * self.vector[0][fid] = value * * def __iter__(self): # <<<<<<<<<<<<<< @@ -2787,7 +2711,7 @@ static PyObject *__pyx_gb_5_cdec_11DenseVector_12generator(__pyx_GeneratorObject __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":33 + /* "/home/cdyer/cdec/python/src/vectors.pxi":33 * def __iter__(self): * cdef unsigned fid * for fid in range(1, self.vector.size()): # <<<<<<<<<<<<<< @@ -2798,7 +2722,7 @@ static PyObject *__pyx_gb_5_cdec_11DenseVector_12generator(__pyx_GeneratorObject for (__pyx_t_2 = 1; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_fid = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":34 + /* "/home/cdyer/cdec/python/src/vectors.pxi":34 * cdef unsigned fid * for fid in range(1, self.vector.size()): * yield FDConvert(fid).c_str(), self.vector[0][fid] # <<<<<<<<<<<<<< @@ -2841,7 +2765,6 @@ static PyObject *__pyx_gb_5_cdec_11DenseVector_12generator(__pyx_GeneratorObject __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -2862,7 +2785,7 @@ static PyObject *__pyx_pw_5_cdec_11DenseVector_14dot(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":36 +/* "/home/cdyer/cdec/python/src/vectors.pxi":36 * yield FDConvert(fid).c_str(), self.vector[0][fid] * * def dot(self, SparseVector other): # <<<<<<<<<<<<<< @@ -2881,7 +2804,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_13dot(struct __pyx_obj_5_cdec_Den int __pyx_clineno = 0; __Pyx_RefNannySetupContext("dot", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":37 + /* "/home/cdyer/cdec/python/src/vectors.pxi":37 * * def dot(self, SparseVector other): * return other.dot(self) # <<<<<<<<<<<<<< @@ -2929,7 +2852,7 @@ static PyObject *__pyx_pw_5_cdec_11DenseVector_16tosparse(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":39 +/* "/home/cdyer/cdec/python/src/vectors.pxi":39 * return other.dot(self) * * def tosparse(self): # <<<<<<<<<<<<<< @@ -2947,7 +2870,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_15tosparse(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("tosparse", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":40 + /* "/home/cdyer/cdec/python/src/vectors.pxi":40 * * def tosparse(self): * cdef SparseVector sparse = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -2960,7 +2883,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_15tosparse(struct __pyx_obj_5_cde __pyx_v_sparse = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":41 + /* "/home/cdyer/cdec/python/src/vectors.pxi":41 * def tosparse(self): * cdef SparseVector sparse = SparseVector.__new__(SparseVector) * sparse.vector = new FastSparseVector[weight_t]() # <<<<<<<<<<<<<< @@ -2969,7 +2892,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_15tosparse(struct __pyx_obj_5_cde */ __pyx_v_sparse->vector = new FastSparseVector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":42 + /* "/home/cdyer/cdec/python/src/vectors.pxi":42 * cdef SparseVector sparse = SparseVector.__new__(SparseVector) * sparse.vector = new FastSparseVector[weight_t]() * InitSparseVector(self.vector[0], sparse.vector) # <<<<<<<<<<<<<< @@ -2978,7 +2901,7 @@ static PyObject *__pyx_pf_5_cdec_11DenseVector_15tosparse(struct __pyx_obj_5_cde */ Weights::InitSparseVector((__pyx_v_self->vector[0]), __pyx_v_sparse->vector); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":43 + /* "/home/cdyer/cdec/python/src/vectors.pxi":43 * sparse.vector = new FastSparseVector[weight_t]() * InitSparseVector(self.vector[0], sparse.vector) * return sparse # <<<<<<<<<<<<<< @@ -3017,7 +2940,7 @@ static int __pyx_pw_5_cdec_12SparseVector_1__init__(PyObject *__pyx_v_self, PyOb return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":48 +/* "/home/cdyer/cdec/python/src/vectors.pxi":48 * cdef FastSparseVector[weight_t]* vector * * def __init__(self): # <<<<<<<<<<<<<< @@ -3030,7 +2953,7 @@ static int __pyx_pf_5_cdec_12SparseVector___init__(struct __pyx_obj_5_cdec_Spars __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":49 + /* "/home/cdyer/cdec/python/src/vectors.pxi":49 * * def __init__(self): * self.vector = new FastSparseVector[weight_t]() # <<<<<<<<<<<<<< @@ -3053,7 +2976,7 @@ static void __pyx_pw_5_cdec_12SparseVector_3__dealloc__(PyObject *__pyx_v_self) __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":51 +/* "/home/cdyer/cdec/python/src/vectors.pxi":51 * self.vector = new FastSparseVector[weight_t]() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -3065,7 +2988,7 @@ static void __pyx_pf_5_cdec_12SparseVector_2__dealloc__(CYTHON_UNUSED struct __p __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":52 + /* "/home/cdyer/cdec/python/src/vectors.pxi":52 * * def __dealloc__(self): * del self.vector # <<<<<<<<<<<<<< @@ -3088,7 +3011,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_5copy(PyObject *__pyx_v_self, CY return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":54 +/* "/home/cdyer/cdec/python/src/vectors.pxi":54 * del self.vector * * def copy(self): # <<<<<<<<<<<<<< @@ -3105,7 +3028,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_4copy(struct __pyx_obj_5_cdec_Sp int __pyx_clineno = 0; __Pyx_RefNannySetupContext("copy", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":55 + /* "/home/cdyer/cdec/python/src/vectors.pxi":55 * * def copy(self): * return self * 1 # <<<<<<<<<<<<<< @@ -3152,7 +3075,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_7__getitem__(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":57 +/* "/home/cdyer/cdec/python/src/vectors.pxi":57 * return self * 1 * * def __getitem__(self, char* fname): # <<<<<<<<<<<<<< @@ -3172,7 +3095,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_6__getitem__(struct __pyx_obj_5_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":58 + /* "/home/cdyer/cdec/python/src/vectors.pxi":58 * * def __getitem__(self, char* fname): * cdef int fid = FDConvert(fname) # <<<<<<<<<<<<<< @@ -3181,7 +3104,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_6__getitem__(struct __pyx_obj_5_ */ __pyx_v_fid = FD::Convert(__pyx_v_fname); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":59 + /* "/home/cdyer/cdec/python/src/vectors.pxi":59 * def __getitem__(self, char* fname): * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) # <<<<<<<<<<<<<< @@ -3207,7 +3130,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_6__getitem__(struct __pyx_obj_5_ } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":60 + /* "/home/cdyer/cdec/python/src/vectors.pxi":60 * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) * return self.vector.value(fid) # <<<<<<<<<<<<<< @@ -3259,7 +3182,7 @@ static int __pyx_pw_5_cdec_12SparseVector_9__setitem__(PyObject *__pyx_v_self, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":62 +/* "/home/cdyer/cdec/python/src/vectors.pxi":62 * return self.vector.value(fid) * * def __setitem__(self, char* fname, float value): # <<<<<<<<<<<<<< @@ -3279,7 +3202,7 @@ static int __pyx_pf_5_cdec_12SparseVector_8__setitem__(struct __pyx_obj_5_cdec_S int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":63 + /* "/home/cdyer/cdec/python/src/vectors.pxi":63 * * def __setitem__(self, char* fname, float value): * cdef int fid = FDConvert(fname) # <<<<<<<<<<<<<< @@ -3288,7 +3211,7 @@ static int __pyx_pf_5_cdec_12SparseVector_8__setitem__(struct __pyx_obj_5_cdec_S */ __pyx_v_fid = FD::Convert(((char *)__pyx_v_fname)); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":64 + /* "/home/cdyer/cdec/python/src/vectors.pxi":64 * def __setitem__(self, char* fname, float value): * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) # <<<<<<<<<<<<<< @@ -3314,7 +3237,7 @@ static int __pyx_pf_5_cdec_12SparseVector_8__setitem__(struct __pyx_obj_5_cdec_S } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":65 + /* "/home/cdyer/cdec/python/src/vectors.pxi":65 * cdef int fid = FDConvert(fname) * if fid < 0: raise KeyError(fname) * self.vector.set_value(fid, value) # <<<<<<<<<<<<<< @@ -3347,7 +3270,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_11__iter__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":67 +/* "/home/cdyer/cdec/python/src/vectors.pxi":67 * self.vector.set_value(fid, value) * * def __iter__(self): # <<<<<<<<<<<<<< @@ -3412,7 +3335,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":68 + /* "/home/cdyer/cdec/python/src/vectors.pxi":68 * * def __iter__(self): * cdef FastSparseVector[weight_t].const_iterator* it = new FastSparseVector[weight_t].const_iterator(self.vector[0], False) # <<<<<<<<<<<<<< @@ -3421,7 +3344,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje */ __pyx_cur_scope->__pyx_v_it = new FastSparseVector::const_iterator((__pyx_cur_scope->__pyx_v_self->vector[0]), 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":70 + /* "/home/cdyer/cdec/python/src/vectors.pxi":70 * cdef FastSparseVector[weight_t].const_iterator* it = new FastSparseVector[weight_t].const_iterator(self.vector[0], False) * cdef unsigned i * try: # <<<<<<<<<<<<<< @@ -3430,7 +3353,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje */ /*try:*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":71 + /* "/home/cdyer/cdec/python/src/vectors.pxi":71 * cdef unsigned i * try: * for i in range(self.vector.size()): # <<<<<<<<<<<<<< @@ -3441,7 +3364,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":72 + /* "/home/cdyer/cdec/python/src/vectors.pxi":72 * try: * for i in range(self.vector.size()): * yield (FDConvert(it[0].ptr().first).c_str(), it[0].ptr().second) # <<<<<<<<<<<<<< @@ -3474,7 +3397,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L5;} - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":73 + /* "/home/cdyer/cdec/python/src/vectors.pxi":73 * for i in range(self.vector.size()): * yield (FDConvert(it[0].ptr().first).c_str(), it[0].ptr().second) * pinc(it[0]) # ++it # <<<<<<<<<<<<<< @@ -3485,7 +3408,7 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje } } - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":75 + /* "/home/cdyer/cdec/python/src/vectors.pxi":75 * pinc(it[0]) # ++it * finally: * del it # <<<<<<<<<<<<<< @@ -3530,7 +3453,6 @@ static PyObject *__pyx_gb_5_cdec_12SparseVector_12generator1(__pyx_GeneratorObje __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -3546,7 +3468,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_14dot(PyObject *__pyx_v_self, Py return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":77 +/* "/home/cdyer/cdec/python/src/vectors.pxi":77 * del it * * def dot(self, other): # <<<<<<<<<<<<<< @@ -3565,7 +3487,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_13dot(struct __pyx_obj_5_cdec_Sp int __pyx_clineno = 0; __Pyx_RefNannySetupContext("dot", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":78 + /* "/home/cdyer/cdec/python/src/vectors.pxi":78 * * def dot(self, other): * if isinstance(other, DenseVector): # <<<<<<<<<<<<<< @@ -3578,7 +3500,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_13dot(struct __pyx_obj_5_cdec_Sp __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":79 + /* "/home/cdyer/cdec/python/src/vectors.pxi":79 * def dot(self, other): * if isinstance(other, DenseVector): * return self.vector.dot(( other).vector[0]) # <<<<<<<<<<<<<< @@ -3594,7 +3516,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_13dot(struct __pyx_obj_5_cdec_Sp goto __pyx_L3; } - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":80 + /* "/home/cdyer/cdec/python/src/vectors.pxi":80 * if isinstance(other, DenseVector): * return self.vector.dot(( other).vector[0]) * elif isinstance(other, SparseVector): # <<<<<<<<<<<<<< @@ -3607,7 +3529,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_13dot(struct __pyx_obj_5_cdec_Sp __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":81 + /* "/home/cdyer/cdec/python/src/vectors.pxi":81 * return self.vector.dot(( other).vector[0]) * elif isinstance(other, SparseVector): * return self.vector.dot(( other).vector[0]) # <<<<<<<<<<<<<< @@ -3624,7 +3546,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_13dot(struct __pyx_obj_5_cdec_Sp } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":82 + /* "/home/cdyer/cdec/python/src/vectors.pxi":82 * elif isinstance(other, SparseVector): * return self.vector.dot(( other).vector[0]) * raise TypeError('cannot take the dot product of %s and SparseVector' % type(other)) # <<<<<<<<<<<<<< @@ -3675,7 +3597,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_16__richcmp__(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":84 +/* "/home/cdyer/cdec/python/src/vectors.pxi":84 * raise TypeError('cannot take the dot product of %s and SparseVector' % type(other)) * * def __richcmp__(SparseVector x, SparseVector y, int op): # <<<<<<<<<<<<<< @@ -3693,7 +3615,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__richcmp__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":87 + /* "/home/cdyer/cdec/python/src/vectors.pxi":87 * if op == 2: # == * return x.vector[0] == y.vector[0] * elif op == 3: # != # <<<<<<<<<<<<<< @@ -3702,7 +3624,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 */ switch (__pyx_v_op) { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":85 + /* "/home/cdyer/cdec/python/src/vectors.pxi":85 * * def __richcmp__(SparseVector x, SparseVector y, int op): * if op == 2: # == # <<<<<<<<<<<<<< @@ -3711,7 +3633,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 */ case 2: - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":86 + /* "/home/cdyer/cdec/python/src/vectors.pxi":86 * def __richcmp__(SparseVector x, SparseVector y, int op): * if op == 2: # == * return x.vector[0] == y.vector[0] # <<<<<<<<<<<<<< @@ -3726,7 +3648,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 goto __pyx_L0; break; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":87 + /* "/home/cdyer/cdec/python/src/vectors.pxi":87 * if op == 2: # == * return x.vector[0] == y.vector[0] * elif op == 3: # != # <<<<<<<<<<<<<< @@ -3735,7 +3657,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 */ case 3: - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":88 + /* "/home/cdyer/cdec/python/src/vectors.pxi":88 * return x.vector[0] == y.vector[0] * elif op == 3: # != * return not (x == y) # <<<<<<<<<<<<<< @@ -3755,7 +3677,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_15__richcmp__(struct __pyx_obj_5 break; } - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":89 + /* "/home/cdyer/cdec/python/src/vectors.pxi":89 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for SparseVector') # <<<<<<<<<<<<<< @@ -3791,7 +3713,7 @@ static Py_ssize_t __pyx_pw_5_cdec_12SparseVector_18__len__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":91 +/* "/home/cdyer/cdec/python/src/vectors.pxi":91 * raise NotImplemented('comparison not implemented for SparseVector') * * def __len__(self): # <<<<<<<<<<<<<< @@ -3804,7 +3726,7 @@ static Py_ssize_t __pyx_pf_5_cdec_12SparseVector_17__len__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":92 + /* "/home/cdyer/cdec/python/src/vectors.pxi":92 * * def __len__(self): * return self.vector.size() # <<<<<<<<<<<<<< @@ -3841,7 +3763,7 @@ static int __pyx_pw_5_cdec_12SparseVector_20__contains__(PyObject *__pyx_v_self, return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":94 +/* "/home/cdyer/cdec/python/src/vectors.pxi":94 * return self.vector.size() * * def __contains__(self, char* fname): # <<<<<<<<<<<<<< @@ -3854,7 +3776,7 @@ static int __pyx_pf_5_cdec_12SparseVector_19__contains__(struct __pyx_obj_5_cdec __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__contains__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":95 + /* "/home/cdyer/cdec/python/src/vectors.pxi":95 * * def __contains__(self, char* fname): * return self.vector.nonzero(FDConvert(fname)) # <<<<<<<<<<<<<< @@ -3881,7 +3803,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_22__neg__(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":97 +/* "/home/cdyer/cdec/python/src/vectors.pxi":97 * return self.vector.nonzero(FDConvert(fname)) * * def __neg__(self): # <<<<<<<<<<<<<< @@ -3899,7 +3821,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_21__neg__(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__neg__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":98 + /* "/home/cdyer/cdec/python/src/vectors.pxi":98 * * def __neg__(self): * cdef SparseVector result = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -3912,7 +3834,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_21__neg__(struct __pyx_obj_5_cde __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":99 + /* "/home/cdyer/cdec/python/src/vectors.pxi":99 * def __neg__(self): * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](self.vector[0]) # <<<<<<<<<<<<<< @@ -3921,7 +3843,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_21__neg__(struct __pyx_obj_5_cde */ __pyx_v_result->vector = new FastSparseVector((__pyx_v_self->vector[0])); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":100 + /* "/home/cdyer/cdec/python/src/vectors.pxi":100 * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](self.vector[0]) * result.vector[0] *= -1.0 # <<<<<<<<<<<<<< @@ -3930,7 +3852,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_21__neg__(struct __pyx_obj_5_cde */ (__pyx_v_result->vector[0]) *= -1.0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":101 + /* "/home/cdyer/cdec/python/src/vectors.pxi":101 * result.vector = new FastSparseVector[weight_t](self.vector[0]) * result.vector[0] *= -1.0 * return result # <<<<<<<<<<<<<< @@ -3971,7 +3893,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_24__iadd__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":103 +/* "/home/cdyer/cdec/python/src/vectors.pxi":103 * return result * * def __iadd__(SparseVector self, SparseVector other): # <<<<<<<<<<<<<< @@ -3984,7 +3906,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_23__iadd__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iadd__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":104 + /* "/home/cdyer/cdec/python/src/vectors.pxi":104 * * def __iadd__(SparseVector self, SparseVector other): * self.vector[0] += other.vector[0] # <<<<<<<<<<<<<< @@ -3993,7 +3915,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_23__iadd__(struct __pyx_obj_5_cd */ (__pyx_v_self->vector[0]) += (__pyx_v_other->vector[0]); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":105 + /* "/home/cdyer/cdec/python/src/vectors.pxi":105 * def __iadd__(SparseVector self, SparseVector other): * self.vector[0] += other.vector[0] * return self # <<<<<<<<<<<<<< @@ -4028,7 +3950,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_26__isub__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":107 +/* "/home/cdyer/cdec/python/src/vectors.pxi":107 * return self * * def __isub__(SparseVector self, SparseVector other): # <<<<<<<<<<<<<< @@ -4041,7 +3963,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_25__isub__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__isub__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":108 + /* "/home/cdyer/cdec/python/src/vectors.pxi":108 * * def __isub__(SparseVector self, SparseVector other): * self.vector[0] -= other.vector[0] # <<<<<<<<<<<<<< @@ -4050,7 +3972,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_25__isub__(struct __pyx_obj_5_cd */ (__pyx_v_self->vector[0]) -= (__pyx_v_other->vector[0]); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":109 + /* "/home/cdyer/cdec/python/src/vectors.pxi":109 * def __isub__(SparseVector self, SparseVector other): * self.vector[0] -= other.vector[0] * return self # <<<<<<<<<<<<<< @@ -4090,7 +4012,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_28__imul__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":111 +/* "/home/cdyer/cdec/python/src/vectors.pxi":111 * return self * * def __imul__(SparseVector self, float scalar): # <<<<<<<<<<<<<< @@ -4103,7 +4025,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_27__imul__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__imul__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":112 + /* "/home/cdyer/cdec/python/src/vectors.pxi":112 * * def __imul__(SparseVector self, float scalar): * self.vector[0] *= scalar # <<<<<<<<<<<<<< @@ -4112,7 +4034,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_27__imul__(struct __pyx_obj_5_cd */ (__pyx_v_self->vector[0]) *= __pyx_v_scalar; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":113 + /* "/home/cdyer/cdec/python/src/vectors.pxi":113 * def __imul__(SparseVector self, float scalar): * self.vector[0] *= scalar * return self # <<<<<<<<<<<<<< @@ -4154,7 +4076,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_30__idiv__(PyObject *__pyx_v_sel } #endif /*!(#if PY_MAJOR_VERSION < 3)*/ -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":115 +/* "/home/cdyer/cdec/python/src/vectors.pxi":115 * return self * * def __idiv__(SparseVector self, float scalar): # <<<<<<<<<<<<<< @@ -4168,7 +4090,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_29__idiv__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__idiv__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":116 + /* "/home/cdyer/cdec/python/src/vectors.pxi":116 * * def __idiv__(SparseVector self, float scalar): * self.vector[0] /= scalar # <<<<<<<<<<<<<< @@ -4177,7 +4099,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_29__idiv__(struct __pyx_obj_5_cd */ (__pyx_v_self->vector[0]) /= __pyx_v_scalar; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":117 + /* "/home/cdyer/cdec/python/src/vectors.pxi":117 * def __idiv__(SparseVector self, float scalar): * self.vector[0] /= scalar * return self # <<<<<<<<<<<<<< @@ -4214,7 +4136,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_32__add__(PyObject *__pyx_v_x, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":119 +/* "/home/cdyer/cdec/python/src/vectors.pxi":119 * return self * * def __add__(SparseVector x, SparseVector y): # <<<<<<<<<<<<<< @@ -4232,7 +4154,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_31__add__(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__add__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":120 + /* "/home/cdyer/cdec/python/src/vectors.pxi":120 * * def __add__(SparseVector x, SparseVector y): * cdef SparseVector result = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -4245,7 +4167,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_31__add__(struct __pyx_obj_5_cde __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":121 + /* "/home/cdyer/cdec/python/src/vectors.pxi":121 * def __add__(SparseVector x, SparseVector y): * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0]) # <<<<<<<<<<<<<< @@ -4254,7 +4176,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_31__add__(struct __pyx_obj_5_cde */ __pyx_v_result->vector = new FastSparseVector(((__pyx_v_x->vector[0]) + (__pyx_v_y->vector[0]))); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":122 + /* "/home/cdyer/cdec/python/src/vectors.pxi":122 * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0]) * return result # <<<<<<<<<<<<<< @@ -4296,7 +4218,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_34__sub__(PyObject *__pyx_v_x, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":124 +/* "/home/cdyer/cdec/python/src/vectors.pxi":124 * return result * * def __sub__(SparseVector x, SparseVector y): # <<<<<<<<<<<<<< @@ -4314,7 +4236,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_33__sub__(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__sub__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":125 + /* "/home/cdyer/cdec/python/src/vectors.pxi":125 * * def __sub__(SparseVector x, SparseVector y): * cdef SparseVector result = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -4327,7 +4249,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_33__sub__(struct __pyx_obj_5_cde __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":126 + /* "/home/cdyer/cdec/python/src/vectors.pxi":126 * def __sub__(SparseVector x, SparseVector y): * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0]) # <<<<<<<<<<<<<< @@ -4336,7 +4258,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_33__sub__(struct __pyx_obj_5_cde */ __pyx_v_result->vector = new FastSparseVector(((__pyx_v_x->vector[0]) - (__pyx_v_y->vector[0]))); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":127 + /* "/home/cdyer/cdec/python/src/vectors.pxi":127 * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0]) * return result # <<<<<<<<<<<<<< @@ -4372,7 +4294,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_36__mul__(PyObject *__pyx_v_x, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":129 +/* "/home/cdyer/cdec/python/src/vectors.pxi":129 * return result * * def __mul__(x, y): # <<<<<<<<<<<<<< @@ -4394,7 +4316,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_35__mul__(PyObject *__pyx_v_x, P int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__mul__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":132 + /* "/home/cdyer/cdec/python/src/vectors.pxi":132 * cdef SparseVector vector * cdef float scalar * if isinstance(x, SparseVector): vector, scalar = x, y # <<<<<<<<<<<<<< @@ -4417,7 +4339,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_35__mul__(PyObject *__pyx_v_x, P } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":133 + /* "/home/cdyer/cdec/python/src/vectors.pxi":133 * cdef float scalar * if isinstance(x, SparseVector): vector, scalar = x, y * else: vector, scalar = y, x # <<<<<<<<<<<<<< @@ -4434,7 +4356,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_35__mul__(PyObject *__pyx_v_x, P } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":134 + /* "/home/cdyer/cdec/python/src/vectors.pxi":134 * if isinstance(x, SparseVector): vector, scalar = x, y * else: vector, scalar = y, x * cdef SparseVector result = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -4447,7 +4369,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_35__mul__(PyObject *__pyx_v_x, P __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":135 + /* "/home/cdyer/cdec/python/src/vectors.pxi":135 * else: vector, scalar = y, x * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar) # <<<<<<<<<<<<<< @@ -4456,7 +4378,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_35__mul__(PyObject *__pyx_v_x, P */ __pyx_v_result->vector = new FastSparseVector(((__pyx_v_vector->vector[0]) * __pyx_v_scalar)); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":136 + /* "/home/cdyer/cdec/python/src/vectors.pxi":136 * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar) * return result # <<<<<<<<<<<<<< @@ -4495,7 +4417,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_38__div__(PyObject *__pyx_v_x, P } #endif /*!(#if PY_MAJOR_VERSION < 3)*/ -/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":138 +/* "/home/cdyer/cdec/python/src/vectors.pxi":138 * return result * * def __div__(x, y): # <<<<<<<<<<<<<< @@ -4518,7 +4440,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_37__div__(PyObject *__pyx_v_x, P int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__div__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":141 + /* "/home/cdyer/cdec/python/src/vectors.pxi":141 * cdef SparseVector vector * cdef float scalar * if isinstance(x, SparseVector): vector, scalar = x, y # <<<<<<<<<<<<<< @@ -4541,7 +4463,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_37__div__(PyObject *__pyx_v_x, P } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":142 + /* "/home/cdyer/cdec/python/src/vectors.pxi":142 * cdef float scalar * if isinstance(x, SparseVector): vector, scalar = x, y * else: vector, scalar = y, x # <<<<<<<<<<<<<< @@ -4558,7 +4480,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_37__div__(PyObject *__pyx_v_x, P } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":143 + /* "/home/cdyer/cdec/python/src/vectors.pxi":143 * if isinstance(x, SparseVector): vector, scalar = x, y * else: vector, scalar = y, x * cdef SparseVector result = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -4571,7 +4493,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_37__div__(PyObject *__pyx_v_x, P __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":144 + /* "/home/cdyer/cdec/python/src/vectors.pxi":144 * else: vector, scalar = y, x * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar) # <<<<<<<<<<<<<< @@ -4579,7 +4501,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_37__div__(PyObject *__pyx_v_x, P */ __pyx_v_result->vector = new FastSparseVector(((__pyx_v_vector->vector[0]) / __pyx_v_scalar)); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":145 + /* "/home/cdyer/cdec/python/src/vectors.pxi":145 * cdef SparseVector result = SparseVector.__new__(SparseVector) * result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar) * return result # <<<<<<<<<<<<<< @@ -4611,13 +4533,14 @@ static PyObject *__pyx_pw_5_cdec_1_phrase(PyObject *__pyx_self, PyObject *__pyx_ PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("_phrase (wrapper)", 0); + __pyx_self = __pyx_self; __pyx_r = __pyx_pf_5_cdec__phrase(__pyx_self, ((PyObject *)__pyx_v_phrase)); __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7_phrase_2generator17(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_7_phrase_2generator18(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":6 +/* "/home/cdyer/cdec/python/src/grammar.pxi":6 * * def _phrase(phrase): * return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase) # <<<<<<<<<<<<<< @@ -4643,7 +4566,7 @@ static PyObject *__pyx_pf_5_cdec_7_phrase_genexpr(PyObject *__pyx_self) { __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope)); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7_phrase_2generator17, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7_phrase_2generator18, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -4661,7 +4584,7 @@ static PyObject *__pyx_pf_5_cdec_7_phrase_genexpr(PyObject *__pyx_self) { return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7_phrase_2generator17(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_7_phrase_2generator18(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { struct __pyx_obj_5_cdec___pyx_scope_struct_3_genexpr *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_3_genexpr *)__pyx_generator->closure); PyObject *__pyx_r = NULL; @@ -4695,18 +4618,10 @@ static PyObject *__pyx_gb_5_cdec_7_phrase_2generator17(__pyx_GeneratorObject *__ for (;;) { if (!__pyx_t_3 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_3 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_4 = __pyx_t_3(__pyx_t_1); if (unlikely(!__pyx_t_4)) { @@ -4778,12 +4693,11 @@ static PyObject *__pyx_gb_5_cdec_7_phrase_2generator17(__pyx_GeneratorObject *__ __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":5 +/* "/home/cdyer/cdec/python/src/grammar.pxi":5 * import cdec.sa._sa as _sa * * def _phrase(phrase): # <<<<<<<<<<<<<< @@ -4812,7 +4726,7 @@ static PyObject *__pyx_pf_5_cdec__phrase(CYTHON_UNUSED PyObject *__pyx_self, PyO __Pyx_INCREF(__pyx_cur_scope->__pyx_v_phrase); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_phrase); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":6 + /* "/home/cdyer/cdec/python/src/grammar.pxi":6 * * def _phrase(phrase): * return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase) # <<<<<<<<<<<<<< @@ -4857,11 +4771,11 @@ static int __pyx_pw_5_cdec_2NT_1__init__(PyObject *__pyx_v_self, PyObject *__pyx static int __pyx_pw_5_cdec_2NT_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { char *__pyx_v_cat; unsigned int __pyx_v_ref; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__cat,&__pyx_n_s__ref,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__cat,&__pyx_n_s__ref,0}; PyObject* values[2] = {0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -4875,7 +4789,8 @@ static int __pyx_pw_5_cdec_2NT_1__init__(PyObject *__pyx_v_self, PyObject *__pyx kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__cat)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__cat); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (kw_args > 0) { @@ -4886,6 +4801,10 @@ static int __pyx_pw_5_cdec_2NT_1__init__(PyObject *__pyx_v_self, PyObject *__pyx if (unlikely(kw_args > 0)) { if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } + if (values[1]) { + } else { + __pyx_v_ref = ((unsigned int)0); + } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); @@ -4914,7 +4833,7 @@ static int __pyx_pw_5_cdec_2NT_1__init__(PyObject *__pyx_v_self, PyObject *__pyx return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":11 +/* "/home/cdyer/cdec/python/src/grammar.pxi":11 * cdef public bytes cat * cdef public unsigned ref * def __init__(self, char* cat, unsigned ref=0): # <<<<<<<<<<<<<< @@ -4931,7 +4850,7 @@ static int __pyx_pf_5_cdec_2NT___init__(struct __pyx_obj_5_cdec_NT *__pyx_v_self int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":12 + /* "/home/cdyer/cdec/python/src/grammar.pxi":12 * cdef public unsigned ref * def __init__(self, char* cat, unsigned ref=0): * self.cat = cat # <<<<<<<<<<<<<< @@ -4946,7 +4865,7 @@ static int __pyx_pf_5_cdec_2NT___init__(struct __pyx_obj_5_cdec_NT *__pyx_v_self __pyx_v_self->cat = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":13 + /* "/home/cdyer/cdec/python/src/grammar.pxi":13 * def __init__(self, char* cat, unsigned ref=0): * self.cat = cat * self.ref = ref # <<<<<<<<<<<<<< @@ -4977,7 +4896,7 @@ static PyObject *__pyx_pw_5_cdec_2NT_3__str__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":15 +/* "/home/cdyer/cdec/python/src/grammar.pxi":15 * self.ref = ref * * def __str__(self): # <<<<<<<<<<<<<< @@ -4996,7 +4915,7 @@ static PyObject *__pyx_pf_5_cdec_2NT_2__str__(struct __pyx_obj_5_cdec_NT *__pyx_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":16 + /* "/home/cdyer/cdec/python/src/grammar.pxi":16 * * def __str__(self): * if self.ref > 0: # <<<<<<<<<<<<<< @@ -5006,7 +4925,7 @@ static PyObject *__pyx_pf_5_cdec_2NT_2__str__(struct __pyx_obj_5_cdec_NT *__pyx_ __pyx_t_1 = (__pyx_v_self->ref > 0); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":17 + /* "/home/cdyer/cdec/python/src/grammar.pxi":17 * def __str__(self): * if self.ref > 0: * return '[%s,%d]' % (self.cat, self.ref) # <<<<<<<<<<<<<< @@ -5034,7 +4953,7 @@ static PyObject *__pyx_pf_5_cdec_2NT_2__str__(struct __pyx_obj_5_cdec_NT *__pyx_ } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":18 + /* "/home/cdyer/cdec/python/src/grammar.pxi":18 * if self.ref > 0: * return '[%s,%d]' % (self.cat, self.ref) * return '[%s]' % self.cat # <<<<<<<<<<<<<< @@ -5072,7 +4991,7 @@ static PyObject *__pyx_pw_5_cdec_2NT_3cat_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":9 +/* "/home/cdyer/cdec/python/src/grammar.pxi":9 * * cdef class NT: * cdef public bytes cat # <<<<<<<<<<<<<< @@ -5168,7 +5087,7 @@ static PyObject *__pyx_pw_5_cdec_2NT_3ref_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":10 +/* "/home/cdyer/cdec/python/src/grammar.pxi":10 * cdef class NT: * cdef public bytes cat * cdef public unsigned ref # <<<<<<<<<<<<<< @@ -5239,11 +5158,11 @@ static int __pyx_pf_5_cdec_2NT_3ref_2__set__(struct __pyx_obj_5_cdec_NT *__pyx_v static int __pyx_pw_5_cdec_5NTRef_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5_cdec_5NTRef_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { unsigned int __pyx_v_ref; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__ref,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__ref,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -5256,7 +5175,8 @@ static int __pyx_pw_5_cdec_5NTRef_1__init__(PyObject *__pyx_v_self, PyObject *__ kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__ref)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__ref); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -5282,7 +5202,7 @@ static int __pyx_pw_5_cdec_5NTRef_1__init__(PyObject *__pyx_v_self, PyObject *__ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":22 +/* "/home/cdyer/cdec/python/src/grammar.pxi":22 * cdef class NTRef: * cdef public unsigned ref * def __init__(self, unsigned ref): # <<<<<<<<<<<<<< @@ -5295,7 +5215,7 @@ static int __pyx_pf_5_cdec_5NTRef___init__(struct __pyx_obj_5_cdec_NTRef *__pyx_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":23 + /* "/home/cdyer/cdec/python/src/grammar.pxi":23 * cdef public unsigned ref * def __init__(self, unsigned ref): * self.ref = ref # <<<<<<<<<<<<<< @@ -5320,7 +5240,7 @@ static PyObject *__pyx_pw_5_cdec_5NTRef_3__str__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":25 +/* "/home/cdyer/cdec/python/src/grammar.pxi":25 * self.ref = ref * * def __str__(self): # <<<<<<<<<<<<<< @@ -5338,7 +5258,7 @@ static PyObject *__pyx_pf_5_cdec_5NTRef_2__str__(struct __pyx_obj_5_cdec_NTRef * int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":26 + /* "/home/cdyer/cdec/python/src/grammar.pxi":26 * * def __str__(self): * return '[%d]' % self.ref # <<<<<<<<<<<<<< @@ -5379,7 +5299,7 @@ static PyObject *__pyx_pw_5_cdec_5NTRef_3ref_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":21 +/* "/home/cdyer/cdec/python/src/grammar.pxi":21 * * cdef class NTRef: * cdef public unsigned ref # <<<<<<<<<<<<<< @@ -5446,7 +5366,7 @@ static int __pyx_pf_5_cdec_5NTRef_3ref_2__set__(struct __pyx_obj_5_cdec_NTRef *_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":28 +/* "/home/cdyer/cdec/python/src/grammar.pxi":28 * return '[%d]' % self.ref * * cdef TRule convert_rule(_sa.Rule rule): # <<<<<<<<<<<<<< @@ -5482,7 +5402,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o int __pyx_clineno = 0; __Pyx_RefNannySetupContext("convert_rule", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":30 + /* "/home/cdyer/cdec/python/src/grammar.pxi":30 * cdef TRule convert_rule(_sa.Rule rule): * cdef unsigned i * cdef lhs = _sa.sym_tocat(rule.lhs) # <<<<<<<<<<<<<< @@ -5494,7 +5414,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_v_lhs = ((PyObject *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":31 + /* "/home/cdyer/cdec/python/src/grammar.pxi":31 * cdef unsigned i * cdef lhs = _sa.sym_tocat(rule.lhs) * cdef scores = {} # <<<<<<<<<<<<<< @@ -5506,7 +5426,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_v_scores = ((PyObject *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":32 + /* "/home/cdyer/cdec/python/src/grammar.pxi":32 * cdef lhs = _sa.sym_tocat(rule.lhs) * cdef scores = {} * for i in range(rule.n_scores): # <<<<<<<<<<<<<< @@ -5517,7 +5437,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":33 + /* "/home/cdyer/cdec/python/src/grammar.pxi":33 * cdef scores = {} * for i in range(rule.n_scores): * scores['PhraseModel_'+str(i)] = rule.cscores[i] # <<<<<<<<<<<<<< @@ -5544,7 +5464,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":34 + /* "/home/cdyer/cdec/python/src/grammar.pxi":34 * for i in range(rule.n_scores): * scores['PhraseModel_'+str(i)] = rule.cscores[i] * f, e = [], [] # <<<<<<<<<<<<<< @@ -5560,7 +5480,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_v_e = __pyx_t_5; __pyx_t_5 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":35 + /* "/home/cdyer/cdec/python/src/grammar.pxi":35 * scores['PhraseModel_'+str(i)] = rule.cscores[i] * f, e = [], [] * cdef int* fsyms = rule.f.syms # <<<<<<<<<<<<<< @@ -5569,7 +5489,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o */ __pyx_v_fsyms = __pyx_v_rule->f->syms; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":36 + /* "/home/cdyer/cdec/python/src/grammar.pxi":36 * f, e = [], [] * cdef int* fsyms = rule.f.syms * for i in range(rule.f.n): # <<<<<<<<<<<<<< @@ -5580,7 +5500,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":37 + /* "/home/cdyer/cdec/python/src/grammar.pxi":37 * cdef int* fsyms = rule.f.syms * for i in range(rule.f.n): * if _sa.sym_isvar(fsyms[i]): # <<<<<<<<<<<<<< @@ -5590,7 +5510,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_t_6 = __pyx_f_4cdec_2sa_3_sa_sym_isvar((__pyx_v_fsyms[__pyx_v_i])); if (__pyx_t_6) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":38 + /* "/home/cdyer/cdec/python/src/grammar.pxi":38 * for i in range(rule.f.n): * if _sa.sym_isvar(fsyms[i]): * f.append(NT(_sa.sym_tocat(fsyms[i]))) # <<<<<<<<<<<<<< @@ -5613,7 +5533,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":40 + /* "/home/cdyer/cdec/python/src/grammar.pxi":40 * f.append(NT(_sa.sym_tocat(fsyms[i]))) * else: * f.append(_sa.sym_tostring(fsyms[i])) # <<<<<<<<<<<<<< @@ -5628,7 +5548,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_L7:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":41 + /* "/home/cdyer/cdec/python/src/grammar.pxi":41 * else: * f.append(_sa.sym_tostring(fsyms[i])) * cdef int* esyms = rule.e.syms # <<<<<<<<<<<<<< @@ -5637,7 +5557,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o */ __pyx_v_esyms = __pyx_v_rule->e->syms; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":42 + /* "/home/cdyer/cdec/python/src/grammar.pxi":42 * f.append(_sa.sym_tostring(fsyms[i])) * cdef int* esyms = rule.e.syms * for i in range(rule.e.n): # <<<<<<<<<<<<<< @@ -5648,7 +5568,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":43 + /* "/home/cdyer/cdec/python/src/grammar.pxi":43 * cdef int* esyms = rule.e.syms * for i in range(rule.e.n): * if _sa.sym_isvar(esyms[i]): # <<<<<<<<<<<<<< @@ -5658,7 +5578,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_t_6 = __pyx_f_4cdec_2sa_3_sa_sym_isvar((__pyx_v_esyms[__pyx_v_i])); if (__pyx_t_6) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":44 + /* "/home/cdyer/cdec/python/src/grammar.pxi":44 * for i in range(rule.e.n): * if _sa.sym_isvar(esyms[i]): * e.append(NTRef(_sa.sym_getindex(esyms[i]))) # <<<<<<<<<<<<<< @@ -5681,7 +5601,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":46 + /* "/home/cdyer/cdec/python/src/grammar.pxi":46 * e.append(NTRef(_sa.sym_getindex(esyms[i]))) * else: * e.append(_sa.sym_tostring(esyms[i])) # <<<<<<<<<<<<<< @@ -5696,7 +5616,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_L10:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":47 + /* "/home/cdyer/cdec/python/src/grammar.pxi":47 * else: * e.append(_sa.sym_tostring(esyms[i])) * cdef a = [(point/65536, point%65536) for point in rule.word_alignments] # <<<<<<<<<<<<<< @@ -5716,18 +5636,10 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o for (;;) { if (!__pyx_t_9 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_8); __Pyx_INCREF(__pyx_t_4); __pyx_t_8++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_9 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_8); __Pyx_INCREF(__pyx_t_4); __pyx_t_8++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_4 = __pyx_t_9(__pyx_t_1); if (unlikely(!__pyx_t_4)) { @@ -5754,7 +5666,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __Pyx_GIVEREF(__pyx_t_10); __pyx_t_4 = 0; __pyx_t_10 = 0; - if (unlikely(__Pyx_PyList_Append(__pyx_t_5, (PyObject*)__pyx_t_11))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(PyList_Append(__pyx_t_5, (PyObject*)__pyx_t_11))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_11)); __pyx_t_11 = 0; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -5762,7 +5674,7 @@ static struct __pyx_obj_5_cdec_TRule *__pyx_f_5_cdec_convert_rule(struct __pyx_o __pyx_v_a = ((PyObject *)__pyx_t_5); __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":48 + /* "/home/cdyer/cdec/python/src/grammar.pxi":48 * e.append(_sa.sym_tostring(esyms[i])) * cdef a = [(point/65536, point%65536) for point in rule.word_alignments] * return TRule(lhs, f, e, scores, a) # <<<<<<<<<<<<<< @@ -5824,14 +5736,14 @@ static int __pyx_pw_5_cdec_5TRule_1__init__(PyObject *__pyx_v_self, PyObject *__ PyObject *__pyx_v_e = 0; PyObject *__pyx_v_scores = 0; PyObject *__pyx_v_a = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__lhs,&__pyx_n_s__f,&__pyx_n_s__e,&__pyx_n_s__scores,&__pyx_n_s__a,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__lhs,&__pyx_n_s__f,&__pyx_n_s__e,&__pyx_n_s__scores,&__pyx_n_s__a,0}; PyObject* values[5] = {0,0,0,0,0}; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":53 + /* "/home/cdyer/cdec/python/src/grammar.pxi":53 * cdef shared_ptr[grammar.TRule]* rule * * def __init__(self, lhs, f, e, scores, a=None): # <<<<<<<<<<<<<< @@ -5854,20 +5766,24 @@ static int __pyx_pw_5_cdec_5TRule_1__init__(PyObject *__pyx_v_self, PyObject *__ kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__lhs)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__lhs); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__f)) != 0)) kw_args--; + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__f); + if (likely(values[1])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 4, 5, 1); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__e)) != 0)) kw_args--; + values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__e); + if (likely(values[2])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 4, 5, 2); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 3: - if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores)) != 0)) kw_args--; + values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores); + if (likely(values[3])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 4, 5, 3); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } @@ -5913,24 +5829,22 @@ static int __pyx_pw_5_cdec_5TRule_1__init__(PyObject *__pyx_v_self, PyObject *__ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_v_self, PyObject *__pyx_v_lhs, PyObject *__pyx_v_f, PyObject *__pyx_v_e, PyObject *__pyx_v_scores, PyObject *__pyx_v_a) { int __pyx_r; __Pyx_RefNannyDeclarations - TRule *__pyx_t_1; - int __pyx_t_2; + int __pyx_t_1; int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":54 + /* "/home/cdyer/cdec/python/src/grammar.pxi":54 * * def __init__(self, lhs, f, e, scores, a=None): * self.rule = new shared_ptr[grammar.TRule](new grammar.TRule()) # <<<<<<<<<<<<<< * self.lhs = lhs * self.e = e */ - try {__pyx_t_1 = new TRule();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_v_self->rule = new boost::shared_ptr(__pyx_t_1); + __pyx_v_self->rule = new boost::shared_ptr(new TRule()); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":55 + /* "/home/cdyer/cdec/python/src/grammar.pxi":55 * def __init__(self, lhs, f, e, scores, a=None): * self.rule = new shared_ptr[grammar.TRule](new grammar.TRule()) * self.lhs = lhs # <<<<<<<<<<<<<< @@ -5939,7 +5853,7 @@ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_ */ if (PyObject_SetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__lhs, __pyx_v_lhs) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":56 + /* "/home/cdyer/cdec/python/src/grammar.pxi":56 * self.rule = new shared_ptr[grammar.TRule](new grammar.TRule()) * self.lhs = lhs * self.e = e # <<<<<<<<<<<<<< @@ -5948,7 +5862,7 @@ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_ */ if (PyObject_SetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__e, __pyx_v_e) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":57 + /* "/home/cdyer/cdec/python/src/grammar.pxi":57 * self.lhs = lhs * self.e = e * self.f = f # <<<<<<<<<<<<<< @@ -5957,7 +5871,7 @@ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_ */ if (PyObject_SetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__f, __pyx_v_f) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":58 + /* "/home/cdyer/cdec/python/src/grammar.pxi":58 * self.e = e * self.f = f * self.scores = scores # <<<<<<<<<<<<<< @@ -5966,17 +5880,17 @@ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_ */ if (PyObject_SetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__scores, __pyx_v_scores) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":59 + /* "/home/cdyer/cdec/python/src/grammar.pxi":59 * self.f = f * self.scores = scores * if a: # <<<<<<<<<<<<<< * self.a = a * self.rule.get().ComputeArity() */ - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_a); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_a); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":60 + /* "/home/cdyer/cdec/python/src/grammar.pxi":60 * self.scores = scores * if a: * self.a = a # <<<<<<<<<<<<<< @@ -5988,7 +5902,7 @@ static int __pyx_pf_5_cdec_5TRule___init__(struct __pyx_obj_5_cdec_TRule *__pyx_ } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":61 + /* "/home/cdyer/cdec/python/src/grammar.pxi":61 * if a: * self.a = a * self.rule.get().ComputeArity() # <<<<<<<<<<<<<< @@ -6016,7 +5930,7 @@ static void __pyx_pw_5_cdec_5TRule_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":63 +/* "/home/cdyer/cdec/python/src/grammar.pxi":63 * self.rule.get().ComputeArity() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -6028,7 +5942,7 @@ static void __pyx_pf_5_cdec_5TRule_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5 __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":64 + /* "/home/cdyer/cdec/python/src/grammar.pxi":64 * * def __dealloc__(self): * del self.rule # <<<<<<<<<<<<<< @@ -6051,7 +5965,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_5arity_1__get__(PyObject *__pyx_v_self) return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":67 +/* "/home/cdyer/cdec/python/src/grammar.pxi":67 * * property arity: * def __get__(self): # <<<<<<<<<<<<<< @@ -6068,7 +5982,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_5arity___get__(struct __pyx_obj_5_cdec_T int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":68 + /* "/home/cdyer/cdec/python/src/grammar.pxi":68 * property arity: * def __get__(self): * return self.rule.get().arity_ # <<<<<<<<<<<<<< @@ -6105,7 +6019,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_1f_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":71 +/* "/home/cdyer/cdec/python/src/grammar.pxi":71 * * property f: * def __get__(self): # <<<<<<<<<<<<<< @@ -6132,7 +6046,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":72 + /* "/home/cdyer/cdec/python/src/grammar.pxi":72 * property f: * def __get__(self): * cdef vector[WordID]* f_ = &self.rule.get().f_ # <<<<<<<<<<<<<< @@ -6141,7 +6055,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_f_ = (&__pyx_v_self->rule->get()->f_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":74 + /* "/home/cdyer/cdec/python/src/grammar.pxi":74 * cdef vector[WordID]* f_ = &self.rule.get().f_ * cdef WordID w * cdef f = [] # <<<<<<<<<<<<<< @@ -6153,7 +6067,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule __pyx_v_f = ((PyObject *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":76 + /* "/home/cdyer/cdec/python/src/grammar.pxi":76 * cdef f = [] * cdef unsigned i * cdef int idx = 0 # <<<<<<<<<<<<<< @@ -6162,7 +6076,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_idx = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":77 + /* "/home/cdyer/cdec/python/src/grammar.pxi":77 * cdef unsigned i * cdef int idx = 0 * for i in range(f_.size()): # <<<<<<<<<<<<<< @@ -6173,7 +6087,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":78 + /* "/home/cdyer/cdec/python/src/grammar.pxi":78 * cdef int idx = 0 * for i in range(f_.size()): * w = f_[0][i] # <<<<<<<<<<<<<< @@ -6182,7 +6096,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_w = ((__pyx_v_f_[0])[__pyx_v_i]); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":79 + /* "/home/cdyer/cdec/python/src/grammar.pxi":79 * for i in range(f_.size()): * w = f_[0][i] * if w < 0: # <<<<<<<<<<<<<< @@ -6192,7 +6106,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule __pyx_t_4 = (__pyx_v_w < 0); if (__pyx_t_4) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":80 + /* "/home/cdyer/cdec/python/src/grammar.pxi":80 * w = f_[0][i] * if w < 0: * idx += 1 # <<<<<<<<<<<<<< @@ -6201,7 +6115,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_idx = (__pyx_v_idx + 1); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":81 + /* "/home/cdyer/cdec/python/src/grammar.pxi":81 * if w < 0: * idx += 1 * f.append(NT(TDConvert(-w), idx)) # <<<<<<<<<<<<<< @@ -6231,7 +6145,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":83 + /* "/home/cdyer/cdec/python/src/grammar.pxi":83 * f.append(NT(TDConvert(-w), idx)) * else: * f.append(unicode(TDConvert(w), encoding='utf8')) # <<<<<<<<<<<<<< @@ -6260,7 +6174,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1f___get__(struct __pyx_obj_5_cdec_TRule __pyx_L5:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":84 + /* "/home/cdyer/cdec/python/src/grammar.pxi":84 * else: * f.append(unicode(TDConvert(w), encoding='utf8')) * return f # <<<<<<<<<<<<<< @@ -6298,7 +6212,7 @@ static int __pyx_pw_5_cdec_5TRule_1f_3__set__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":86 +/* "/home/cdyer/cdec/python/src/grammar.pxi":86 * return f * * def __set__(self, f): # <<<<<<<<<<<<<< @@ -6323,7 +6237,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":87 + /* "/home/cdyer/cdec/python/src/grammar.pxi":87 * * def __set__(self, f): * cdef vector[WordID]* f_ = &self.rule.get().f_ # <<<<<<<<<<<<<< @@ -6332,7 +6246,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p */ __pyx_v_f_ = (&__pyx_v_self->rule->get()->f_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":88 + /* "/home/cdyer/cdec/python/src/grammar.pxi":88 * def __set__(self, f): * cdef vector[WordID]* f_ = &self.rule.get().f_ * f_.resize(len(f)) # <<<<<<<<<<<<<< @@ -6342,7 +6256,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p __pyx_t_1 = PyObject_Length(__pyx_v_f); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_f_->resize(__pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":90 + /* "/home/cdyer/cdec/python/src/grammar.pxi":90 * f_.resize(len(f)) * cdef unsigned i * cdef int idx = 0 # <<<<<<<<<<<<<< @@ -6351,7 +6265,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p */ __pyx_v_idx = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":91 + /* "/home/cdyer/cdec/python/src/grammar.pxi":91 * cdef unsigned i * cdef int idx = 0 * for i in range(len(f)): # <<<<<<<<<<<<<< @@ -6362,7 +6276,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":92 + /* "/home/cdyer/cdec/python/src/grammar.pxi":92 * cdef int idx = 0 * for i in range(len(f)): * if isinstance(f[i], NT): # <<<<<<<<<<<<<< @@ -6378,7 +6292,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_5) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":93 + /* "/home/cdyer/cdec/python/src/grammar.pxi":93 * for i in range(len(f)): * if isinstance(f[i], NT): * f_[0][i] = -TDConvert(f[i].cat) # <<<<<<<<<<<<<< @@ -6397,7 +6311,7 @@ static int __pyx_pf_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_5_cdec_TRule *__p } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":95 + /* "/home/cdyer/cdec/python/src/grammar.pxi":95 * f_[0][i] = -TDConvert(f[i].cat) * else: * f_[0][i] = TDConvert(as_str(f[i])) # <<<<<<<<<<<<<< @@ -6435,7 +6349,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_1e_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":98 +/* "/home/cdyer/cdec/python/src/grammar.pxi":98 * * property e: * def __get__(self): # <<<<<<<<<<<<<< @@ -6462,7 +6376,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":99 + /* "/home/cdyer/cdec/python/src/grammar.pxi":99 * property e: * def __get__(self): * cdef vector[WordID]* e_ = &self.rule.get().e_ # <<<<<<<<<<<<<< @@ -6471,7 +6385,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_e_ = (&__pyx_v_self->rule->get()->e_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":101 + /* "/home/cdyer/cdec/python/src/grammar.pxi":101 * cdef vector[WordID]* e_ = &self.rule.get().e_ * cdef WordID w * cdef e = [] # <<<<<<<<<<<<<< @@ -6483,7 +6397,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule __pyx_v_e = ((PyObject *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":103 + /* "/home/cdyer/cdec/python/src/grammar.pxi":103 * cdef e = [] * cdef unsigned i * cdef int idx = 0 # <<<<<<<<<<<<<< @@ -6492,7 +6406,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_idx = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":104 + /* "/home/cdyer/cdec/python/src/grammar.pxi":104 * cdef unsigned i * cdef int idx = 0 * for i in range(e_.size()): # <<<<<<<<<<<<<< @@ -6503,7 +6417,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":105 + /* "/home/cdyer/cdec/python/src/grammar.pxi":105 * cdef int idx = 0 * for i in range(e_.size()): * w = e_[0][i] # <<<<<<<<<<<<<< @@ -6512,7 +6426,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_w = ((__pyx_v_e_[0])[__pyx_v_i]); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":106 + /* "/home/cdyer/cdec/python/src/grammar.pxi":106 * for i in range(e_.size()): * w = e_[0][i] * if w < 1: # <<<<<<<<<<<<<< @@ -6522,7 +6436,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule __pyx_t_4 = (__pyx_v_w < 1); if (__pyx_t_4) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":107 + /* "/home/cdyer/cdec/python/src/grammar.pxi":107 * w = e_[0][i] * if w < 1: * idx += 1 # <<<<<<<<<<<<<< @@ -6531,7 +6445,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_idx = (__pyx_v_idx + 1); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":108 + /* "/home/cdyer/cdec/python/src/grammar.pxi":108 * if w < 1: * idx += 1 * e.append(NTRef(1-w)) # <<<<<<<<<<<<<< @@ -6556,7 +6470,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":110 + /* "/home/cdyer/cdec/python/src/grammar.pxi":110 * e.append(NTRef(1-w)) * else: * e.append(unicode(TDConvert(w), encoding='utf8')) # <<<<<<<<<<<<<< @@ -6585,7 +6499,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_1e___get__(struct __pyx_obj_5_cdec_TRule __pyx_L5:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":111 + /* "/home/cdyer/cdec/python/src/grammar.pxi":111 * else: * e.append(unicode(TDConvert(w), encoding='utf8')) * return e # <<<<<<<<<<<<<< @@ -6623,7 +6537,7 @@ static int __pyx_pw_5_cdec_5TRule_1e_3__set__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":113 +/* "/home/cdyer/cdec/python/src/grammar.pxi":113 * return e * * def __set__(self, e): # <<<<<<<<<<<<<< @@ -6647,7 +6561,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":114 + /* "/home/cdyer/cdec/python/src/grammar.pxi":114 * * def __set__(self, e): * cdef vector[WordID]* e_ = &self.rule.get().e_ # <<<<<<<<<<<<<< @@ -6656,7 +6570,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p */ __pyx_v_e_ = (&__pyx_v_self->rule->get()->e_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":115 + /* "/home/cdyer/cdec/python/src/grammar.pxi":115 * def __set__(self, e): * cdef vector[WordID]* e_ = &self.rule.get().e_ * e_.resize(len(e)) # <<<<<<<<<<<<<< @@ -6666,7 +6580,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p __pyx_t_1 = PyObject_Length(__pyx_v_e); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_e_->resize(__pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":117 + /* "/home/cdyer/cdec/python/src/grammar.pxi":117 * e_.resize(len(e)) * cdef unsigned i * for i in range(len(e)): # <<<<<<<<<<<<<< @@ -6677,7 +6591,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":118 + /* "/home/cdyer/cdec/python/src/grammar.pxi":118 * cdef unsigned i * for i in range(len(e)): * if isinstance(e[i], NTRef): # <<<<<<<<<<<<<< @@ -6693,7 +6607,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_5) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":119 + /* "/home/cdyer/cdec/python/src/grammar.pxi":119 * for i in range(len(e)): * if isinstance(e[i], NTRef): * e_[0][i] = 1-e[i].ref # <<<<<<<<<<<<<< @@ -6715,7 +6629,7 @@ static int __pyx_pf_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_5_cdec_TRule *__p } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":121 + /* "/home/cdyer/cdec/python/src/grammar.pxi":121 * e_[0][i] = 1-e[i].ref * else: * e_[0][i] = TDConvert(as_str(e[i])) # <<<<<<<<<<<<<< @@ -6754,7 +6668,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_1a_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":124 +/* "/home/cdyer/cdec/python/src/grammar.pxi":124 * * property a: * def __get__(self): # <<<<<<<<<<<<<< @@ -6819,7 +6733,7 @@ static PyObject *__pyx_gb_5_cdec_5TRule_1a_2generator2(__pyx_GeneratorObject *__ __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":126 + /* "/home/cdyer/cdec/python/src/grammar.pxi":126 * def __get__(self): * cdef unsigned i * cdef vector[grammar.AlignmentPoint]* a = &self.rule.get().a_ # <<<<<<<<<<<<<< @@ -6828,7 +6742,7 @@ static PyObject *__pyx_gb_5_cdec_5TRule_1a_2generator2(__pyx_GeneratorObject *__ */ __pyx_cur_scope->__pyx_v_a = (&__pyx_cur_scope->__pyx_v_self->rule->get()->a_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":127 + /* "/home/cdyer/cdec/python/src/grammar.pxi":127 * cdef unsigned i * cdef vector[grammar.AlignmentPoint]* a = &self.rule.get().a_ * for i in range(a.size()): # <<<<<<<<<<<<<< @@ -6839,7 +6753,7 @@ static PyObject *__pyx_gb_5_cdec_5TRule_1a_2generator2(__pyx_GeneratorObject *__ for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":128 + /* "/home/cdyer/cdec/python/src/grammar.pxi":128 * cdef vector[grammar.AlignmentPoint]* a = &self.rule.get().a_ * for i in range(a.size()): * yield (a[0][i].s_, a[0][i].t_) # <<<<<<<<<<<<<< @@ -6882,7 +6796,6 @@ static PyObject *__pyx_gb_5_cdec_5TRule_1a_2generator2(__pyx_GeneratorObject *__ __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -6898,7 +6811,7 @@ static int __pyx_pw_5_cdec_5TRule_1a_4__set__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":130 +/* "/home/cdyer/cdec/python/src/grammar.pxi":130 * yield (a[0][i].s_, a[0][i].t_) * * def __set__(self, a): # <<<<<<<<<<<<<< @@ -6927,7 +6840,7 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":131 + /* "/home/cdyer/cdec/python/src/grammar.pxi":131 * * def __set__(self, a): * cdef vector[grammar.AlignmentPoint]* a_ = &self.rule.get().a_ # <<<<<<<<<<<<<< @@ -6936,7 +6849,7 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p */ __pyx_v_a_ = (&__pyx_v_self->rule->get()->a_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":132 + /* "/home/cdyer/cdec/python/src/grammar.pxi":132 * def __set__(self, a): * cdef vector[grammar.AlignmentPoint]* a_ = &self.rule.get().a_ * a_.resize(len(a)) # <<<<<<<<<<<<<< @@ -6946,7 +6859,7 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p __pyx_t_1 = PyObject_Length(__pyx_v_a); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 132; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_a_->resize(__pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":135 + /* "/home/cdyer/cdec/python/src/grammar.pxi":135 * cdef unsigned i * cdef int s, t * for i in range(len(a)): # <<<<<<<<<<<<<< @@ -6957,7 +6870,7 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":136 + /* "/home/cdyer/cdec/python/src/grammar.pxi":136 * cdef int s, t * for i in range(len(a)): * s, t = a[i] # <<<<<<<<<<<<<< @@ -6968,33 +6881,27 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p __Pyx_GOTREF(__pyx_t_3); if ((likely(PyTuple_CheckExact(__pyx_t_3))) || (PyList_CheckExact(__pyx_t_3))) { PyObject* sequence = __pyx_t_3; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_4 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_5 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_4 = PyList_GET_ITEM(sequence, 0); __pyx_t_5 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(__pyx_t_5); - #else - __pyx_t_4 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_5 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_6 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_6); @@ -7005,13 +6912,12 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p index = 1; __pyx_t_5 = __pyx_t_7(__pyx_t_6); if (unlikely(!__pyx_t_5)) goto __pyx_L5_unpacking_failed; __Pyx_GOTREF(__pyx_t_5); if (__Pyx_IternextUnpackEndCheck(__pyx_t_7(__pyx_t_6), 2) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_7 = NULL; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; goto __pyx_L6_unpacking_done; __pyx_L5_unpacking_failed:; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_7 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L6_unpacking_done:; } @@ -7022,7 +6928,7 @@ static int __pyx_pf_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_5_cdec_TRule *__p __pyx_v_s = __pyx_t_8; __pyx_v_t = __pyx_t_9; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":137 + /* "/home/cdyer/cdec/python/src/grammar.pxi":137 * for i in range(len(a)): * s, t = a[i] * a_[0][i] = grammar.AlignmentPoint(s, t) # <<<<<<<<<<<<<< @@ -7057,7 +6963,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_6scores_1__get__(PyObject *__pyx_v_self) return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":140 +/* "/home/cdyer/cdec/python/src/grammar.pxi":140 * * property scores: * def __get__(self): # <<<<<<<<<<<<<< @@ -7075,7 +6981,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_6scores___get__(struct __pyx_obj_5_cdec_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":141 + /* "/home/cdyer/cdec/python/src/grammar.pxi":141 * property scores: * def __get__(self): * cdef SparseVector scores = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -7088,7 +6994,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_6scores___get__(struct __pyx_obj_5_cdec_ __pyx_v_scores = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":142 + /* "/home/cdyer/cdec/python/src/grammar.pxi":142 * def __get__(self): * cdef SparseVector scores = SparseVector.__new__(SparseVector) * scores.vector = new FastSparseVector[double](self.rule.get().scores_) # <<<<<<<<<<<<<< @@ -7097,7 +7003,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_6scores___get__(struct __pyx_obj_5_cdec_ */ __pyx_v_scores->vector = new FastSparseVector(__pyx_v_self->rule->get()->scores_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":143 + /* "/home/cdyer/cdec/python/src/grammar.pxi":143 * cdef SparseVector scores = SparseVector.__new__(SparseVector) * scores.vector = new FastSparseVector[double](self.rule.get().scores_) * return scores # <<<<<<<<<<<<<< @@ -7133,7 +7039,7 @@ static int __pyx_pw_5_cdec_5TRule_6scores_3__set__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":145 +/* "/home/cdyer/cdec/python/src/grammar.pxi":145 * return scores * * def __set__(self, scores): # <<<<<<<<<<<<<< @@ -7163,7 +7069,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":146 + /* "/home/cdyer/cdec/python/src/grammar.pxi":146 * * def __set__(self, scores): * cdef FastSparseVector[double]* scores_ = &self.rule.get().scores_ # <<<<<<<<<<<<<< @@ -7172,7 +7078,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_scores_ = (&__pyx_v_self->rule->get()->scores_); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":147 + /* "/home/cdyer/cdec/python/src/grammar.pxi":147 * def __set__(self, scores): * cdef FastSparseVector[double]* scores_ = &self.rule.get().scores_ * scores_.clear() # <<<<<<<<<<<<<< @@ -7181,7 +7087,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_scores_->clear(); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":150 + /* "/home/cdyer/cdec/python/src/grammar.pxi":150 * cdef int fid * cdef float fval * for fname, fval in scores.items(): # <<<<<<<<<<<<<< @@ -7205,18 +7111,10 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule for (;;) { if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_2 = __pyx_t_4(__pyx_t_1); if (unlikely(!__pyx_t_2)) { @@ -7230,33 +7128,27 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule } if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { PyObject* sequence = __pyx_t_2; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyList_GET_ITEM(sequence, 0); __pyx_t_6 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_6); - #else - __pyx_t_5 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_6 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); @@ -7267,13 +7159,12 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule index = 1; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L5_unpacking_failed; __Pyx_GOTREF(__pyx_t_6); if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_8 = NULL; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L6_unpacking_done; __pyx_L5_unpacking_failed:; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L6_unpacking_done:; } @@ -7284,7 +7175,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule __pyx_t_5 = 0; __pyx_v_fval = __pyx_t_9; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":151 + /* "/home/cdyer/cdec/python/src/grammar.pxi":151 * cdef float fval * for fname, fval in scores.items(): * fid = FDConvert(as_str(fname)) # <<<<<<<<<<<<<< @@ -7293,7 +7184,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule */ __pyx_v_fid = FD::Convert(((char *)__pyx_f_5_cdec_as_str(__pyx_v_fname, NULL))); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":152 + /* "/home/cdyer/cdec/python/src/grammar.pxi":152 * for fname, fval in scores.items(): * fid = FDConvert(as_str(fname)) * if fid < 0: raise KeyError(fname) # <<<<<<<<<<<<<< @@ -7317,7 +7208,7 @@ static int __pyx_pf_5_cdec_5TRule_6scores_2__set__(struct __pyx_obj_5_cdec_TRule } __pyx_L7:; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":153 + /* "/home/cdyer/cdec/python/src/grammar.pxi":153 * fid = FDConvert(as_str(fname)) * if fid < 0: raise KeyError(fname) * scores_.set_value(fid, fval) # <<<<<<<<<<<<<< @@ -7355,7 +7246,7 @@ static PyObject *__pyx_pw_5_cdec_5TRule_3lhs_1__get__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":156 +/* "/home/cdyer/cdec/python/src/grammar.pxi":156 * * property lhs: * def __get__(self): # <<<<<<<<<<<<<< @@ -7373,7 +7264,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_3lhs___get__(struct __pyx_obj_5_cdec_TRu int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":157 + /* "/home/cdyer/cdec/python/src/grammar.pxi":157 * property lhs: * def __get__(self): * return NT(TDConvert(-self.rule.get().lhs_)) # <<<<<<<<<<<<<< @@ -7419,7 +7310,7 @@ static int __pyx_pw_5_cdec_5TRule_3lhs_3__set__(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":159 +/* "/home/cdyer/cdec/python/src/grammar.pxi":159 * return NT(TDConvert(-self.rule.get().lhs_)) * * def __set__(self, lhs): # <<<<<<<<<<<<<< @@ -7441,7 +7332,7 @@ static int __pyx_pf_5_cdec_5TRule_3lhs_2__set__(struct __pyx_obj_5_cdec_TRule *_ __Pyx_RefNannySetupContext("__set__", 0); __Pyx_INCREF(__pyx_v_lhs); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":160 + /* "/home/cdyer/cdec/python/src/grammar.pxi":160 * * def __set__(self, lhs): * if not isinstance(lhs, NT): # <<<<<<<<<<<<<< @@ -7455,7 +7346,7 @@ static int __pyx_pf_5_cdec_5TRule_3lhs_2__set__(struct __pyx_obj_5_cdec_TRule *_ __pyx_t_3 = (!__pyx_t_2); if (__pyx_t_3) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":161 + /* "/home/cdyer/cdec/python/src/grammar.pxi":161 * def __set__(self, lhs): * if not isinstance(lhs, NT): * lhs = NT(lhs) # <<<<<<<<<<<<<< @@ -7477,7 +7368,7 @@ static int __pyx_pf_5_cdec_5TRule_3lhs_2__set__(struct __pyx_obj_5_cdec_TRule *_ } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":162 + /* "/home/cdyer/cdec/python/src/grammar.pxi":162 * if not isinstance(lhs, NT): * lhs = NT(lhs) * self.rule.get().lhs_ = -TDConvert(lhs.cat) # <<<<<<<<<<<<<< @@ -7513,9 +7404,9 @@ static PyObject *__pyx_pw_5_cdec_5TRule_5__str__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator18(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator19(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":165 +/* "/home/cdyer/cdec/python/src/grammar.pxi":165 * * def __str__(self): * scores = ' '.join('%s=%s' % feat for feat in self.scores) # <<<<<<<<<<<<<< @@ -7541,7 +7432,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_7__str___genexpr(PyObject *__pyx_self) { __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope)); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_5TRule_7__str___2generator18, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_5TRule_7__str___2generator19, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -7559,7 +7450,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_7__str___genexpr(PyObject *__pyx_self) { return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator18(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator19(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { struct __pyx_obj_5_cdec___pyx_scope_struct_6_genexpr *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_6_genexpr *)__pyx_generator->closure); PyObject *__pyx_r = NULL; @@ -7593,18 +7484,10 @@ static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator18(__pyx_GeneratorObj for (;;) { if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_2)) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_2)) { if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 165; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_1 = __pyx_t_4(__pyx_t_2); if (unlikely(!__pyx_t_1)) { @@ -7652,12 +7535,11 @@ static PyObject *__pyx_gb_5_cdec_5TRule_7__str___2generator18(__pyx_GeneratorObj __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":164 +/* "/home/cdyer/cdec/python/src/grammar.pxi":164 * self.rule.get().lhs_ = -TDConvert(lhs.cat) * * def __str__(self): # <<<<<<<<<<<<<< @@ -7689,7 +7571,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_4__str__(struct __pyx_obj_5_cdec_TRule * __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":165 + /* "/home/cdyer/cdec/python/src/grammar.pxi":165 * * def __str__(self): * scores = ' '.join('%s=%s' % feat for feat in self.scores) # <<<<<<<<<<<<<< @@ -7712,7 +7594,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_4__str__(struct __pyx_obj_5_cdec_TRule * __pyx_v_scores = __pyx_t_2; __pyx_t_2 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":166 + /* "/home/cdyer/cdec/python/src/grammar.pxi":166 * def __str__(self): * scores = ' '.join('%s=%s' % feat for feat in self.scores) * return '%s ||| %s ||| %s ||| %s' % (self.lhs, # <<<<<<<<<<<<<< @@ -7723,7 +7605,7 @@ static PyObject *__pyx_pf_5_cdec_5TRule_4__str__(struct __pyx_obj_5_cdec_TRule * __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_n_s__lhs); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 166; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":167 + /* "/home/cdyer/cdec/python/src/grammar.pxi":167 * scores = ' '.join('%s=%s' % feat for feat in self.scores) * return '%s ||| %s ||| %s ||| %s' % (self.lhs, * _phrase(self.f), _phrase(self.e), scores) # <<<<<<<<<<<<<< @@ -7802,14 +7684,14 @@ static int __pyx_pw_5_cdec_5MRule_1__init__(PyObject *__pyx_v_self, PyObject *__ PyObject *__pyx_v_rhs = 0; PyObject *__pyx_v_scores = 0; PyObject *__pyx_v_a = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__lhs,&__pyx_n_s__rhs,&__pyx_n_s__scores,&__pyx_n_s__a,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__lhs,&__pyx_n_s__rhs,&__pyx_n_s__scores,&__pyx_n_s__a,0}; PyObject* values[4] = {0,0,0,0}; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":170 + /* "/home/cdyer/cdec/python/src/grammar.pxi":170 * * cdef class MRule(TRule): * def __init__(self, lhs, rhs, scores, a=None): # <<<<<<<<<<<<<< @@ -7831,15 +7713,18 @@ static int __pyx_pw_5_cdec_5MRule_1__init__(PyObject *__pyx_v_self, PyObject *__ kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__lhs)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__lhs); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__rhs)) != 0)) kw_args--; + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__rhs); + if (likely(values[1])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 3, 4, 1); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores)) != 0)) kw_args--; + values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__scores); + if (likely(values[2])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 3, 4, 2); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } @@ -7898,7 +7783,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__init__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":171 + /* "/home/cdyer/cdec/python/src/grammar.pxi":171 * cdef class MRule(TRule): * def __init__(self, lhs, rhs, scores, a=None): * cdef unsigned i = 1 # <<<<<<<<<<<<<< @@ -7907,7 +7792,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ */ __pyx_v_i = 1; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":172 + /* "/home/cdyer/cdec/python/src/grammar.pxi":172 * def __init__(self, lhs, rhs, scores, a=None): * cdef unsigned i = 1 * e = [] # <<<<<<<<<<<<<< @@ -7919,7 +7804,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ __pyx_v_e = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":173 + /* "/home/cdyer/cdec/python/src/grammar.pxi":173 * cdef unsigned i = 1 * e = [] * for s in rhs: # <<<<<<<<<<<<<< @@ -7937,18 +7822,10 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ for (;;) { if (!__pyx_t_3 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_3 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_4 = __pyx_t_3(__pyx_t_1); if (unlikely(!__pyx_t_4)) { @@ -7964,7 +7841,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ __pyx_v_s = __pyx_t_4; __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":174 + /* "/home/cdyer/cdec/python/src/grammar.pxi":174 * e = [] * for s in rhs: * if isinstance(s, NT): # <<<<<<<<<<<<<< @@ -7977,7 +7854,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_5) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":175 + /* "/home/cdyer/cdec/python/src/grammar.pxi":175 * for s in rhs: * if isinstance(s, NT): * e.append(NTRef(i)) # <<<<<<<<<<<<<< @@ -7997,7 +7874,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ __pyx_t_7 = PyList_Append(__pyx_v_e, __pyx_t_4); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":176 + /* "/home/cdyer/cdec/python/src/grammar.pxi":176 * if isinstance(s, NT): * e.append(NTRef(i)) * i += 1 # <<<<<<<<<<<<<< @@ -8009,7 +7886,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":178 + /* "/home/cdyer/cdec/python/src/grammar.pxi":178 * i += 1 * else: * e.append(s) # <<<<<<<<<<<<<< @@ -8022,7 +7899,7 @@ static int __pyx_pf_5_cdec_5MRule___init__(struct __pyx_obj_5_cdec_MRule *__pyx_ } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":179 + /* "/home/cdyer/cdec/python/src/grammar.pxi":179 * else: * e.append(s) * super(MRule, self).__init__(lhs, rhs, e, scores, a) # <<<<<<<<<<<<<< @@ -8090,7 +7967,7 @@ static void __pyx_pw_5_cdec_7Grammar_1__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":184 +/* "/home/cdyer/cdec/python/src/grammar.pxi":184 * cdef shared_ptr[grammar.Grammar]* grammar * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -8102,7 +7979,7 @@ static void __pyx_pf_5_cdec_7Grammar___dealloc__(CYTHON_UNUSED struct __pyx_obj_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":185 + /* "/home/cdyer/cdec/python/src/grammar.pxi":185 * * def __dealloc__(self): * del self.grammar # <<<<<<<<<<<<<< @@ -8126,7 +8003,7 @@ static PyObject *__pyx_pw_5_cdec_7Grammar_3__iter__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":187 +/* "/home/cdyer/cdec/python/src/grammar.pxi":187 * del self.grammar * * def __iter__(self): # <<<<<<<<<<<<<< @@ -8189,7 +8066,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":188 + /* "/home/cdyer/cdec/python/src/grammar.pxi":188 * * def __iter__(self): * cdef grammar.const_GrammarIter* root = self.grammar.get().GetRoot() # <<<<<<<<<<<<<< @@ -8198,7 +8075,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p */ __pyx_cur_scope->__pyx_v_root = __pyx_cur_scope->__pyx_v_self->grammar->get()->GetRoot(); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":189 + /* "/home/cdyer/cdec/python/src/grammar.pxi":189 * def __iter__(self): * cdef grammar.const_GrammarIter* root = self.grammar.get().GetRoot() * cdef grammar.const_RuleBin* rbin = root.GetRules() # <<<<<<<<<<<<<< @@ -8207,7 +8084,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p */ __pyx_cur_scope->__pyx_v_rbin = __pyx_cur_scope->__pyx_v_root->GetRules(); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":192 + /* "/home/cdyer/cdec/python/src/grammar.pxi":192 * cdef TRule trule * cdef unsigned i * for i in range(rbin.GetNumRules()): # <<<<<<<<<<<<<< @@ -8218,7 +8095,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":193 + /* "/home/cdyer/cdec/python/src/grammar.pxi":193 * cdef unsigned i * for i in range(rbin.GetNumRules()): * trule = TRule.__new__(TRule) # <<<<<<<<<<<<<< @@ -8234,7 +8111,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p __pyx_cur_scope->__pyx_v_trule = ((struct __pyx_obj_5_cdec_TRule *)__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":194 + /* "/home/cdyer/cdec/python/src/grammar.pxi":194 * for i in range(rbin.GetNumRules()): * trule = TRule.__new__(TRule) * trule.rule = new shared_ptr[grammar.TRule](rbin.GetIthRule(i)) # <<<<<<<<<<<<<< @@ -8243,7 +8120,7 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p */ __pyx_cur_scope->__pyx_v_trule->rule = new boost::shared_ptr(__pyx_cur_scope->__pyx_v_rbin->GetIthRule(__pyx_cur_scope->__pyx_v_i)); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":195 + /* "/home/cdyer/cdec/python/src/grammar.pxi":195 * trule = TRule.__new__(TRule) * trule.rule = new shared_ptr[grammar.TRule](rbin.GetIthRule(i)) * yield trule # <<<<<<<<<<<<<< @@ -8272,7 +8149,6 @@ static PyObject *__pyx_gb_5_cdec_7Grammar_4generator3(__pyx_GeneratorObject *__p __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -8288,7 +8164,7 @@ static PyObject *__pyx_pw_5_cdec_7Grammar_4name_1__get__(PyObject *__pyx_v_self) return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":198 +/* "/home/cdyer/cdec/python/src/grammar.pxi":198 * * property name: * def __get__(self): # <<<<<<<<<<<<<< @@ -8301,7 +8177,7 @@ static PyObject *__pyx_pf_5_cdec_7Grammar_4name___get__(struct __pyx_obj_5_cdec_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":199 + /* "/home/cdyer/cdec/python/src/grammar.pxi":199 * property name: * def __get__(self): * self.grammar.get().GetGrammarName().c_str() # <<<<<<<<<<<<<< @@ -8327,7 +8203,7 @@ static int __pyx_pw_5_cdec_7Grammar_4name_3__set__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":201 +/* "/home/cdyer/cdec/python/src/grammar.pxi":201 * self.grammar.get().GetGrammarName().c_str() * * def __set__(self, name): # <<<<<<<<<<<<<< @@ -8344,7 +8220,7 @@ static int __pyx_pf_5_cdec_7Grammar_4name_2__set__(struct __pyx_obj_5_cdec_Gramm int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":202 + /* "/home/cdyer/cdec/python/src/grammar.pxi":202 * * def __set__(self, name): * self.grammar.get().SetGrammarName(string(name)) # <<<<<<<<<<<<<< @@ -8368,11 +8244,11 @@ static int __pyx_pf_5_cdec_7Grammar_4name_2__set__(struct __pyx_obj_5_cdec_Gramm static int __pyx_pw_5_cdec_11TextGrammar_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5_cdec_11TextGrammar_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_rules = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__rules,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__rules,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -8385,7 +8261,8 @@ static int __pyx_pw_5_cdec_11TextGrammar_1__cinit__(PyObject *__pyx_v_self, PyOb kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__rules)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__rules); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -8411,7 +8288,7 @@ static int __pyx_pw_5_cdec_11TextGrammar_1__cinit__(PyObject *__pyx_v_self, PyOb return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":205 +/* "/home/cdyer/cdec/python/src/grammar.pxi":205 * * cdef class TextGrammar(Grammar): * def __cinit__(self, rules): # <<<<<<<<<<<<<< @@ -8436,7 +8313,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":206 + /* "/home/cdyer/cdec/python/src/grammar.pxi":206 * cdef class TextGrammar(Grammar): * def __cinit__(self, rules): * self.grammar = new shared_ptr[grammar.Grammar](new grammar.TextGrammar()) # <<<<<<<<<<<<<< @@ -8445,7 +8322,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG */ __pyx_v_self->__pyx_base.grammar = new boost::shared_ptr(new TextGrammar()); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":207 + /* "/home/cdyer/cdec/python/src/grammar.pxi":207 * def __cinit__(self, rules): * self.grammar = new shared_ptr[grammar.Grammar](new grammar.TextGrammar()) * cdef grammar.TextGrammar* _g = self.grammar.get() # <<<<<<<<<<<<<< @@ -8454,7 +8331,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG */ __pyx_v__g = ((TextGrammar *)__pyx_v_self->__pyx_base.grammar->get()); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":208 + /* "/home/cdyer/cdec/python/src/grammar.pxi":208 * self.grammar = new shared_ptr[grammar.Grammar](new grammar.TextGrammar()) * cdef grammar.TextGrammar* _g = self.grammar.get() * for trule in rules: # <<<<<<<<<<<<<< @@ -8472,18 +8349,10 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG for (;;) { if (!__pyx_t_3 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 208; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_3 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_4 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_2); __Pyx_INCREF(__pyx_t_4); __pyx_t_2++; - #else - __pyx_t_4 = PySequence_ITEM(__pyx_t_1, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 208; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_4 = __pyx_t_3(__pyx_t_1); if (unlikely(!__pyx_t_4)) { @@ -8499,7 +8368,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG __pyx_v_trule = __pyx_t_4; __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":209 + /* "/home/cdyer/cdec/python/src/grammar.pxi":209 * cdef grammar.TextGrammar* _g = self.grammar.get() * for trule in rules: * if isinstance(trule, _sa.Rule): # <<<<<<<<<<<<<< @@ -8512,7 +8381,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_5) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":210 + /* "/home/cdyer/cdec/python/src/grammar.pxi":210 * for trule in rules: * if isinstance(trule, _sa.Rule): * trule = convert_rule(trule) # <<<<<<<<<<<<<< @@ -8531,7 +8400,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG goto __pyx_L5; } - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":211 + /* "/home/cdyer/cdec/python/src/grammar.pxi":211 * if isinstance(trule, _sa.Rule): * trule = convert_rule(trule) * elif not isinstance(trule, TRule): # <<<<<<<<<<<<<< @@ -8545,7 +8414,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG __pyx_t_7 = (!__pyx_t_5); if (__pyx_t_7) { - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":212 + /* "/home/cdyer/cdec/python/src/grammar.pxi":212 * trule = convert_rule(trule) * elif not isinstance(trule, TRule): * raise ValueError('the grammar should contain TRule objects') # <<<<<<<<<<<<<< @@ -8560,7 +8429,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG } __pyx_L5:; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":213 + /* "/home/cdyer/cdec/python/src/grammar.pxi":213 * elif not isinstance(trule, TRule): * raise ValueError('the grammar should contain TRule objects') * _g.AddRule(( trule).rule[0]) # <<<<<<<<<<<<<< @@ -8592,7 +8461,7 @@ static void __pyx_pw_5_cdec_10Hypergraph_1__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":8 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":8 * cdef MT19937* rng * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -8605,7 +8474,7 @@ static void __pyx_pf_5_cdec_10Hypergraph___dealloc__(struct __pyx_obj_5_cdec_Hyp int __pyx_t_1; __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":9 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":9 * * def __dealloc__(self): * del self.hg # <<<<<<<<<<<<<< @@ -8614,7 +8483,7 @@ static void __pyx_pf_5_cdec_10Hypergraph___dealloc__(struct __pyx_obj_5_cdec_Hyp */ delete __pyx_v_self->hg; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":10 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":10 * def __dealloc__(self): * del self.hg * if self.rng != NULL: # <<<<<<<<<<<<<< @@ -8624,7 +8493,7 @@ static void __pyx_pf_5_cdec_10Hypergraph___dealloc__(struct __pyx_obj_5_cdec_Hyp __pyx_t_1 = (__pyx_v_self->rng != NULL); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":11 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":11 * del self.hg * if self.rng != NULL: * del self.rng # <<<<<<<<<<<<<< @@ -8650,7 +8519,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_3viterbi(PyObject *__pyx_v_self, C return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":13 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":13 * del self.rng * * def viterbi(self): # <<<<<<<<<<<<<< @@ -8669,7 +8538,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_2viterbi(struct __pyx_obj_5_cdec_H int __pyx_clineno = 0; __Pyx_RefNannySetupContext("viterbi", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":15 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":15 * def viterbi(self): * cdef vector[WordID] trans * hypergraph.ViterbiESentence(self.hg[0], &trans) # <<<<<<<<<<<<<< @@ -8678,7 +8547,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_2viterbi(struct __pyx_obj_5_cdec_H */ ViterbiESentence((__pyx_v_self->hg[0]), (&__pyx_v_trans)); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":16 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":16 * cdef vector[WordID] trans * hypergraph.ViterbiESentence(self.hg[0], &trans) * return unicode(GetString(trans).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -8727,7 +8596,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5viterbi_trees(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":18 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":18 * return unicode(GetString(trans).c_str(), 'utf8') * * def viterbi_trees(self): # <<<<<<<<<<<<<< @@ -8747,7 +8616,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4viterbi_trees(struct __pyx_obj_5_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("viterbi_trees", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":19 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":19 * * def viterbi_trees(self): * f_tree = unicode(hypergraph.ViterbiFTree(self.hg[0]).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -8770,7 +8639,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4viterbi_trees(struct __pyx_obj_5_ __pyx_v_f_tree = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":20 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":20 * def viterbi_trees(self): * f_tree = unicode(hypergraph.ViterbiFTree(self.hg[0]).c_str(), 'utf8') * e_tree = unicode(hypergraph.ViterbiETree(self.hg[0]).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -8793,7 +8662,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4viterbi_trees(struct __pyx_obj_5_ __pyx_v_e_tree = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":21 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":21 * f_tree = unicode(hypergraph.ViterbiFTree(self.hg[0]).c_str(), 'utf8') * e_tree = unicode(hypergraph.ViterbiETree(self.hg[0]).c_str(), 'utf8') * return (f_tree, e_tree) # <<<<<<<<<<<<<< @@ -8839,7 +8708,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_7viterbi_features(PyObject *__pyx_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":23 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":23 * return (f_tree, e_tree) * * def viterbi_features(self): # <<<<<<<<<<<<<< @@ -8857,7 +8726,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6viterbi_features(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("viterbi_features", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":24 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":24 * * def viterbi_features(self): * cdef SparseVector fmap = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -8870,7 +8739,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6viterbi_features(struct __pyx_obj __pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":25 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":25 * def viterbi_features(self): * cdef SparseVector fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](hypergraph.ViterbiFeatures(self.hg[0])) # <<<<<<<<<<<<<< @@ -8879,7 +8748,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6viterbi_features(struct __pyx_obj */ __pyx_v_fmap->vector = new FastSparseVector(ViterbiFeatures((__pyx_v_self->hg[0]))); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":26 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":26 * cdef SparseVector fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](hypergraph.ViterbiFeatures(self.hg[0])) * return fmap # <<<<<<<<<<<<<< @@ -8915,7 +8784,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_9viterbi_joshua(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":28 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":28 * return fmap * * def viterbi_joshua(self): # <<<<<<<<<<<<<< @@ -8933,7 +8802,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_8viterbi_joshua(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("viterbi_joshua", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":29 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":29 * * def viterbi_joshua(self): * return unicode(hypergraph.JoshuaVisualizationString(self.hg[0]).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -8983,7 +8852,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_11kbest(PyObject *__pyx_v_self, Py return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":31 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":31 * return unicode(hypergraph.JoshuaVisualizationString(self.hg[0]).c_str(), 'utf8') * * def kbest(self, size): # <<<<<<<<<<<<<< @@ -9051,7 +8920,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":32 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":32 * * def kbest(self, size): * cdef kbest.KBestDerivations[vector[WordID], kbest.ESentenceTraversal]* derivations = new kbest.KBestDerivations[vector[WordID], kbest.ESentenceTraversal](self.hg[0], size) # <<<<<<<<<<<<<< @@ -9061,7 +8930,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject __pyx_t_1 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_1 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_cur_scope->__pyx_v_derivations = new KBest::KBestDerivations,ESentenceTraversal>((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":35 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":35 * cdef kbest.KBestDerivations[vector[WordID], kbest.ESentenceTraversal].Derivation* derivation * cdef unsigned k * try: # <<<<<<<<<<<<<< @@ -9070,7 +8939,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject */ /*try:*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":36 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":36 * cdef unsigned k * try: * for k in range(size): # <<<<<<<<<<<<<< @@ -9081,7 +8950,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_2; __pyx_t_1+=1) { __pyx_cur_scope->__pyx_v_k = __pyx_t_1; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":37 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":37 * try: * for k in range(size): * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) # <<<<<<<<<<<<<< @@ -9090,7 +8959,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_derivation = __pyx_cur_scope->__pyx_v_derivations->LazyKthBest((__pyx_cur_scope->__pyx_v_self->hg->nodes_.size() - 1), __pyx_cur_scope->__pyx_v_k); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":38 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":38 * for k in range(size): * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not derivation: break # <<<<<<<<<<<<<< @@ -9104,7 +8973,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject } __pyx_L9:; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":39 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":39 * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not derivation: break * yield unicode(GetString(derivation._yield).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -9141,7 +9010,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject __pyx_L8_break:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":41 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":41 * yield unicode(GetString(derivation._yield).c_str(), 'utf8') * finally: * del derivations # <<<<<<<<<<<<<< @@ -9184,7 +9053,6 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_12generator4(__pyx_GeneratorObject __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -9201,7 +9069,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_14kbest_trees(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":43 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":43 * del derivations * * def kbest_trees(self, size): # <<<<<<<<<<<<<< @@ -9271,7 +9139,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":44 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":44 * * def kbest_trees(self, size): * cdef kbest.KBestDerivations[vector[WordID], kbest.FTreeTraversal]* f_derivations = new kbest.KBestDerivations[vector[WordID], kbest.FTreeTraversal](self.hg[0], size) # <<<<<<<<<<<<<< @@ -9281,7 +9149,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_t_1 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_1 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_cur_scope->__pyx_v_f_derivations = new KBest::KBestDerivations,FTreeTraversal>((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":46 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":46 * cdef kbest.KBestDerivations[vector[WordID], kbest.FTreeTraversal]* f_derivations = new kbest.KBestDerivations[vector[WordID], kbest.FTreeTraversal](self.hg[0], size) * cdef kbest.KBestDerivations[vector[WordID], kbest.FTreeTraversal].Derivation* f_derivation * cdef kbest.KBestDerivations[vector[WordID], kbest.ETreeTraversal]* e_derivations = new kbest.KBestDerivations[vector[WordID], kbest.ETreeTraversal](self.hg[0], size) # <<<<<<<<<<<<<< @@ -9291,7 +9159,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_t_1 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_1 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_cur_scope->__pyx_v_e_derivations = new KBest::KBestDerivations,ETreeTraversal>((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":49 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":49 * cdef kbest.KBestDerivations[vector[WordID], kbest.ETreeTraversal].Derivation* e_derivation * cdef unsigned k * try: # <<<<<<<<<<<<<< @@ -9300,7 +9168,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject */ /*try:*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":50 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":50 * cdef unsigned k * try: * for k in range(size): # <<<<<<<<<<<<<< @@ -9311,7 +9179,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_2; __pyx_t_1+=1) { __pyx_cur_scope->__pyx_v_k = __pyx_t_1; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":51 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":51 * try: * for k in range(size): * f_derivation = f_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) # <<<<<<<<<<<<<< @@ -9320,7 +9188,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_f_derivation = __pyx_cur_scope->__pyx_v_f_derivations->LazyKthBest((__pyx_cur_scope->__pyx_v_self->hg->nodes_.size() - 1), __pyx_cur_scope->__pyx_v_k); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":52 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":52 * for k in range(size): * f_derivation = f_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * e_derivation = e_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) # <<<<<<<<<<<<<< @@ -9329,7 +9197,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_e_derivation = __pyx_cur_scope->__pyx_v_e_derivations->LazyKthBest((__pyx_cur_scope->__pyx_v_self->hg->nodes_.size() - 1), __pyx_cur_scope->__pyx_v_k); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":53 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":53 * f_derivation = f_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * e_derivation = e_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not f_derivation or not e_derivation: break # <<<<<<<<<<<<<< @@ -9349,7 +9217,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject } __pyx_L9:; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":54 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":54 * e_derivation = e_derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not f_derivation or not e_derivation: break * f_tree = unicode(GetString(f_derivation._yield).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -9375,7 +9243,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_cur_scope->__pyx_v_f_tree = ((PyObject*)__pyx_t_6); __pyx_t_6 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":55 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":55 * if not f_derivation or not e_derivation: break * f_tree = unicode(GetString(f_derivation._yield).c_str(), 'utf8') * e_tree = unicode(GetString(e_derivation._yield).c_str(), 'utf8') # <<<<<<<<<<<<<< @@ -9401,7 +9269,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_cur_scope->__pyx_v_e_tree = ((PyObject*)__pyx_t_6); __pyx_t_6 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":56 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":56 * f_tree = unicode(GetString(f_derivation._yield).c_str(), 'utf8') * e_tree = unicode(GetString(e_derivation._yield).c_str(), 'utf8') * yield (f_tree, e_tree) # <<<<<<<<<<<<<< @@ -9433,7 +9301,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_L8_break:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":58 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":58 * yield (f_tree, e_tree) * finally: * del f_derivations # <<<<<<<<<<<<<< @@ -9457,7 +9325,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_L6:; delete __pyx_cur_scope->__pyx_v_f_derivations; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":59 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":59 * finally: * del f_derivations * del e_derivations # <<<<<<<<<<<<<< @@ -9485,7 +9353,6 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator5(__pyx_GeneratorObject __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -9502,7 +9369,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_17kbest_features(PyObject *__pyx_v return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":61 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":61 * del e_derivations * * def kbest_features(self, size): # <<<<<<<<<<<<<< @@ -9569,7 +9436,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":62 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":62 * * def kbest_features(self, size): * cdef kbest.KBestDerivations[FastSparseVector[weight_t], kbest.FeatureVectorTraversal]* derivations = new kbest.KBestDerivations[FastSparseVector[weight_t], kbest.FeatureVectorTraversal](self.hg[0], size) # <<<<<<<<<<<<<< @@ -9579,7 +9446,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject __pyx_t_1 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_1 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_cur_scope->__pyx_v_derivations = new KBest::KBestDerivations,FeatureVectorTraversal>((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_t_1); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":66 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":66 * cdef SparseVector fmap * cdef unsigned k * try: # <<<<<<<<<<<<<< @@ -9588,7 +9455,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject */ /*try:*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":67 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":67 * cdef unsigned k * try: * for k in range(size): # <<<<<<<<<<<<<< @@ -9599,7 +9466,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_2; __pyx_t_1+=1) { __pyx_cur_scope->__pyx_v_k = __pyx_t_1; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":68 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":68 * try: * for k in range(size): * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) # <<<<<<<<<<<<<< @@ -9608,7 +9475,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_derivation = __pyx_cur_scope->__pyx_v_derivations->LazyKthBest((__pyx_cur_scope->__pyx_v_self->hg->nodes_.size() - 1), __pyx_cur_scope->__pyx_v_k); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":69 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":69 * for k in range(size): * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not derivation: break # <<<<<<<<<<<<<< @@ -9622,7 +9489,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject } __pyx_L9:; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":70 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":70 * derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k) * if not derivation: break * fmap = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -9638,7 +9505,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject __pyx_cur_scope->__pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_4); __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":71 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":71 * if not derivation: break * fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](derivation._yield) # <<<<<<<<<<<<<< @@ -9647,7 +9514,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_fmap->vector = new FastSparseVector(__pyx_cur_scope->__pyx_v_derivation->yield); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":72 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":72 * fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](derivation._yield) * yield fmap # <<<<<<<<<<<<<< @@ -9671,7 +9538,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject __pyx_L8_break:; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":74 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":74 * yield fmap * finally: * del derivations # <<<<<<<<<<<<<< @@ -9712,7 +9579,6 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator6(__pyx_GeneratorObject __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -9739,7 +9605,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_20sample(PyObject *__pyx_v_self, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":76 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":76 * del derivations * * def sample(self, unsigned n): # <<<<<<<<<<<<<< @@ -9789,11 +9655,10 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject struct __pyx_obj_5_cdec___pyx_scope_struct_11_sample *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_11_sample *)__pyx_generator->closure); PyObject *__pyx_r = NULL; int __pyx_t_1; - MT19937 *__pyx_t_2; - size_t __pyx_t_3; - unsigned int __pyx_t_4; + size_t __pyx_t_2; + unsigned int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("None", 0); switch (__pyx_generator->resume_label) { @@ -9806,7 +9671,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":77 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":77 * * def sample(self, unsigned n): * cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]() # <<<<<<<<<<<<<< @@ -9815,7 +9680,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject */ __pyx_cur_scope->__pyx_v_hypos = new std::vector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":78 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":78 * def sample(self, unsigned n): * cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]() * if self.rng == NULL: # <<<<<<<<<<<<<< @@ -9825,20 +9690,19 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject __pyx_t_1 = (__pyx_cur_scope->__pyx_v_self->rng == NULL); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":79 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":79 * cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]() * if self.rng == NULL: * self.rng = new MT19937() # <<<<<<<<<<<<<< * hypergraph.sample_hypotheses(self.hg[0], n, self.rng, hypos) * cdef unsigned k */ - try {__pyx_t_2 = new MT19937();} catch(...) {__Pyx_CppExn2PyErr(); {__pyx_filename = __pyx_f[3]; __pyx_lineno = 79; __pyx_clineno = __LINE__; goto __pyx_L1_error;}} - __pyx_cur_scope->__pyx_v_self->rng = __pyx_t_2; + __pyx_cur_scope->__pyx_v_self->rng = new MT19937(); goto __pyx_L4; } __pyx_L4:; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":80 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":80 * if self.rng == NULL: * self.rng = new MT19937() * hypergraph.sample_hypotheses(self.hg[0], n, self.rng, hypos) # <<<<<<<<<<<<<< @@ -9847,7 +9711,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject */ HypergraphSampler::sample_hypotheses((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_cur_scope->__pyx_v_n, __pyx_cur_scope->__pyx_v_self->rng, __pyx_cur_scope->__pyx_v_hypos); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":82 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":82 * hypergraph.sample_hypotheses(self.hg[0], n, self.rng, hypos) * cdef unsigned k * try: # <<<<<<<<<<<<<< @@ -9856,59 +9720,59 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject */ /*try:*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":83 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":83 * cdef unsigned k * try: * for k in range(hypos.size()): # <<<<<<<<<<<<<< * yield unicode(GetString(hypos[0][k].words).c_str(), 'utf8') * finally: */ - __pyx_t_3 = __pyx_cur_scope->__pyx_v_hypos->size(); - for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { - __pyx_cur_scope->__pyx_v_k = __pyx_t_4; + __pyx_t_2 = __pyx_cur_scope->__pyx_v_hypos->size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_cur_scope->__pyx_v_k = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":84 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":84 * try: * for k in range(hypos.size()): * yield unicode(GetString(hypos[0][k].words).c_str(), 'utf8') # <<<<<<<<<<<<<< * finally: * del hypos */ - __pyx_t_5 = PyBytes_FromString(TD::GetString(((__pyx_cur_scope->__pyx_v_hypos[0])[__pyx_cur_scope->__pyx_v_k]).words).c_str()); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} - __Pyx_GOTREF(((PyObject *)__pyx_t_5)); - __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} - __Pyx_GOTREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_6, 0, ((PyObject *)__pyx_t_5)); - __Pyx_GIVEREF(((PyObject *)__pyx_t_5)); + __pyx_t_4 = PyBytes_FromString(TD::GetString(((__pyx_cur_scope->__pyx_v_hypos[0])[__pyx_cur_scope->__pyx_v_k]).words).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_5, 0, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_4)); __Pyx_INCREF(((PyObject *)__pyx_n_s__utf8)); - PyTuple_SET_ITEM(__pyx_t_6, 1, ((PyObject *)__pyx_n_s__utf8)); + PyTuple_SET_ITEM(__pyx_t_5, 1, ((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); - __pyx_t_5 = 0; - __pyx_t_5 = PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0; - __pyx_r = __pyx_t_5; - __pyx_t_5 = 0; - __pyx_cur_scope->__pyx_t_0 = __pyx_t_3; - __pyx_cur_scope->__pyx_t_1 = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); /* return from generator, yielding value */ __pyx_generator->resume_label = 1; return __pyx_r; __pyx_L10_resume_from_yield:; - __pyx_t_3 = __pyx_cur_scope->__pyx_t_0; - __pyx_t_4 = __pyx_cur_scope->__pyx_t_1; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L6;} } } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":86 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":86 * yield unicode(GetString(hypos[0][k].words).c_str(), 'utf8') * finally: * del hypos # <<<<<<<<<<<<<< * - * def intersect(self, Lattice lat): + * def sample_trees(self, unsigned n): */ /*finally:*/ { int __pyx_why; @@ -9918,8 +9782,8 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject __pyx_why = 0; goto __pyx_L7; __pyx_L6: { __pyx_why = 4; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_ErrFetch(&__pyx_exc_type, &__pyx_exc_value, &__pyx_exc_tb); __pyx_exc_lineno = __pyx_lineno; goto __pyx_L7; @@ -9940,25 +9804,255 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator7(__pyx_GeneratorObject PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); __Pyx_XDECREF(__pyx_t_5); - __Pyx_XDECREF(__pyx_t_6); __Pyx_AddTraceback("sample", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); + __Pyx_RefNannyFinishContext(); + return NULL; +} +static PyObject *__pyx_gb_5_cdec_10Hypergraph_24generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_23sample_trees(PyObject *__pyx_v_self, PyObject *__pyx_arg_n); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_23sample_trees(PyObject *__pyx_v_self, PyObject *__pyx_arg_n) { + unsigned int __pyx_v_n; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("sample_trees (wrapper)", 0); + assert(__pyx_arg_n); { + __pyx_v_n = __Pyx_PyInt_AsUnsignedInt(__pyx_arg_n); if (unlikely((__pyx_v_n == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + __Pyx_AddTraceback("_cdec.Hypergraph.sample_trees", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_22sample_trees(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((unsigned int)__pyx_v_n)); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":88 + * del hypos + * + * def sample_trees(self, unsigned n): # <<<<<<<<<<<<<< + * cdef vector[string]* trees = new vector[string]() + * if self.rng == NULL: + */ + +static PyObject *__pyx_pf_5_cdec_10Hypergraph_22sample_trees(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n) { + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *__pyx_cur_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("sample_trees", 0); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)__pyx_ptype_5_cdec___pyx_scope_struct_12_sample_trees->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_12_sample_trees, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __Pyx_RefNannyFinishContext(); + return NULL; + } + __Pyx_GOTREF(__pyx_cur_scope); + __pyx_cur_scope->__pyx_v_self = __pyx_v_self; + __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); + __pyx_cur_scope->__pyx_v_n = __pyx_v_n; + { + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_24generator8, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __Pyx_DECREF(__pyx_cur_scope); + __Pyx_RefNannyFinishContext(); + return (PyObject *) gen; + } + + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("_cdec.Hypergraph.sample_trees", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_gb_5_cdec_10Hypergraph_24generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +{ + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)__pyx_generator->closure); + PyObject *__pyx_r = NULL; + int __pyx_t_1; + size_t __pyx_t_2; + unsigned int __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("None", 0); + switch (__pyx_generator->resume_label) { + case 0: goto __pyx_L3_first_run; + case 1: goto __pyx_L10_resume_from_yield; + default: /* CPython raises the right error here */ + __Pyx_RefNannyFinishContext(); + return NULL; + } + __pyx_L3_first_run:; + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":89 + * + * def sample_trees(self, unsigned n): + * cdef vector[string]* trees = new vector[string]() # <<<<<<<<<<<<<< + * if self.rng == NULL: + * self.rng = new MT19937() + */ + __pyx_cur_scope->__pyx_v_trees = new std::vector(); + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":90 + * def sample_trees(self, unsigned n): + * cdef vector[string]* trees = new vector[string]() + * if self.rng == NULL: # <<<<<<<<<<<<<< + * self.rng = new MT19937() + * hypergraph.sample_trees(self.hg[0], n, self.rng, trees) + */ + __pyx_t_1 = (__pyx_cur_scope->__pyx_v_self->rng == NULL); + if (__pyx_t_1) { + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":91 + * cdef vector[string]* trees = new vector[string]() + * if self.rng == NULL: + * self.rng = new MT19937() # <<<<<<<<<<<<<< + * hypergraph.sample_trees(self.hg[0], n, self.rng, trees) + * cdef unsigned k + */ + __pyx_cur_scope->__pyx_v_self->rng = new MT19937(); + goto __pyx_L4; + } + __pyx_L4:; + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":92 + * if self.rng == NULL: + * self.rng = new MT19937() + * hypergraph.sample_trees(self.hg[0], n, self.rng, trees) # <<<<<<<<<<<<<< + * cdef unsigned k + * try: + */ + HypergraphSampler::sample_trees((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_cur_scope->__pyx_v_n, __pyx_cur_scope->__pyx_v_self->rng, __pyx_cur_scope->__pyx_v_trees); + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":94 + * hypergraph.sample_trees(self.hg[0], n, self.rng, trees) + * cdef unsigned k + * try: # <<<<<<<<<<<<<< + * for k in range(trees.size()): + * yield unicode(trees[0][k].c_str(), 'utf8') + */ + /*try:*/ { + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":95 + * cdef unsigned k + * try: + * for k in range(trees.size()): # <<<<<<<<<<<<<< + * yield unicode(trees[0][k].c_str(), 'utf8') + * finally: + */ + __pyx_t_2 = __pyx_cur_scope->__pyx_v_trees->size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_cur_scope->__pyx_v_k = __pyx_t_3; + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":96 + * try: + * for k in range(trees.size()): + * yield unicode(trees[0][k].c_str(), 'utf8') # <<<<<<<<<<<<<< + * finally: + * del trees + */ + __pyx_t_4 = PyBytes_FromString(((__pyx_cur_scope->__pyx_v_trees[0])[__pyx_cur_scope->__pyx_v_k]).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(((PyObject *)__pyx_t_4)); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_5, 0, ((PyObject *)__pyx_t_4)); + __Pyx_GIVEREF(((PyObject *)__pyx_t_4)); + __Pyx_INCREF(((PyObject *)__pyx_n_s__utf8)); + PyTuple_SET_ITEM(__pyx_t_5, 1, ((PyObject *)__pyx_n_s__utf8)); + __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); + __pyx_t_4 = 0; + __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L6;} + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + __pyx_cur_scope->__pyx_t_0 = __pyx_t_2; + __pyx_cur_scope->__pyx_t_1 = __pyx_t_3; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + /* return from generator, yielding value */ + __pyx_generator->resume_label = 1; + return __pyx_r; + __pyx_L10_resume_from_yield:; + __pyx_t_2 = __pyx_cur_scope->__pyx_t_0; + __pyx_t_3 = __pyx_cur_scope->__pyx_t_1; + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L6;} + } + } + + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":98 + * yield unicode(trees[0][k].c_str(), 'utf8') + * finally: + * del trees # <<<<<<<<<<<<<< + * + * def intersect(self, Lattice lat): + */ + /*finally:*/ { + int __pyx_why; + PyObject *__pyx_exc_type, *__pyx_exc_value, *__pyx_exc_tb; + int __pyx_exc_lineno; + __pyx_exc_type = 0; __pyx_exc_value = 0; __pyx_exc_tb = 0; __pyx_exc_lineno = 0; + __pyx_why = 0; goto __pyx_L7; + __pyx_L6: { + __pyx_why = 4; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_ErrFetch(&__pyx_exc_type, &__pyx_exc_value, &__pyx_exc_tb); + __pyx_exc_lineno = __pyx_lineno; + goto __pyx_L7; + } + __pyx_L7:; + delete __pyx_cur_scope->__pyx_v_trees; + switch (__pyx_why) { + case 4: { + __Pyx_ErrRestore(__pyx_exc_type, __pyx_exc_value, __pyx_exc_tb); + __pyx_lineno = __pyx_exc_lineno; + __pyx_exc_type = 0; + __pyx_exc_value = 0; + __pyx_exc_tb = 0; + goto __pyx_L1_error; + } + } + } + PyErr_SetNone(PyExc_StopIteration); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("sample_trees", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_XDECREF(__pyx_r); + __pyx_generator->resume_label = -1; __Pyx_RefNannyFinishContext(); return NULL; } /* Python wrapper */ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_23intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat); /*proto*/ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_23intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat) { +static PyObject *__pyx_pw_5_cdec_10Hypergraph_26intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_26intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("intersect (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lat), __pyx_ptype_5_cdec_Lattice, 1, "lat", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_r = __pyx_pf_5_cdec_10Hypergraph_22intersect(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((struct __pyx_obj_5_cdec_Lattice *)__pyx_v_lat)); + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lat), __pyx_ptype_5_cdec_Lattice, 1, "lat", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_25intersect(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((struct __pyx_obj_5_cdec_Lattice *)__pyx_v_lat)); goto __pyx_L0; __pyx_L1_error:; __pyx_r = NULL; @@ -9967,15 +10061,15 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_23intersect(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":88 - * del hypos +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":100 + * del trees * * def intersect(self, Lattice lat): # <<<<<<<<<<<<<< * return hypergraph.Intersect(lat.lattice[0], self.hg) * */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat) { +static PyObject *__pyx_pf_5_cdec_10Hypergraph_25intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; @@ -9984,7 +10078,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("intersect", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":89 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":101 * * def intersect(self, Lattice lat): * return hypergraph.Intersect(lat.lattice[0], self.hg) # <<<<<<<<<<<<<< @@ -9992,7 +10086,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cde * def prune(self, beam_alpha=0, density=0, **kwargs): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong(HG::Intersect((__pyx_v_lat->lattice[0]), __pyx_v_self->hg)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong(HG::Intersect((__pyx_v_lat->lattice[0]), __pyx_v_self->hg)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -10011,18 +10105,18 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cde } /* Python wrapper */ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_5_cdec_10Hypergraph_28prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_28prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_beam_alpha = 0; PyObject *__pyx_v_density = 0; PyObject *__pyx_v_kwargs = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__beam_alpha,&__pyx_n_s__density,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("prune (wrapper)", 0); __pyx_v_kwargs = PyDict_New(); if (unlikely(!__pyx_v_kwargs)) return NULL; __Pyx_GOTREF(__pyx_v_kwargs); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__beam_alpha,&__pyx_n_s__density,0}; PyObject* values[2] = {0,0}; values[0] = ((PyObject *)__pyx_int_0); values[1] = ((PyObject *)__pyx_int_0); @@ -10049,7 +10143,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, Py } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, pos_args, "prune") < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, pos_args, "prune") < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -10064,20 +10158,20 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, Py } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("prune", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[3]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("prune", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[3]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; __Pyx_AddTraceback("_cdec.Hypergraph.prune", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5_cdec_10Hypergraph_24prune(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), __pyx_v_beam_alpha, __pyx_v_density, __pyx_v_kwargs); + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_27prune(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), __pyx_v_beam_alpha, __pyx_v_density, __pyx_v_kwargs); __Pyx_XDECREF(__pyx_v_kwargs); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":91 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":103 * return hypergraph.Intersect(lat.lattice[0], self.hg) * * def prune(self, beam_alpha=0, density=0, **kwargs): # <<<<<<<<<<<<<< @@ -10085,7 +10179,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, Py * if 'csplit_preserve_full_word' in kwargs: */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs) { +static PyObject *__pyx_pf_5_cdec_10Hypergraph_27prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs) { std::vector *__pyx_v_preserve_mask; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -10097,7 +10191,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy int __pyx_clineno = 0; __Pyx_RefNannySetupContext("prune", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":92 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":104 * * def prune(self, beam_alpha=0, density=0, **kwargs): * cdef hypergraph.EdgeMask* preserve_mask = NULL # <<<<<<<<<<<<<< @@ -10106,17 +10200,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy */ __pyx_v_preserve_mask = NULL; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":93 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":105 * def prune(self, beam_alpha=0, density=0, **kwargs): * cdef hypergraph.EdgeMask* preserve_mask = NULL * if 'csplit_preserve_full_word' in kwargs: # <<<<<<<<<<<<<< * preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size()) * preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True */ - __pyx_t_1 = ((PyDict_Contains(((PyObject *)__pyx_v_kwargs), ((PyObject *)__pyx_n_s_15)))); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = ((PyDict_Contains(((PyObject *)__pyx_v_kwargs), ((PyObject *)__pyx_n_s_15)))); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":94 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":106 * cdef hypergraph.EdgeMask* preserve_mask = NULL * if 'csplit_preserve_full_word' in kwargs: * preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size()) # <<<<<<<<<<<<<< @@ -10125,7 +10219,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy */ __pyx_v_preserve_mask = new std::vector(__pyx_v_self->hg->edges_.size()); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":95 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":107 * if 'csplit_preserve_full_word' in kwargs: * preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size()) * preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True # <<<<<<<<<<<<<< @@ -10137,18 +10231,18 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":96 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":108 * preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size()) * preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True * self.hg.PruneInsideOutside(beam_alpha, density, preserve_mask, False, 1, False) # <<<<<<<<<<<<<< * if preserve_mask: * del preserve_mask */ - __pyx_t_2 = __pyx_PyFloat_AsDouble(__pyx_v_beam_alpha); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_3 = __pyx_PyFloat_AsDouble(__pyx_v_density); if (unlikely((__pyx_t_3 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __pyx_PyFloat_AsDouble(__pyx_v_beam_alpha); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = __pyx_PyFloat_AsDouble(__pyx_v_density); if (unlikely((__pyx_t_3 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_self->hg->PruneInsideOutside(__pyx_t_2, __pyx_t_3, __pyx_v_preserve_mask, 0, 1.0, 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":97 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":109 * preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True * self.hg.PruneInsideOutside(beam_alpha, density, preserve_mask, False, 1, False) * if preserve_mask: # <<<<<<<<<<<<<< @@ -10158,7 +10252,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy __pyx_t_1 = (__pyx_v_preserve_mask != 0); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":98 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":110 * self.hg.PruneInsideOutside(beam_alpha, density, preserve_mask, False, 1, False) * if preserve_mask: * del preserve_mask # <<<<<<<<<<<<<< @@ -10182,17 +10276,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hy } /* Python wrapper */ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_27lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_27lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5_cdec_10Hypergraph_30lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_30lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("lattice (wrapper)", 0); - __pyx_r = __pyx_pf_5_cdec_10Hypergraph_26lattice(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self)); + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_29lattice(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":100 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":112 * del preserve_mask * * def lattice(self): # TODO direct hg -> lattice conversion in cdec # <<<<<<<<<<<<<< @@ -10200,7 +10294,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_27lattice(PyObject *__pyx_v_self, * return Lattice(eval(plf)) */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { +static PyObject *__pyx_pf_5_cdec_10Hypergraph_29lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { PyObject *__pyx_v_plf = 0; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -10212,20 +10306,20 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("lattice", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":101 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":113 * * def lattice(self): # TODO direct hg -> lattice conversion in cdec * cdef str plf = hypergraph.AsPLF(self.hg[0], True).c_str() # <<<<<<<<<<<<<< * return Lattice(eval(plf)) * */ - __pyx_t_1 = PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->hg[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->hg[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_1)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_1))->tp_name), 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_1)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_1))->tp_name), 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_plf = ((PyObject*)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":102 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":114 * def lattice(self): # TODO direct hg -> lattice conversion in cdec * cdef str plf = hypergraph.AsPLF(self.hg[0], True).c_str() * return Lattice(eval(plf)) # <<<<<<<<<<<<<< @@ -10233,17 +10327,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_ * def reweight(self, weights): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_Globals(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_Globals(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_2)); if (((PyObject *)__pyx_v_plf)) { - if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__plf), ((PyObject *)__pyx_v_plf)) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__plf), ((PyObject *)__pyx_v_plf)) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } if (((PyObject *)__pyx_v_self)) { - if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__self), ((PyObject *)__pyx_v_self)) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__self), ((PyObject *)__pyx_v_self)) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); __Pyx_INCREF(((PyObject *)__pyx_v_plf)); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_plf)); @@ -10254,15 +10348,15 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_ __Pyx_GIVEREF(((PyObject *)__pyx_t_2)); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = PyObject_Call(__pyx_builtin_eval, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_eval, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __Pyx_GIVEREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Lattice)), ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Lattice)), ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; @@ -10285,17 +10379,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_ } /* Python wrapper */ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_29reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights); /*proto*/ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_29reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights) { +static PyObject *__pyx_pw_5_cdec_10Hypergraph_32reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_32reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("reweight (wrapper)", 0); - __pyx_r = __pyx_pf_5_cdec_10Hypergraph_28reweight(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((PyObject *)__pyx_v_weights)); + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_31reweight(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((PyObject *)__pyx_v_weights)); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":104 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":116 * return Lattice(eval(plf)) * * def reweight(self, weights): # <<<<<<<<<<<<<< @@ -10303,7 +10397,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_29reweight(PyObject *__pyx_v_self, * self.hg.Reweight(( weights).vector[0]) */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights) { +static PyObject *__pyx_pf_5_cdec_10Hypergraph_31reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights) { PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; @@ -10314,7 +10408,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec int __pyx_clineno = 0; __Pyx_RefNannySetupContext("reweight", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":105 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":117 * * def reweight(self, weights): * if isinstance(weights, SparseVector): # <<<<<<<<<<<<<< @@ -10327,7 +10421,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":106 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":118 * def reweight(self, weights): * if isinstance(weights, SparseVector): * self.hg.Reweight(( weights).vector[0]) # <<<<<<<<<<<<<< @@ -10338,7 +10432,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec goto __pyx_L3; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":107 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":119 * if isinstance(weights, SparseVector): * self.hg.Reweight(( weights).vector[0]) * elif isinstance(weights, DenseVector): # <<<<<<<<<<<<<< @@ -10351,7 +10445,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":108 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":120 * self.hg.Reweight(( weights).vector[0]) * elif isinstance(weights, DenseVector): * self.hg.Reweight(( weights).vector[0]) # <<<<<<<<<<<<<< @@ -10363,26 +10457,26 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":110 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":122 * self.hg.Reweight(( weights).vector[0]) * else: * raise TypeError('cannot reweight hypergraph with %s' % type(weights)) # <<<<<<<<<<<<<< * * property edges: */ - __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_16), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_16), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_1)); __Pyx_GIVEREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - {__pyx_filename = __pyx_f[3]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; @@ -10398,7 +10492,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator9(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5edges_1__get__(PyObject *__pyx_v_self); /*proto*/ @@ -10411,7 +10505,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5edges_1__get__(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":113 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":125 * * property edges: * def __get__(self): # <<<<<<<<<<<<<< @@ -10420,14 +10514,14 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5edges_1__get__(PyObject *__pyx_v_ */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5edges___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_12___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_12___get__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_13___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_13___get__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -10437,7 +10531,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5edges___get__(struct __pyx_obj_5_ __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_5edges_2generator8, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_5edges_2generator9, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -10455,9 +10549,9 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5edges___get__(struct __pyx_obj_5_ return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator9(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; size_t __pyx_t_1; unsigned int __pyx_t_2; @@ -10473,9 +10567,9 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_Generator return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":115 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":127 * def __get__(self): * cdef unsigned i * for i in range(self.hg.edges_.size()): # <<<<<<<<<<<<<< @@ -10486,16 +10580,16 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_Generator for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":116 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":128 * cdef unsigned i * for i in range(self.hg.edges_.size()): * yield HypergraphEdge().init(self.hg, i) # <<<<<<<<<<<<<< * * property nodes: */ - __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, __pyx_cur_scope->__pyx_v_i); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, __pyx_cur_scope->__pyx_v_i); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -10510,7 +10604,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_Generator __pyx_L6_resume_from_yield:; __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; @@ -10521,11 +10615,10 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5edges_2generator8(__pyx_Generator __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } -static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator10(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5nodes_1__get__(PyObject *__pyx_v_self); /*proto*/ @@ -10538,7 +10631,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5nodes_1__get__(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":119 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":131 * * property nodes: * def __get__(self): # <<<<<<<<<<<<<< @@ -10547,14 +10640,14 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_5nodes_1__get__(PyObject *__pyx_v_ */ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5nodes___get__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_13___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_13___get__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_14___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_14___get__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -10564,7 +10657,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5nodes___get__(struct __pyx_obj_5_ __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_5nodes_2generator10, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -10582,9 +10675,9 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_5nodes___get__(struct __pyx_obj_5_ return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator10(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; size_t __pyx_t_1; unsigned int __pyx_t_2; @@ -10600,9 +10693,9 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_Generator return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":121 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":133 * def __get__(self): * cdef unsigned i * for i in range(self.hg.nodes_.size()): # <<<<<<<<<<<<<< @@ -10613,16 +10706,16 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_Generator for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":122 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":134 * cdef unsigned i * for i in range(self.hg.nodes_.size()): * yield HypergraphNode().init(self.hg, i) # <<<<<<<<<<<<<< * * property goal: */ - __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, __pyx_cur_scope->__pyx_v_i); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, __pyx_cur_scope->__pyx_v_i); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -10637,7 +10730,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_Generator __pyx_L6_resume_from_yield:; __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 122; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; @@ -10648,7 +10741,6 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_5nodes_2generator9(__pyx_Generator __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -10664,7 +10756,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_4goal_1__get__(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":125 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":137 * * property goal: * def __get__(self): # <<<<<<<<<<<<<< @@ -10682,7 +10774,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4goal___get__(struct __pyx_obj_5_c int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":126 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":138 * property goal: * def __get__(self): * return HypergraphNode().init(self.hg, self.hg.GoalNode()) # <<<<<<<<<<<<<< @@ -10690,9 +10782,9 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_4goal___get__(struct __pyx_obj_5_c * property npaths: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1), __pyx_v_self->hg, __pyx_v_self->hg->GoalNode()); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 126; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1), __pyx_v_self->hg, __pyx_v_self->hg->GoalNode()); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 138; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; @@ -10723,7 +10815,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_6npaths_1__get__(PyObject *__pyx_v return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":129 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":141 * * property npaths: * def __get__(self): # <<<<<<<<<<<<<< @@ -10740,7 +10832,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6npaths___get__(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":130 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":142 * property npaths: * def __get__(self): * return self.hg.NumberOfPaths() # <<<<<<<<<<<<<< @@ -10748,7 +10840,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6npaths___get__(struct __pyx_obj_5 * def inside_outside(self): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->hg->NumberOfPaths()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->hg->NumberOfPaths()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 142; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -10767,17 +10859,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6npaths___get__(struct __pyx_obj_5 } /* Python wrapper */ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_31inside_outside(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5_cdec_10Hypergraph_31inside_outside(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5_cdec_10Hypergraph_34inside_outside(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5_cdec_10Hypergraph_34inside_outside(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("inside_outside (wrapper)", 0); - __pyx_r = __pyx_pf_5_cdec_10Hypergraph_30inside_outside(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self)); + __pyx_r = __pyx_pf_5_cdec_10Hypergraph_33inside_outside(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self)); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":132 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":144 * return self.hg.NumberOfPaths() * * def inside_outside(self): # <<<<<<<<<<<<<< @@ -10785,7 +10877,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_31inside_outside(PyObject *__pyx_v * cdef prob_t z = hypergraph.InsideOutside(self.hg[0], result) */ -static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { +static PyObject *__pyx_pf_5_cdec_10Hypergraph_33inside_outside(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) { FastSparseVector *__pyx_v_result; prob_t __pyx_v_z; struct __pyx_obj_5_cdec_SparseVector *__pyx_v_vector = 0; @@ -10801,7 +10893,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("inside_outside", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":133 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":145 * * def inside_outside(self): * cdef FastSparseVector[prob_t]* result = new FastSparseVector[prob_t]() # <<<<<<<<<<<<<< @@ -10810,7 +10902,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ __pyx_v_result = new FastSparseVector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":134 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":146 * def inside_outside(self): * cdef FastSparseVector[prob_t]* result = new FastSparseVector[prob_t]() * cdef prob_t z = hypergraph.InsideOutside(self.hg[0], result) # <<<<<<<<<<<<<< @@ -10819,7 +10911,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ __pyx_v_z = InsideOutside, EdgeFeaturesAndProbWeightFunction>((__pyx_v_self->hg[0]), __pyx_v_result); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":135 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":147 * cdef FastSparseVector[prob_t]* result = new FastSparseVector[prob_t]() * cdef prob_t z = hypergraph.InsideOutside(self.hg[0], result) * result[0] /= z # <<<<<<<<<<<<<< @@ -10828,20 +10920,20 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ (__pyx_v_result[0]) /= __pyx_v_z; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":136 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":148 * cdef prob_t z = hypergraph.InsideOutside(self.hg[0], result) * result[0] /= z * cdef SparseVector vector = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< * vector.vector = new FastSparseVector[double]() * cdef FastSparseVector[prob_t].const_iterator* it = new FastSparseVector[prob_t].const_iterator(result[0], False) */ - __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_SparseVector)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_SparseVector)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_SparseVector)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 136; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_SparseVector)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 148; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":137 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":149 * result[0] /= z * cdef SparseVector vector = SparseVector.__new__(SparseVector) * vector.vector = new FastSparseVector[double]() # <<<<<<<<<<<<<< @@ -10850,7 +10942,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ __pyx_v_vector->vector = new FastSparseVector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":138 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":150 * cdef SparseVector vector = SparseVector.__new__(SparseVector) * vector.vector = new FastSparseVector[double]() * cdef FastSparseVector[prob_t].const_iterator* it = new FastSparseVector[prob_t].const_iterator(result[0], False) # <<<<<<<<<<<<<< @@ -10859,7 +10951,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ __pyx_v_it = new FastSparseVector::const_iterator((__pyx_v_result[0]), 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":140 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":152 * cdef FastSparseVector[prob_t].const_iterator* it = new FastSparseVector[prob_t].const_iterator(result[0], False) * cdef unsigned i * for i in range(result.size()): # <<<<<<<<<<<<<< @@ -10870,7 +10962,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":141 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":153 * cdef unsigned i * for i in range(result.size()): * vector.vector.set_value(it[0].ptr().first, log(it[0].ptr().second)) # <<<<<<<<<<<<<< @@ -10879,7 +10971,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ __pyx_v_vector->vector->set_value((__pyx_v_it[0]).operator->()->first, log((__pyx_v_it[0]).operator->()->second)); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":142 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":154 * for i in range(result.size()): * vector.vector.set_value(it[0].ptr().first, log(it[0].ptr().second)) * pinc(it[0]) # ++it # <<<<<<<<<<<<<< @@ -10889,7 +10981,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ (++(__pyx_v_it[0])); } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":143 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":155 * vector.vector.set_value(it[0].ptr().first, log(it[0].ptr().second)) * pinc(it[0]) # ++it * del it # <<<<<<<<<<<<<< @@ -10898,7 +10990,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ delete __pyx_v_it; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":144 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":156 * pinc(it[0]) # ++it * del it * del result # <<<<<<<<<<<<<< @@ -10907,7 +10999,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ */ delete __pyx_v_result; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":145 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":157 * del it * del result * return vector # <<<<<<<<<<<<<< @@ -10932,7 +11024,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_30inside_outside(struct __pyx_obj_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":152 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":164 * cdef public TRule trule * * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): # <<<<<<<<<<<<<< @@ -10949,7 +11041,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphEdge_init(struct __pyx_obj_5_cdec_Hy int __pyx_clineno = 0; __Pyx_RefNannySetupContext("init", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":153 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":165 * * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): * self.hg = hg # <<<<<<<<<<<<<< @@ -10958,7 +11050,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphEdge_init(struct __pyx_obj_5_cdec_Hy */ __pyx_v_self->hg = __pyx_v_hg; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":154 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":166 * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): * self.hg = hg * self.edge = &hg.edges_[i] # <<<<<<<<<<<<<< @@ -10967,23 +11059,23 @@ static PyObject *__pyx_f_5_cdec_14HypergraphEdge_init(struct __pyx_obj_5_cdec_Hy */ __pyx_v_self->edge = (&(__pyx_v_hg->edges_[__pyx_v_i])); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":155 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":167 * self.hg = hg * self.edge = &hg.edges_[i] * self.trule = TRule.__new__(TRule) # <<<<<<<<<<<<<< * self.trule.rule = new shared_ptr[grammar.TRule](self.edge.rule_) * return self */ - __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_TRule)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_TRule)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_TRule)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 155; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_TRule)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GIVEREF(__pyx_t_1); __Pyx_GOTREF(__pyx_v_self->trule); __Pyx_DECREF(((PyObject *)__pyx_v_self->trule)); __pyx_v_self->trule = ((struct __pyx_obj_5_cdec_TRule *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":156 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":168 * self.edge = &hg.edges_[i] * self.trule = TRule.__new__(TRule) * self.trule.rule = new shared_ptr[grammar.TRule](self.edge.rule_) # <<<<<<<<<<<<<< @@ -10992,7 +11084,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphEdge_init(struct __pyx_obj_5_cdec_Hy */ __pyx_v_self->trule->rule = new boost::shared_ptr(__pyx_v_self->edge->rule_); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":157 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":169 * self.trule = TRule.__new__(TRule) * self.trule.rule = new shared_ptr[grammar.TRule](self.edge.rule_) * return self # <<<<<<<<<<<<<< @@ -11027,7 +11119,7 @@ static Py_ssize_t __pyx_pw_5_cdec_14HypergraphEdge_1__len__(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":159 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":171 * return self * * def __len__(self): # <<<<<<<<<<<<<< @@ -11040,7 +11132,7 @@ static Py_ssize_t __pyx_pf_5_cdec_14HypergraphEdge___len__(struct __pyx_obj_5_cd __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":160 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":172 * * def __len__(self): * return self.edge.tail_nodes_.size() # <<<<<<<<<<<<<< @@ -11067,7 +11159,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_9head_node_1__get__(PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":163 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":175 * * property head_node: * def __get__(self): # <<<<<<<<<<<<<< @@ -11085,7 +11177,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_9head_node___get__(struct __py int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":164 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":176 * property head_node: * def __get__(self): * return HypergraphNode().init(self.hg, self.edge.head_node_) # <<<<<<<<<<<<<< @@ -11093,9 +11185,9 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_9head_node___get__(struct __py * property tail_nodes: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1), __pyx_v_self->hg, __pyx_v_self->edge->head_node_); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_1), __pyx_v_self->hg, __pyx_v_self->edge->head_node_); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; @@ -11114,7 +11206,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_9head_node___get__(struct __py __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator11(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_10tail_nodes_1__get__(PyObject *__pyx_v_self); /*proto*/ @@ -11127,7 +11219,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_10tail_nodes_1__get__(PyObject return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":167 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":179 * * property tail_nodes: * def __get__(self): # <<<<<<<<<<<<<< @@ -11136,14 +11228,14 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_10tail_nodes_1__get__(PyObject */ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_10tail_nodes___get__(struct __pyx_obj_5_cdec_HypergraphEdge *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_14___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_14___get__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_15___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_15___get__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -11153,7 +11245,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_10tail_nodes___get__(struct __ __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator11, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 179; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -11171,9 +11263,9 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_10tail_nodes___get__(struct __ return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator11(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; unsigned int __pyx_t_1; unsigned int __pyx_t_2; @@ -11189,9 +11281,9 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__py return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 179; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":169 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":181 * def __get__(self): * cdef unsigned i * for i in range(self.edge.tail_nodes_.size()): # <<<<<<<<<<<<<< @@ -11202,16 +11294,16 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__py for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":170 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":182 * cdef unsigned i * for i in range(self.edge.tail_nodes_.size()): * yield HypergraphNode().init(self.hg, self.edge.tail_nodes_[i]) # <<<<<<<<<<<<<< * * property span: */ - __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphNode)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 182; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->edge->tail_nodes_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphNode *)((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->edge->tail_nodes_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 182; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -11226,7 +11318,7 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__py __pyx_L6_resume_from_yield:; __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 182; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; @@ -11237,7 +11329,6 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphEdge_10tail_nodes_2generator10(__py __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -11253,7 +11344,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_4span_1__get__(PyObject *__pyx return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":173 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":185 * * property span: * def __get__(self): # <<<<<<<<<<<<<< @@ -11272,7 +11363,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_4span___get__(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":174 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":186 * property span: * def __get__(self): * return (self.edge.i_, self.edge.j_) # <<<<<<<<<<<<<< @@ -11280,11 +11371,11 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_4span___get__(struct __pyx_obj * property feature_values: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyInt_FromLong(__pyx_v_self->edge->i_); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyInt_FromLong(__pyx_v_self->edge->i_); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyInt_FromLong(__pyx_v_self->edge->j_); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyInt_FromLong(__pyx_v_self->edge->j_); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 174; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __Pyx_GIVEREF(__pyx_t_1); @@ -11321,7 +11412,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_14feature_values_1__get__(PyOb return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":177 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":189 * * property feature_values: * def __get__(self): # <<<<<<<<<<<<<< @@ -11339,20 +11430,20 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_14feature_values___get__(struc int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":178 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":190 * property feature_values: * def __get__(self): * cdef SparseVector vector = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< * vector.vector = new FastSparseVector[double](self.edge.feature_values_) * return vector */ - __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_SparseVector)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_tp_new(((PyObject*)__pyx_ptype_5_cdec_SparseVector)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_SparseVector)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 178; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_5_cdec_SparseVector)))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":179 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":191 * def __get__(self): * cdef SparseVector vector = SparseVector.__new__(SparseVector) * vector.vector = new FastSparseVector[double](self.edge.feature_values_) # <<<<<<<<<<<<<< @@ -11361,7 +11452,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_14feature_values___get__(struc */ __pyx_v_vector->vector = new FastSparseVector(__pyx_v_self->edge->feature_values_); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":180 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":192 * cdef SparseVector vector = SparseVector.__new__(SparseVector) * vector.vector = new FastSparseVector[double](self.edge.feature_values_) * return vector # <<<<<<<<<<<<<< @@ -11397,7 +11488,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_4prob_1__get__(PyObject *__pyx return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":183 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":195 * * property prob: * def __get__(self): # <<<<<<<<<<<<<< @@ -11414,7 +11505,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_4prob___get__(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":184 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":196 * property prob: * def __get__(self): * return self.edge.edge_prob_.as_float() # <<<<<<<<<<<<<< @@ -11422,7 +11513,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_4prob___get__(struct __pyx_obj * def __richcmp__(HypergraphEdge x, HypergraphEdge y, int op): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->edge->edge_prob_.as_float()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 184; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->edge->edge_prob_.as_float()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -11446,8 +11537,8 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_3__richcmp__(PyObject *__pyx_v PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_HypergraphEdge, 1, "x", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_HypergraphEdge, 1, "y", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 186; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_HypergraphEdge, 1, "x", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_HypergraphEdge, 1, "y", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = __pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_v_x), ((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_v_y), ((int)__pyx_v_op)); goto __pyx_L0; __pyx_L1_error:; @@ -11457,7 +11548,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_3__richcmp__(PyObject *__pyx_v return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":186 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":198 * return self.edge.edge_prob_.as_float() * * def __richcmp__(HypergraphEdge x, HypergraphEdge y, int op): # <<<<<<<<<<<<<< @@ -11475,7 +11566,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__richcmp__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":189 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":201 * if op == 2: # == * return x.edge == y.edge * elif op == 3: # != # <<<<<<<<<<<<<< @@ -11484,7 +11575,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ */ switch (__pyx_v_op) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":187 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":199 * * def __richcmp__(HypergraphEdge x, HypergraphEdge y, int op): * if op == 2: # == # <<<<<<<<<<<<<< @@ -11493,7 +11584,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ */ case 2: - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":188 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":200 * def __richcmp__(HypergraphEdge x, HypergraphEdge y, int op): * if op == 2: # == * return x.edge == y.edge # <<<<<<<<<<<<<< @@ -11501,14 +11592,14 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ * return not (x == y) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_x->edge == __pyx_v_y->edge)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_x->edge == __pyx_v_y->edge)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; break; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":189 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":201 * if op == 2: # == * return x.edge == y.edge * elif op == 3: # != # <<<<<<<<<<<<<< @@ -11517,7 +11608,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ */ case 3: - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":190 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":202 * return x.edge == y.edge * elif op == 3: # != * return not (x == y) # <<<<<<<<<<<<<< @@ -11525,11 +11616,11 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ * */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong((!__pyx_t_2)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong((!__pyx_t_2)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 202; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -11537,18 +11628,18 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphEdge_2__richcmp__(struct __pyx_obj_ break; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":191 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":203 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for HypergraphEdge') # <<<<<<<<<<<<<< * * cdef class HypergraphNode: */ - __pyx_t_1 = PyObject_Call(__pyx_builtin_NotImplemented, ((PyObject *)__pyx_k_tuple_18), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_NotImplemented, ((PyObject *)__pyx_k_tuple_18), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - {__pyx_filename = __pyx_f[3]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -11573,7 +11664,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphEdge_5trule_1__get__(PyObject *__py return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":150 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":162 * cdef hypergraph.Hypergraph* hg * cdef hypergraph.HypergraphEdge* edge * cdef public TRule trule # <<<<<<<<<<<<<< @@ -11615,7 +11706,7 @@ static int __pyx_pf_5_cdec_14HypergraphEdge_5trule_2__set__(struct __pyx_obj_5_c const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__set__", 0); - if (!(likely(((__pyx_v_value) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_value, __pyx_ptype_5_cdec_TRule))))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 150; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(((__pyx_v_value) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_value, __pyx_ptype_5_cdec_TRule))))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 162; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_INCREF(__pyx_v_value); __Pyx_GIVEREF(__pyx_v_value); __Pyx_GOTREF(__pyx_v_self->trule); @@ -11658,7 +11749,7 @@ static int __pyx_pf_5_cdec_14HypergraphEdge_5trule_4__del__(struct __pyx_obj_5_c return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":197 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":209 * cdef hypergraph.HypergraphNode* node * * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): # <<<<<<<<<<<<<< @@ -11671,7 +11762,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphNode_init(struct __pyx_obj_5_cdec_Hy __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("init", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":198 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":210 * * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): * self.hg = hg # <<<<<<<<<<<<<< @@ -11680,7 +11771,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphNode_init(struct __pyx_obj_5_cdec_Hy */ __pyx_v_self->hg = __pyx_v_hg; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":199 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":211 * cdef init(self, hypergraph.Hypergraph* hg, unsigned i): * self.hg = hg * self.node = &hg.nodes_[i] # <<<<<<<<<<<<<< @@ -11689,7 +11780,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphNode_init(struct __pyx_obj_5_cdec_Hy */ __pyx_v_self->node = (&(__pyx_v_hg->nodes_[__pyx_v_i])); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":200 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":212 * self.hg = hg * self.node = &hg.nodes_[i] * return self # <<<<<<<<<<<<<< @@ -11707,7 +11798,7 @@ static PyObject *__pyx_f_5_cdec_14HypergraphNode_init(struct __pyx_obj_5_cdec_Hy __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator12(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_8in_edges_1__get__(PyObject *__pyx_v_self); /*proto*/ @@ -11720,7 +11811,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_8in_edges_1__get__(PyObject *_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":203 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":215 * * property in_edges: * def __get__(self): # <<<<<<<<<<<<<< @@ -11729,14 +11820,14 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_8in_edges_1__get__(PyObject *_ */ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_8in_edges___get__(struct __pyx_obj_5_cdec_HypergraphNode *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_15___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_15___get__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_16___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_16___get__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -11746,7 +11837,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_8in_edges___get__(struct __pyx __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator12, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -11764,9 +11855,9 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_8in_edges___get__(struct __pyx return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator12(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; size_t __pyx_t_1; unsigned int __pyx_t_2; @@ -11782,9 +11873,9 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_G return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":205 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":217 * def __get__(self): * cdef unsigned i * for i in range(self.node.in_edges_.size()): # <<<<<<<<<<<<<< @@ -11795,16 +11886,16 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_G for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":206 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":218 * cdef unsigned i * for i in range(self.node.in_edges_.size()): * yield HypergraphEdge().init(self.hg, self.node.in_edges_[i]) # <<<<<<<<<<<<<< * * property out_edges: */ - __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 206; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->node->in_edges_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 206; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->node->in_edges_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -11819,7 +11910,7 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_G __pyx_L6_resume_from_yield:; __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 206; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 218; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; @@ -11830,11 +11921,10 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_8in_edges_2generator11(__pyx_G __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } -static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator13(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_9out_edges_1__get__(PyObject *__pyx_v_self); /*proto*/ @@ -11847,7 +11937,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_9out_edges_1__get__(PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":209 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":221 * * property out_edges: * def __get__(self): # <<<<<<<<<<<<<< @@ -11856,14 +11946,14 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_9out_edges_1__get__(PyObject * */ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_9out_edges___get__(struct __pyx_obj_5_cdec_HypergraphNode *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_16___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_16___get__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)__pyx_ptype_5_cdec___pyx_scope_struct_17___get__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_17___get__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -11873,7 +11963,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_9out_edges___get__(struct __py __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 209; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator13, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -11891,9 +11981,9 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_9out_edges___get__(struct __py return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator13(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; size_t __pyx_t_1; unsigned int __pyx_t_2; @@ -11909,9 +11999,9 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_ return NULL; } __pyx_L3_first_run:; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 209; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":211 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":223 * def __get__(self): * cdef unsigned i * for i in range(self.node.out_edges_.size()): # <<<<<<<<<<<<<< @@ -11922,16 +12012,16 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_ for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":212 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":224 * cdef unsigned i * for i in range(self.node.out_edges_.size()): * yield HypergraphEdge().init(self.hg, self.node.out_edges_[i]) # <<<<<<<<<<<<<< * * property span: */ - __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 212; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_HypergraphEdge)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 224; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->node->out_edges_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 212; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = ((struct __pyx_vtabstruct_5_cdec_HypergraphEdge *)((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3)->__pyx_vtab)->init(((struct __pyx_obj_5_cdec_HypergraphEdge *)__pyx_t_3), __pyx_cur_scope->__pyx_v_self->hg, (__pyx_cur_scope->__pyx_v_self->node->out_edges_[__pyx_cur_scope->__pyx_v_i])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 224; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -11946,7 +12036,7 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_ __pyx_L6_resume_from_yield:; __pyx_t_1 = __pyx_cur_scope->__pyx_t_0; __pyx_t_2 = __pyx_cur_scope->__pyx_t_1; - if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 212; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 224; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } PyErr_SetNone(PyExc_StopIteration); goto __pyx_L0; @@ -11957,7 +12047,6 @@ static PyObject *__pyx_gb_5_cdec_14HypergraphNode_9out_edges_2generator12(__pyx_ __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -11973,7 +12062,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_4span_1__get__(PyObject *__pyx return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":215 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":227 * * property span: * def __get__(self): # <<<<<<<<<<<<<< @@ -11991,7 +12080,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_4span___get__(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":216 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":228 * property span: * def __get__(self): * return next(self.in_edges).span # <<<<<<<<<<<<<< @@ -11999,12 +12088,12 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_4span___get__(struct __pyx_obj * property cat: */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__in_edges); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 216; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__in_edges); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyIter_Next(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 216; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyIter_Next(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__span); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 216; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s__span); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; @@ -12035,7 +12124,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_3cat_1__get__(PyObject *__pyx_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":219 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":231 * * property cat: * def __get__(self): # <<<<<<<<<<<<<< @@ -12052,7 +12141,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_3cat___get__(struct __pyx_obj_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":220 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":232 * property cat: * def __get__(self): * if self.node.cat_: # <<<<<<<<<<<<<< @@ -12061,7 +12150,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_3cat___get__(struct __pyx_obj_ */ if (__pyx_v_self->node->cat_) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":221 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":233 * def __get__(self): * if self.node.cat_: * return TDConvert(-self.node.cat_) # <<<<<<<<<<<<<< @@ -12069,7 +12158,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode_3cat___get__(struct __pyx_obj_ * def __richcmp__(HypergraphNode x, HypergraphNode y, int op): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyBytes_FromString(TD::Convert((-__pyx_v_self->node->cat_))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyBytes_FromString(TD::Convert((-__pyx_v_self->node->cat_))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 233; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __pyx_r = ((PyObject *)__pyx_t_1); __pyx_t_1 = 0; @@ -12096,8 +12185,8 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_1__richcmp__(PyObject *__pyx_v PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0); - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_HypergraphNode, 1, "x", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 223; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_HypergraphNode, 1, "y", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 223; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_HypergraphNode, 1, "x", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_HypergraphNode, 1, "y", 0))) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = __pyx_pf_5_cdec_14HypergraphNode___richcmp__(((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_v_x), ((struct __pyx_obj_5_cdec_HypergraphNode *)__pyx_v_y), ((int)__pyx_v_op)); goto __pyx_L0; __pyx_L1_error:; @@ -12107,7 +12196,7 @@ static PyObject *__pyx_pw_5_cdec_14HypergraphNode_1__richcmp__(PyObject *__pyx_v return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":223 +/* "/home/cdyer/cdec/python/src/hypergraph.pxi":235 * return TDConvert(-self.node.cat_) * * def __richcmp__(HypergraphNode x, HypergraphNode y, int op): # <<<<<<<<<<<<<< @@ -12125,7 +12214,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__richcmp__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":226 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":238 * if op == 2: # == * return x.node == y.node * elif op == 3: # != # <<<<<<<<<<<<<< @@ -12134,7 +12223,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 */ switch (__pyx_v_op) { - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":224 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":236 * * def __richcmp__(HypergraphNode x, HypergraphNode y, int op): * if op == 2: # == # <<<<<<<<<<<<<< @@ -12143,7 +12232,7 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 */ case 2: - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":225 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":237 * def __richcmp__(HypergraphNode x, HypergraphNode y, int op): * if op == 2: # == * return x.node == y.node # <<<<<<<<<<<<<< @@ -12151,14 +12240,14 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 * return not (x == y) */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_x->node == __pyx_v_y->node)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 225; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong((__pyx_v_x->node == __pyx_v_y->node)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 237; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; break; - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":226 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":238 * if op == 2: # == * return x.node == y.node * elif op == 3: # != # <<<<<<<<<<<<<< @@ -12167,18 +12256,18 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 */ case 3: - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":227 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":239 * return x.node == y.node * elif op == 3: # != * return not (x == y) # <<<<<<<<<<<<<< * raise NotImplemented('comparison not implemented for HypergraphNode') */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 227; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 239; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 227; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 239; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong((!__pyx_t_2)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 227; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_PyBool_FromLong((!__pyx_t_2)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 239; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -12186,16 +12275,16 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 break; } - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":228 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":240 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for HypergraphNode') # <<<<<<<<<<<<<< */ - __pyx_t_1 = PyObject_Call(__pyx_builtin_NotImplemented, ((PyObject *)__pyx_k_tuple_20), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(__pyx_builtin_NotImplemented, ((PyObject *)__pyx_k_tuple_20), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 240; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[3]; __pyx_lineno = 240; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; @@ -12213,11 +12302,11 @@ static PyObject *__pyx_pf_5_cdec_14HypergraphNode___richcmp__(struct __pyx_obj_5 static int __pyx_pw_5_cdec_7Lattice_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5_cdec_7Lattice_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_inp = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__inp,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__inp,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -12230,7 +12319,8 @@ static int __pyx_pw_5_cdec_7Lattice_1__cinit__(PyObject *__pyx_v_self, PyObject kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__inp)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__inp); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -12256,7 +12346,7 @@ static int __pyx_pw_5_cdec_7Lattice_1__cinit__(PyObject *__pyx_v_self, PyObject return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":6 +/* "/home/cdyer/cdec/python/src/lattice.pxi":6 * cdef lattice.Lattice* lattice * * def __cinit__(self, inp): # <<<<<<<<<<<<<< @@ -12283,7 +12373,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __Pyx_RefNannySetupContext("__cinit__", 0); __Pyx_INCREF(__pyx_v_inp); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":7 + /* "/home/cdyer/cdec/python/src/lattice.pxi":7 * * def __cinit__(self, inp): * if isinstance(inp, tuple): # <<<<<<<<<<<<<< @@ -12296,7 +12386,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":8 + /* "/home/cdyer/cdec/python/src/lattice.pxi":8 * def __cinit__(self, inp): * if isinstance(inp, tuple): * self.lattice = new lattice.Lattice(len(inp)) # <<<<<<<<<<<<<< @@ -12306,7 +12396,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __pyx_t_3 = PyObject_Length(__pyx_v_inp); if (unlikely(__pyx_t_3 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_self->lattice = new Lattice(__pyx_t_3); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":9 + /* "/home/cdyer/cdec/python/src/lattice.pxi":9 * if isinstance(inp, tuple): * self.lattice = new lattice.Lattice(len(inp)) * for i, arcs in enumerate(inp): # <<<<<<<<<<<<<< @@ -12326,18 +12416,10 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ for (;;) { if (!__pyx_t_5 && PyList_CheckExact(__pyx_t_4)) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_3); __Pyx_INCREF(__pyx_t_6); __pyx_t_3++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_4, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_5 && PyTuple_CheckExact(__pyx_t_4)) { if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_3); __Pyx_INCREF(__pyx_t_6); __pyx_t_3++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_4, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_6 = __pyx_t_5(__pyx_t_4); if (unlikely(!__pyx_t_6)) { @@ -12361,7 +12443,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __pyx_t_1 = __pyx_t_6; __pyx_t_6 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":10 + /* "/home/cdyer/cdec/python/src/lattice.pxi":10 * self.lattice = new lattice.Lattice(len(inp)) * for i, arcs in enumerate(inp): * self[i] = arcs # <<<<<<<<<<<<<< @@ -12376,7 +12458,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ } /*else*/ { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":12 + /* "/home/cdyer/cdec/python/src/lattice.pxi":12 * self[i] = arcs * else: * if isinstance(inp, unicode): # <<<<<<<<<<<<<< @@ -12389,7 +12471,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":13 + /* "/home/cdyer/cdec/python/src/lattice.pxi":13 * else: * if isinstance(inp, unicode): * inp = inp.encode('utf8') # <<<<<<<<<<<<<< @@ -12408,7 +12490,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ } __pyx_L6:; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":14 + /* "/home/cdyer/cdec/python/src/lattice.pxi":14 * if isinstance(inp, unicode): * inp = inp.encode('utf8') * if not isinstance(inp, str): # <<<<<<<<<<<<<< @@ -12422,7 +12504,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ __pyx_t_7 = (!__pyx_t_2); if (__pyx_t_7) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":15 + /* "/home/cdyer/cdec/python/src/lattice.pxi":15 * inp = inp.encode('utf8') * if not isinstance(inp, str): * raise TypeError('Cannot create lattice from %s' % type(inp)) # <<<<<<<<<<<<<< @@ -12446,7 +12528,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ } __pyx_L7:; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":16 + /* "/home/cdyer/cdec/python/src/lattice.pxi":16 * if not isinstance(inp, str): * raise TypeError('Cannot create lattice from %s' % type(inp)) * self.lattice = new lattice.Lattice() # <<<<<<<<<<<<<< @@ -12455,7 +12537,7 @@ static int __pyx_pf_5_cdec_7Lattice___cinit__(struct __pyx_obj_5_cdec_Lattice *_ */ __pyx_v_self->lattice = new Lattice(); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":17 + /* "/home/cdyer/cdec/python/src/lattice.pxi":17 * raise TypeError('Cannot create lattice from %s' % type(inp)) * self.lattice = new lattice.Lattice() * lattice.ConvertTextToLattice(string(inp), self.lattice) # <<<<<<<<<<<<<< @@ -12492,7 +12574,7 @@ static void __pyx_pw_5_cdec_7Lattice_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":19 +/* "/home/cdyer/cdec/python/src/lattice.pxi":19 * lattice.ConvertTextToLattice(string(inp), self.lattice) * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -12504,7 +12586,7 @@ static void __pyx_pf_5_cdec_7Lattice_2__dealloc__(CYTHON_UNUSED struct __pyx_obj __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":20 + /* "/home/cdyer/cdec/python/src/lattice.pxi":20 * * def __dealloc__(self): * del self.lattice # <<<<<<<<<<<<<< @@ -12537,7 +12619,7 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_5__getitem__(PyObject *__pyx_v_self, P return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":22 +/* "/home/cdyer/cdec/python/src/lattice.pxi":22 * del self.lattice * * def __getitem__(self, int index): # <<<<<<<<<<<<<< @@ -12567,7 +12649,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":23 + /* "/home/cdyer/cdec/python/src/lattice.pxi":23 * * def __getitem__(self, int index): * if not 0 <= index < len(self): # <<<<<<<<<<<<<< @@ -12582,7 +12664,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L __pyx_t_3 = (!__pyx_t_1); if (__pyx_t_3) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":24 + /* "/home/cdyer/cdec/python/src/lattice.pxi":24 * def __getitem__(self, int index): * if not 0 <= index < len(self): * raise IndexError('lattice index out of range') # <<<<<<<<<<<<<< @@ -12598,7 +12680,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":25 + /* "/home/cdyer/cdec/python/src/lattice.pxi":25 * if not 0 <= index < len(self): * raise IndexError('lattice index out of range') * arcs = [] # <<<<<<<<<<<<<< @@ -12610,7 +12692,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L __pyx_v_arcs = __pyx_t_4; __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":26 + /* "/home/cdyer/cdec/python/src/lattice.pxi":26 * raise IndexError('lattice index out of range') * arcs = [] * cdef vector[lattice.LatticeArc] arc_vector = self.lattice[0][index] # <<<<<<<<<<<<<< @@ -12619,7 +12701,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L */ __pyx_v_arc_vector = ((__pyx_v_self->lattice[0])[__pyx_v_index]); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":29 + /* "/home/cdyer/cdec/python/src/lattice.pxi":29 * cdef lattice.LatticeArc* arc * cdef unsigned i * for i in range(arc_vector.size()): # <<<<<<<<<<<<<< @@ -12630,7 +12712,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) { __pyx_v_i = __pyx_t_6; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":30 + /* "/home/cdyer/cdec/python/src/lattice.pxi":30 * cdef unsigned i * for i in range(arc_vector.size()): * arc = &arc_vector[i] # <<<<<<<<<<<<<< @@ -12639,7 +12721,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L */ __pyx_v_arc = (&(__pyx_v_arc_vector[__pyx_v_i])); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":31 + /* "/home/cdyer/cdec/python/src/lattice.pxi":31 * for i in range(arc_vector.size()): * arc = &arc_vector[i] * label = unicode(TDConvert(arc.label), 'utf8') # <<<<<<<<<<<<<< @@ -12663,7 +12745,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L __pyx_v_label = ((PyObject*)__pyx_t_4); __pyx_t_4 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":32 + /* "/home/cdyer/cdec/python/src/lattice.pxi":32 * arc = &arc_vector[i] * label = unicode(TDConvert(arc.label), 'utf8') * arcs.append((label, arc.cost, arc.dist2next)) # <<<<<<<<<<<<<< @@ -12689,7 +12771,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_4__getitem__(struct __pyx_obj_5_cdec_L __Pyx_DECREF(((PyObject *)__pyx_t_8)); __pyx_t_8 = 0; } - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":33 + /* "/home/cdyer/cdec/python/src/lattice.pxi":33 * label = unicode(TDConvert(arc.label), 'utf8') * arcs.append((label, arc.cost, arc.dist2next)) * return tuple(arcs) # <<<<<<<<<<<<<< @@ -12745,7 +12827,7 @@ static int __pyx_pw_5_cdec_7Lattice_7__setitem__(PyObject *__pyx_v_self, PyObjec return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":35 +/* "/home/cdyer/cdec/python/src/lattice.pxi":35 * return tuple(arcs) * * def __setitem__(self, int index, tuple arcs): # <<<<<<<<<<<<<< @@ -12778,7 +12860,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__setitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":36 + /* "/home/cdyer/cdec/python/src/lattice.pxi":36 * * def __setitem__(self, int index, tuple arcs): * if not 0 <= index < len(self): # <<<<<<<<<<<<<< @@ -12793,7 +12875,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice __pyx_t_3 = (!__pyx_t_1); if (__pyx_t_3) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":37 + /* "/home/cdyer/cdec/python/src/lattice.pxi":37 * def __setitem__(self, int index, tuple arcs): * if not 0 <= index < len(self): * raise IndexError('lattice index out of range') # <<<<<<<<<<<<<< @@ -12809,7 +12891,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":39 + /* "/home/cdyer/cdec/python/src/lattice.pxi":39 * raise IndexError('lattice index out of range') * cdef lattice.LatticeArc* arc * for (label, cost, dist2next) in arcs: # <<<<<<<<<<<<<< @@ -12817,35 +12899,29 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice * label = label.encode('utf8') */ if (unlikely(((PyObject *)__pyx_v_arcs) == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); - {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_t_4 = ((PyObject *)__pyx_v_arcs); __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0; for (;;) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_5); __pyx_t_2++; - #else - __pyx_t_5 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) { PyObject* sequence = __pyx_t_5; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 3)) { - if (size > 3) __Pyx_RaiseTooManyValuesError(3); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 3)) { + if (PyTuple_GET_SIZE(sequence) > 3) __Pyx_RaiseTooManyValuesError(3); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); __pyx_t_8 = PyTuple_GET_ITEM(sequence, 2); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 3)) { + if (PyList_GET_SIZE(sequence) > 3) __Pyx_RaiseTooManyValuesError(3); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_6 = PyList_GET_ITEM(sequence, 0); __pyx_t_7 = PyList_GET_ITEM(sequence, 1); __pyx_t_8 = PyList_GET_ITEM(sequence, 2); @@ -12853,14 +12929,8 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_8); - #else - __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_8 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_9 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_9); @@ -12873,13 +12943,12 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice index = 2; __pyx_t_8 = __pyx_t_10(__pyx_t_9); if (unlikely(!__pyx_t_8)) goto __pyx_L6_unpacking_failed; __Pyx_GOTREF(__pyx_t_8); if (__Pyx_IternextUnpackEndCheck(__pyx_t_10(__pyx_t_9), 3) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_10 = NULL; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L7_unpacking_done; __pyx_L6_unpacking_failed:; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __pyx_t_10 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L7_unpacking_done:; } @@ -12893,7 +12962,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice __pyx_v_dist2next = __pyx_t_8; __pyx_t_8 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":40 + /* "/home/cdyer/cdec/python/src/lattice.pxi":40 * cdef lattice.LatticeArc* arc * for (label, cost, dist2next) in arcs: * if isinstance(label, unicode): # <<<<<<<<<<<<<< @@ -12906,7 +12975,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_3) { - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":41 + /* "/home/cdyer/cdec/python/src/lattice.pxi":41 * for (label, cost, dist2next) in arcs: * if isinstance(label, unicode): * label = label.encode('utf8') # <<<<<<<<<<<<<< @@ -12925,7 +12994,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice } __pyx_L8:; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":42 + /* "/home/cdyer/cdec/python/src/lattice.pxi":42 * if isinstance(label, unicode): * label = label.encode('utf8') * arc = new lattice.LatticeArc(TDConvert(label), cost, dist2next) # <<<<<<<<<<<<<< @@ -12937,7 +13006,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice __pyx_t_13 = __Pyx_PyInt_AsInt(__pyx_v_dist2next); if (unlikely((__pyx_t_13 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_arc = new LatticeArc(TD::Convert(((char *)__pyx_t_11)), __pyx_t_12, __pyx_t_13); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":43 + /* "/home/cdyer/cdec/python/src/lattice.pxi":43 * label = label.encode('utf8') * arc = new lattice.LatticeArc(TDConvert(label), cost, dist2next) * self.lattice[0][index].push_back(arc[0]) # <<<<<<<<<<<<<< @@ -12946,7 +13015,7 @@ static int __pyx_pf_5_cdec_7Lattice_6__setitem__(struct __pyx_obj_5_cdec_Lattice */ ((__pyx_v_self->lattice[0])[__pyx_v_index]).push_back((__pyx_v_arc[0])); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":44 + /* "/home/cdyer/cdec/python/src/lattice.pxi":44 * arc = new lattice.LatticeArc(TDConvert(label), cost, dist2next) * self.lattice[0][index].push_back(arc[0]) * del arc # <<<<<<<<<<<<<< @@ -12987,7 +13056,7 @@ static Py_ssize_t __pyx_pw_5_cdec_7Lattice_9__len__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":46 +/* "/home/cdyer/cdec/python/src/lattice.pxi":46 * del arc * * def __len__(self): # <<<<<<<<<<<<<< @@ -13000,7 +13069,7 @@ static Py_ssize_t __pyx_pf_5_cdec_7Lattice_8__len__(struct __pyx_obj_5_cdec_Latt __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":47 + /* "/home/cdyer/cdec/python/src/lattice.pxi":47 * * def __len__(self): * return self.lattice.size() # <<<<<<<<<<<<<< @@ -13027,7 +13096,7 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_11__str__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":49 +/* "/home/cdyer/cdec/python/src/lattice.pxi":49 * return self.lattice.size() * * def __str__(self): # <<<<<<<<<<<<<< @@ -13044,7 +13113,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_10__str__(struct __pyx_obj_5_cdec_Latt int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":50 + /* "/home/cdyer/cdec/python/src/lattice.pxi":50 * * def __str__(self): * return hypergraph.AsPLF(self.lattice[0], True).c_str() # <<<<<<<<<<<<<< @@ -13069,7 +13138,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_10__str__(struct __pyx_obj_5_cdec_Latt __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Lattice_14generator13(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_7Lattice_14generator14(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_7Lattice_13__iter__(PyObject *__pyx_v_self); /*proto*/ @@ -13082,7 +13151,7 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_13__iter__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":52 +/* "/home/cdyer/cdec/python/src/lattice.pxi":52 * return hypergraph.AsPLF(self.lattice[0], True).c_str() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -13091,14 +13160,14 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_13__iter__(PyObject *__pyx_v_self) { */ static PyObject *__pyx_pf_5_cdec_7Lattice_12__iter__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__iter__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_17___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_17___iter__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_18___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_18___iter__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -13108,7 +13177,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_12__iter__(struct __pyx_obj_5_cdec_Lat __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_14generator13, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_14generator14, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -13126,9 +13195,9 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_12__iter__(struct __pyx_obj_5_cdec_Lat return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Lattice_14generator13(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_7Lattice_14generator14(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; Py_ssize_t __pyx_t_1; unsigned int __pyx_t_2; @@ -13145,7 +13214,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_14generator13(__pyx_GeneratorObject *_ __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":54 + /* "/home/cdyer/cdec/python/src/lattice.pxi":54 * def __iter__(self): * cdef unsigned i * for i in range(len(self)): # <<<<<<<<<<<<<< @@ -13156,7 +13225,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_14generator13(__pyx_GeneratorObject *_ for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":55 + /* "/home/cdyer/cdec/python/src/lattice.pxi":55 * cdef unsigned i * for i in range(len(self)): * yield self[i] # <<<<<<<<<<<<<< @@ -13187,7 +13256,6 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_14generator13(__pyx_GeneratorObject *_ __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -13202,7 +13270,7 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_16todot(PyObject *__pyx_v_self, CYTHON __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator20(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_7Lattice_5todot_1lines(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ @@ -13211,12 +13279,13 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_5todot_1lines(PyObject *__pyx_self, CY PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("lines (wrapper)", 0); + __pyx_self = __pyx_self; __pyx_r = __pyx_pf_5_cdec_7Lattice_5todot_lines(__pyx_self); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":58 +/* "/home/cdyer/cdec/python/src/lattice.pxi":58 * * def todot(self): * def lines(): # <<<<<<<<<<<<<< @@ -13225,24 +13294,24 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_5todot_1lines(PyObject *__pyx_self, CY */ static PyObject *__pyx_pf_5_cdec_7Lattice_5todot_lines(PyObject *__pyx_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("lines", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)__pyx_ptype_5_cdec___pyx_scope_struct_19_lines->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_19_lines, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)__pyx_ptype_5_cdec___pyx_scope_struct_20_lines->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_20_lines, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; } __Pyx_GOTREF(__pyx_cur_scope); - __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *) __Pyx_CyFunction_GetClosure(__pyx_self); __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope)); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_5todot_2generator19, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_5todot_2generator20, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -13260,9 +13329,9 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_5todot_lines(PyObject *__pyx_self) { return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator20(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)__pyx_generator->closure); PyObject *__pyx_r = NULL; PyObject *__pyx_t_1 = NULL; Py_ssize_t __pyx_t_2; @@ -13293,7 +13362,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":59 + /* "/home/cdyer/cdec/python/src/lattice.pxi":59 * def todot(self): * def lines(): * yield 'digraph lattice {' # <<<<<<<<<<<<<< @@ -13310,7 +13379,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L4_resume_from_yield:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":60 + /* "/home/cdyer/cdec/python/src/lattice.pxi":60 * def lines(): * yield 'digraph lattice {' * yield 'rankdir = LR;' # <<<<<<<<<<<<<< @@ -13327,7 +13396,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L5_resume_from_yield:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":61 + /* "/home/cdyer/cdec/python/src/lattice.pxi":61 * yield 'digraph lattice {' * yield 'rankdir = LR;' * yield 'node [shape=circle];' # <<<<<<<<<<<<<< @@ -13344,7 +13413,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L6_resume_from_yield:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":62 + /* "/home/cdyer/cdec/python/src/lattice.pxi":62 * yield 'rankdir = LR;' * yield 'node [shape=circle];' * for i in range(len(self)): # <<<<<<<<<<<<<< @@ -13378,18 +13447,10 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj for (;;) { if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_3)) { if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_3)) { if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 62; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_1 = __pyx_t_4(__pyx_t_3); if (unlikely(!__pyx_t_1)) { @@ -13407,7 +13468,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_cur_scope->__pyx_v_i = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":63 + /* "/home/cdyer/cdec/python/src/lattice.pxi":63 * yield 'node [shape=circle];' * for i in range(len(self)): * for label, weight, delta in self[i]: # <<<<<<<<<<<<<< @@ -13428,18 +13489,10 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj for (;;) { if (!__pyx_t_7 && PyList_CheckExact(__pyx_t_5)) { if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_5)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_7 && PyTuple_CheckExact(__pyx_t_5)) { if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_5)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; - #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_1 = __pyx_t_7(__pyx_t_5); if (unlikely(!__pyx_t_1)) { @@ -13453,22 +13506,21 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj } if ((likely(PyTuple_CheckExact(__pyx_t_1))) || (PyList_CheckExact(__pyx_t_1))) { PyObject* sequence = __pyx_t_1; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 3)) { - if (size > 3) __Pyx_RaiseTooManyValuesError(3); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 3)) { + if (PyTuple_GET_SIZE(sequence) > 3) __Pyx_RaiseTooManyValuesError(3); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_8 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_9 = PyTuple_GET_ITEM(sequence, 1); __pyx_t_10 = PyTuple_GET_ITEM(sequence, 2); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 3)) { + if (PyList_GET_SIZE(sequence) > 3) __Pyx_RaiseTooManyValuesError(3); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_8 = PyList_GET_ITEM(sequence, 0); __pyx_t_9 = PyList_GET_ITEM(sequence, 1); __pyx_t_10 = PyList_GET_ITEM(sequence, 2); @@ -13476,14 +13528,8 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_9); __Pyx_INCREF(__pyx_t_10); - #else - __pyx_t_8 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_10 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_11 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_11); @@ -13496,13 +13542,12 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj index = 2; __pyx_t_10 = __pyx_t_12(__pyx_t_11); if (unlikely(!__pyx_t_10)) goto __pyx_L11_unpacking_failed; __Pyx_GOTREF(__pyx_t_10); if (__Pyx_IternextUnpackEndCheck(__pyx_t_12(__pyx_t_11), 3) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_12 = NULL; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; goto __pyx_L12_unpacking_done; __pyx_L11_unpacking_failed:; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - __pyx_t_12 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L12_unpacking_done:; } @@ -13522,7 +13567,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_cur_scope->__pyx_v_delta = __pyx_t_10; __pyx_t_10 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":64 + /* "/home/cdyer/cdec/python/src/lattice.pxi":64 * for i in range(len(self)): * for label, weight, delta in self[i]: * yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"')) # <<<<<<<<<<<<<< @@ -13582,7 +13627,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":65 + /* "/home/cdyer/cdec/python/src/lattice.pxi":65 * for label, weight, delta in self[i]: * yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"')) * yield '%d [shape=doublecircle]' % len(self) # <<<<<<<<<<<<<< @@ -13608,7 +13653,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L14_resume_from_yield:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":66 + /* "/home/cdyer/cdec/python/src/lattice.pxi":66 * yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"')) * yield '%d [shape=doublecircle]' % len(self) * yield '}' # <<<<<<<<<<<<<< @@ -13637,12 +13682,11 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } -/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":57 +/* "/home/cdyer/cdec/python/src/lattice.pxi":57 * yield self[i] * * def todot(self): # <<<<<<<<<<<<<< @@ -13651,7 +13695,7 @@ static PyObject *__pyx_gb_5_cdec_7Lattice_5todot_2generator19(__pyx_GeneratorObj */ static PyObject *__pyx_pf_5_cdec_7Lattice_15todot(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *__pyx_cur_scope; PyObject *__pyx_v_lines = 0; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations @@ -13662,7 +13706,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_15todot(struct __pyx_obj_5_cdec_Lattic const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("todot", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)__pyx_ptype_5_cdec___pyx_scope_struct_18_todot->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_18_todot, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)__pyx_ptype_5_cdec___pyx_scope_struct_19_todot->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_19_todot, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -13672,7 +13716,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_15todot(struct __pyx_obj_5_cdec_Lattic __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":58 + /* "/home/cdyer/cdec/python/src/lattice.pxi":58 * * def todot(self): * def lines(): # <<<<<<<<<<<<<< @@ -13684,7 +13728,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_15todot(struct __pyx_obj_5_cdec_Lattic __pyx_v_lines = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":67 + /* "/home/cdyer/cdec/python/src/lattice.pxi":67 * yield '%d [shape=doublecircle]' % len(self) * yield '}' * return '\n'.join(lines()).encode('utf8') # <<<<<<<<<<<<<< @@ -13729,7 +13773,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_15todot(struct __pyx_obj_5_cdec_Lattic return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":3 +/* "/home/cdyer/cdec/python/src/mteval.pxi":3 * cimport mteval * * cdef SufficientStats as_stats(x, y): # <<<<<<<<<<<<<< @@ -13750,7 +13794,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject int __pyx_clineno = 0; __Pyx_RefNannySetupContext("as_stats", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":4 + /* "/home/cdyer/cdec/python/src/mteval.pxi":4 * * cdef SufficientStats as_stats(x, y): * if isinstance(x, SufficientStats): # <<<<<<<<<<<<<< @@ -13763,7 +13807,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":5 + /* "/home/cdyer/cdec/python/src/mteval.pxi":5 * cdef SufficientStats as_stats(x, y): * if isinstance(x, SufficientStats): * return x # <<<<<<<<<<<<<< @@ -13778,7 +13822,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject goto __pyx_L3; } - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":6 + /* "/home/cdyer/cdec/python/src/mteval.pxi":6 * if isinstance(x, SufficientStats): * return x * elif x == 0 and isinstance(y, SufficientStats): # <<<<<<<<<<<<<< @@ -13800,7 +13844,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject } if (__pyx_t_4) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":7 + /* "/home/cdyer/cdec/python/src/mteval.pxi":7 * return x * elif x == 0 and isinstance(y, SufficientStats): * stats = SufficientStats() # <<<<<<<<<<<<<< @@ -13812,7 +13856,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject __pyx_v_stats = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":8 + /* "/home/cdyer/cdec/python/src/mteval.pxi":8 * elif x == 0 and isinstance(y, SufficientStats): * stats = SufficientStats() * stats.stats = new mteval.SufficientStats() # <<<<<<<<<<<<<< @@ -13821,7 +13865,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject */ __pyx_v_stats->stats = new SufficientStats(); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":9 + /* "/home/cdyer/cdec/python/src/mteval.pxi":9 * stats = SufficientStats() * stats.stats = new mteval.SufficientStats() * stats.metric = ( y).metric # <<<<<<<<<<<<<< @@ -13830,7 +13874,7 @@ static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject */ __pyx_v_stats->metric = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_y)->metric; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":10 + /* "/home/cdyer/cdec/python/src/mteval.pxi":10 * stats.stats = new mteval.SufficientStats() * stats.metric = ( y).metric * return stats # <<<<<<<<<<<<<< @@ -13869,7 +13913,7 @@ static PyObject *__pyx_pw_5_cdec_9Candidate_5words_1__get__(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":17 +/* "/home/cdyer/cdec/python/src/mteval.pxi":17 * * property words: * def __get__(self): # <<<<<<<<<<<<<< @@ -13888,7 +13932,7 @@ static PyObject *__pyx_pf_5_cdec_9Candidate_5words___get__(struct __pyx_obj_5_cd int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":18 + /* "/home/cdyer/cdec/python/src/mteval.pxi":18 * property words: * def __get__(self): * return unicode(GetString(self.candidate.ewords).c_str(), encoding='utf8') # <<<<<<<<<<<<<< @@ -13939,7 +13983,7 @@ static PyObject *__pyx_pw_5_cdec_9Candidate_4fmap_1__get__(PyObject *__pyx_v_sel return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":21 +/* "/home/cdyer/cdec/python/src/mteval.pxi":21 * * property fmap: * def __get__(self): # <<<<<<<<<<<<<< @@ -13957,7 +14001,7 @@ static PyObject *__pyx_pf_5_cdec_9Candidate_4fmap___get__(struct __pyx_obj_5_cde int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":22 + /* "/home/cdyer/cdec/python/src/mteval.pxi":22 * property fmap: * def __get__(self): * cdef SparseVector fmap = SparseVector.__new__(SparseVector) # <<<<<<<<<<<<<< @@ -13970,7 +14014,7 @@ static PyObject *__pyx_pf_5_cdec_9Candidate_4fmap___get__(struct __pyx_obj_5_cde __pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":23 + /* "/home/cdyer/cdec/python/src/mteval.pxi":23 * def __get__(self): * cdef SparseVector fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap) # <<<<<<<<<<<<<< @@ -13979,7 +14023,7 @@ static PyObject *__pyx_pf_5_cdec_9Candidate_4fmap___get__(struct __pyx_obj_5_cde */ __pyx_v_fmap->vector = new FastSparseVector(__pyx_v_self->candidate->fmap); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":24 + /* "/home/cdyer/cdec/python/src/mteval.pxi":24 * cdef SparseVector fmap = SparseVector.__new__(SparseVector) * fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap) * return fmap # <<<<<<<<<<<<<< @@ -14015,7 +14059,7 @@ static PyObject *__pyx_pw_5_cdec_9Candidate_5score_1__get__(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":14 +/* "/home/cdyer/cdec/python/src/mteval.pxi":14 * cdef class Candidate: * cdef mteval.const_Candidate* candidate * cdef public float score # <<<<<<<<<<<<<< @@ -14091,7 +14135,7 @@ static void __pyx_pw_5_cdec_15SufficientStats_1__dealloc__(PyObject *__pyx_v_sel __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":30 +/* "/home/cdyer/cdec/python/src/mteval.pxi":30 * cdef mteval.EvaluationMetric* metric * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -14103,7 +14147,7 @@ static void __pyx_pf_5_cdec_15SufficientStats___dealloc__(CYTHON_UNUSED struct _ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":31 + /* "/home/cdyer/cdec/python/src/mteval.pxi":31 * * def __dealloc__(self): * del self.stats # <<<<<<<<<<<<<< @@ -14126,7 +14170,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_5score_1__get__(PyObject *__p return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":34 +/* "/home/cdyer/cdec/python/src/mteval.pxi":34 * * property score: * def __get__(self): # <<<<<<<<<<<<<< @@ -14143,7 +14187,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_5score___get__(struct __pyx_o int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":35 + /* "/home/cdyer/cdec/python/src/mteval.pxi":35 * property score: * def __get__(self): * return self.metric.ComputeScore(self.stats[0]) # <<<<<<<<<<<<<< @@ -14180,7 +14224,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_6detail_1__get__(PyObject *__ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":38 +/* "/home/cdyer/cdec/python/src/mteval.pxi":38 * * property detail: * def __get__(self): # <<<<<<<<<<<<<< @@ -14197,7 +14241,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_6detail___get__(struct __pyx_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":39 + /* "/home/cdyer/cdec/python/src/mteval.pxi":39 * property detail: * def __get__(self): * return self.metric.DetailedScore(self.stats[0]).c_str() # <<<<<<<<<<<<<< @@ -14234,7 +14278,7 @@ static Py_ssize_t __pyx_pw_5_cdec_15SufficientStats_3__len__(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":41 +/* "/home/cdyer/cdec/python/src/mteval.pxi":41 * return self.metric.DetailedScore(self.stats[0]).c_str() * * def __len__(self): # <<<<<<<<<<<<<< @@ -14247,7 +14291,7 @@ static Py_ssize_t __pyx_pf_5_cdec_15SufficientStats_2__len__(struct __pyx_obj_5_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":42 + /* "/home/cdyer/cdec/python/src/mteval.pxi":42 * * def __len__(self): * return self.stats.size() # <<<<<<<<<<<<<< @@ -14262,7 +14306,7 @@ static Py_ssize_t __pyx_pf_5_cdec_15SufficientStats_2__len__(struct __pyx_obj_5_ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator15(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_self); /*proto*/ @@ -14275,7 +14319,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":44 +/* "/home/cdyer/cdec/python/src/mteval.pxi":44 * return self.stats.size() * * def __iter__(self): # <<<<<<<<<<<<<< @@ -14284,14 +14328,14 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_s */ static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__iter__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_20___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_20___iter__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_21___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_21___iter__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -14301,7 +14345,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_ __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_15SufficientStats_6generator14, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_15SufficientStats_6generator15, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -14319,9 +14363,9 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_ return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator15(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; Py_ssize_t __pyx_t_1; PyObject *__pyx_t_2 = NULL; @@ -14339,7 +14383,7 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":45 + /* "/home/cdyer/cdec/python/src/mteval.pxi":45 * * def __iter__(self): * for i in range(len(self)): # <<<<<<<<<<<<<< @@ -14369,18 +14413,10 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO for (;;) { if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_3)) { if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_2); __pyx_t_1++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_3)) { if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_3)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_2); __pyx_t_1++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_3, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_2 = __pyx_t_4(__pyx_t_3); if (unlikely(!__pyx_t_2)) { @@ -14398,7 +14434,7 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO __pyx_cur_scope->__pyx_v_i = __pyx_t_2; __pyx_t_2 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":46 + /* "/home/cdyer/cdec/python/src/mteval.pxi":46 * def __iter__(self): * for i in range(len(self)): * yield self[i] # <<<<<<<<<<<<<< @@ -14436,7 +14472,6 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -14462,7 +14497,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__getitem__(PyObject *__pyx_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":48 +/* "/home/cdyer/cdec/python/src/mteval.pxi":48 * yield self[i] * * def __getitem__(self, int index): # <<<<<<<<<<<<<< @@ -14482,7 +14517,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__getitem__(struct __pyx_obj int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":49 + /* "/home/cdyer/cdec/python/src/mteval.pxi":49 * * def __getitem__(self, int index): * if not 0 <= index < len(self): # <<<<<<<<<<<<<< @@ -14497,7 +14532,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__getitem__(struct __pyx_obj __pyx_t_3 = (!__pyx_t_1); if (__pyx_t_3) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50 + /* "/home/cdyer/cdec/python/src/mteval.pxi":50 * def __getitem__(self, int index): * if not 0 <= index < len(self): * raise IndexError('sufficient stats vector index out of range') # <<<<<<<<<<<<<< @@ -14513,7 +14548,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__getitem__(struct __pyx_obj } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":51 + /* "/home/cdyer/cdec/python/src/mteval.pxi":51 * if not 0 <= index < len(self): * raise IndexError('sufficient stats vector index out of range') * return self.stats[0][index] # <<<<<<<<<<<<<< @@ -14555,7 +14590,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__iadd__(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":53 +/* "/home/cdyer/cdec/python/src/mteval.pxi":53 * return self.stats[0][index] * * def __iadd__(SufficientStats self, SufficientStats other): # <<<<<<<<<<<<<< @@ -14568,7 +14603,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__iadd__(struct __pyx_obj_5_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__iadd__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":54 + /* "/home/cdyer/cdec/python/src/mteval.pxi":54 * * def __iadd__(SufficientStats self, SufficientStats other): * self.stats[0] += other.stats[0] # <<<<<<<<<<<<<< @@ -14577,7 +14612,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__iadd__(struct __pyx_obj_5_ */ (__pyx_v_self->stats[0]) += (__pyx_v_other->stats[0]); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":55 + /* "/home/cdyer/cdec/python/src/mteval.pxi":55 * def __iadd__(SufficientStats self, SufficientStats other): * self.stats[0] += other.stats[0] * return self # <<<<<<<<<<<<<< @@ -14607,7 +14642,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_12__add__(PyObject *__pyx_v_x return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":57 +/* "/home/cdyer/cdec/python/src/mteval.pxi":57 * return self * * def __add__(x, y): # <<<<<<<<<<<<<< @@ -14627,7 +14662,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__add__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":58 + /* "/home/cdyer/cdec/python/src/mteval.pxi":58 * * def __add__(x, y): * cdef SufficientStats sx = as_stats(x, y) # <<<<<<<<<<<<<< @@ -14639,7 +14674,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x __pyx_v_sx = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":59 + /* "/home/cdyer/cdec/python/src/mteval.pxi":59 * def __add__(x, y): * cdef SufficientStats sx = as_stats(x, y) * cdef SufficientStats sy = as_stats(y, x) # <<<<<<<<<<<<<< @@ -14651,7 +14686,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x __pyx_v_sy = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":60 + /* "/home/cdyer/cdec/python/src/mteval.pxi":60 * cdef SufficientStats sx = as_stats(x, y) * cdef SufficientStats sy = as_stats(y, x) * cdef SufficientStats result = SufficientStats() # <<<<<<<<<<<<<< @@ -14663,7 +14698,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x __pyx_v_result = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":61 + /* "/home/cdyer/cdec/python/src/mteval.pxi":61 * cdef SufficientStats sy = as_stats(y, x) * cdef SufficientStats result = SufficientStats() * result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0])) # <<<<<<<<<<<<<< @@ -14672,7 +14707,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x */ __pyx_v_result->stats = new SufficientStats(operator+((__pyx_v_sx->stats[0]), (__pyx_v_sy->stats[0]))); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":62 + /* "/home/cdyer/cdec/python/src/mteval.pxi":62 * cdef SufficientStats result = SufficientStats() * result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0])) * result.metric = sx.metric # <<<<<<<<<<<<<< @@ -14681,7 +14716,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x */ __pyx_v_result->metric = __pyx_v_sx->metric; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":63 + /* "/home/cdyer/cdec/python/src/mteval.pxi":63 * result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0])) * result.metric = sx.metric * return result # <<<<<<<<<<<<<< @@ -14712,11 +14747,11 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_evaluator = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__evaluator,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__evaluator,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -14729,7 +14764,8 @@ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyO kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__evaluator)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__evaluator); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -14760,7 +14796,7 @@ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyO return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":70 +/* "/home/cdyer/cdec/python/src/mteval.pxi":70 * cdef mteval.CandidateSet* cs * * def __cinit__(self, SegmentEvaluator evaluator): # <<<<<<<<<<<<<< @@ -14773,7 +14809,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":71 + /* "/home/cdyer/cdec/python/src/mteval.pxi":71 * * def __cinit__(self, SegmentEvaluator evaluator): * self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0]) # <<<<<<<<<<<<<< @@ -14782,7 +14818,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand */ __pyx_v_self->scorer = new boost::shared_ptr((__pyx_v_evaluator->scorer[0])); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":72 + /* "/home/cdyer/cdec/python/src/mteval.pxi":72 * def __cinit__(self, SegmentEvaluator evaluator): * self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0]) * self.metric = evaluator.metric # <<<<<<<<<<<<<< @@ -14791,7 +14827,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand */ __pyx_v_self->metric = __pyx_v_evaluator->metric; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":73 + /* "/home/cdyer/cdec/python/src/mteval.pxi":73 * self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0]) * self.metric = evaluator.metric * self.cs = new mteval.CandidateSet() # <<<<<<<<<<<<<< @@ -14814,7 +14850,7 @@ static void __pyx_pw_5_cdec_12CandidateSet_3__dealloc__(PyObject *__pyx_v_self) __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":75 +/* "/home/cdyer/cdec/python/src/mteval.pxi":75 * self.cs = new mteval.CandidateSet() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -14826,7 +14862,7 @@ static void __pyx_pf_5_cdec_12CandidateSet_2__dealloc__(CYTHON_UNUSED struct __p __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":76 + /* "/home/cdyer/cdec/python/src/mteval.pxi":76 * * def __dealloc__(self): * del self.scorer # <<<<<<<<<<<<<< @@ -14835,7 +14871,7 @@ static void __pyx_pf_5_cdec_12CandidateSet_2__dealloc__(CYTHON_UNUSED struct __p */ delete __pyx_v_self->scorer; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":77 + /* "/home/cdyer/cdec/python/src/mteval.pxi":77 * def __dealloc__(self): * del self.scorer * del self.cs # <<<<<<<<<<<<<< @@ -14858,7 +14894,7 @@ static Py_ssize_t __pyx_pw_5_cdec_12CandidateSet_5__len__(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":79 +/* "/home/cdyer/cdec/python/src/mteval.pxi":79 * del self.cs * * def __len__(self): # <<<<<<<<<<<<<< @@ -14871,7 +14907,7 @@ static Py_ssize_t __pyx_pf_5_cdec_12CandidateSet_4__len__(struct __pyx_obj_5_cde __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__len__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":80 + /* "/home/cdyer/cdec/python/src/mteval.pxi":80 * * def __len__(self): * return self.cs.size() # <<<<<<<<<<<<<< @@ -14908,7 +14944,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_7__getitem__(PyObject *__pyx_v_s return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":82 +/* "/home/cdyer/cdec/python/src/mteval.pxi":82 * return self.cs.size() * * def __getitem__(self,int k): # <<<<<<<<<<<<<< @@ -14928,7 +14964,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__getitem__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":83 + /* "/home/cdyer/cdec/python/src/mteval.pxi":83 * * def __getitem__(self,int k): * if not 0 <= k < self.cs.size(): # <<<<<<<<<<<<<< @@ -14942,7 +14978,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ __pyx_t_2 = (!__pyx_t_1); if (__pyx_t_2) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":84 + /* "/home/cdyer/cdec/python/src/mteval.pxi":84 * def __getitem__(self,int k): * if not 0 <= k < self.cs.size(): * raise IndexError('candidate set index out of range') # <<<<<<<<<<<<<< @@ -14958,7 +14994,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":85 + /* "/home/cdyer/cdec/python/src/mteval.pxi":85 * if not 0 <= k < self.cs.size(): * raise IndexError('candidate set index out of range') * cdef Candidate candidate = Candidate() # <<<<<<<<<<<<<< @@ -14970,7 +15006,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ __pyx_v_candidate = ((struct __pyx_obj_5_cdec_Candidate *)__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":86 + /* "/home/cdyer/cdec/python/src/mteval.pxi":86 * raise IndexError('candidate set index out of range') * cdef Candidate candidate = Candidate() * candidate.candidate = &self.cs[0][k] # <<<<<<<<<<<<<< @@ -14979,7 +15015,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ */ __pyx_v_candidate->candidate = (&((__pyx_v_self->cs[0])[__pyx_v_k])); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":87 + /* "/home/cdyer/cdec/python/src/mteval.pxi":87 * cdef Candidate candidate = Candidate() * candidate.candidate = &self.cs[0][k] * candidate.score = self.metric.ComputeScore(self.cs[0][k].eval_feats) # <<<<<<<<<<<<<< @@ -14988,7 +15024,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ */ __pyx_v_candidate->score = __pyx_v_self->metric->ComputeScore(((__pyx_v_self->cs[0])[__pyx_v_k]).eval_feats); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":88 + /* "/home/cdyer/cdec/python/src/mteval.pxi":88 * candidate.candidate = &self.cs[0][k] * candidate.score = self.metric.ComputeScore(self.cs[0][k].eval_feats) * return candidate # <<<<<<<<<<<<<< @@ -15012,7 +15048,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator16(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_12CandidateSet_9__iter__(PyObject *__pyx_v_self); /*proto*/ @@ -15025,7 +15061,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_9__iter__(PyObject *__pyx_v_self return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90 +/* "/home/cdyer/cdec/python/src/mteval.pxi":90 * return candidate * * def __iter__(self): # <<<<<<<<<<<<<< @@ -15034,14 +15070,14 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_9__iter__(PyObject *__pyx_v_self */ static PyObject *__pyx_pf_5_cdec_12CandidateSet_8__iter__(struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__iter__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_21___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_21___iter__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_22___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_22___iter__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -15051,7 +15087,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_8__iter__(struct __pyx_obj_5_cde __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self); __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_12CandidateSet_10generator15, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_12CandidateSet_10generator16, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -15069,9 +15105,9 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_8__iter__(struct __pyx_obj_5_cde return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator16(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)__pyx_generator->closure); PyObject *__pyx_r = NULL; Py_ssize_t __pyx_t_1; unsigned int __pyx_t_2; @@ -15088,7 +15124,7 @@ static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObj __pyx_L3_first_run:; if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":92 + /* "/home/cdyer/cdec/python/src/mteval.pxi":92 * def __iter__(self): * cdef unsigned i * for i in range(len(self)): # <<<<<<<<<<<<<< @@ -15099,7 +15135,7 @@ static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObj for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) { __pyx_cur_scope->__pyx_v_i = __pyx_t_2; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":93 + /* "/home/cdyer/cdec/python/src/mteval.pxi":93 * cdef unsigned i * for i in range(len(self)): * yield self[i] # <<<<<<<<<<<<<< @@ -15130,7 +15166,6 @@ static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObj __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -15140,11 +15175,11 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_hypergraph = 0; unsigned int __pyx_v_k; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hypergraph,&__pyx_n_s__k,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("add_kbest (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hypergraph,&__pyx_n_s__k,0}; PyObject* values[2] = {0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -15158,10 +15193,12 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hypergraph)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hypergraph); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__k)) != 0)) kw_args--; + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__k); + if (likely(values[1])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("add_kbest", 1, 2, 2, 1); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } @@ -15196,7 +15233,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":95 +/* "/home/cdyer/cdec/python/src/mteval.pxi":95 * yield self[i] * * def add_kbest(self, Hypergraph hypergraph, unsigned k): # <<<<<<<<<<<<<< @@ -15209,7 +15246,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_11add_kbest(struct __pyx_obj_5_c __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("add_kbest", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":96 + /* "/home/cdyer/cdec/python/src/mteval.pxi":96 * * def add_kbest(self, Hypergraph hypergraph, unsigned k): * self.cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get()) # <<<<<<<<<<<<<< @@ -15233,7 +15270,7 @@ static void __pyx_pw_5_cdec_16SegmentEvaluator_1__dealloc__(PyObject *__pyx_v_se __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":102 +/* "/home/cdyer/cdec/python/src/mteval.pxi":102 * cdef mteval.EvaluationMetric* metric * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -15245,7 +15282,7 @@ static void __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(CYTHON_UNUSED struct __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":103 + /* "/home/cdyer/cdec/python/src/mteval.pxi":103 * * def __dealloc__(self): * del self.scorer # <<<<<<<<<<<<<< @@ -15268,7 +15305,7 @@ static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_3evaluate(PyObject *__pyx_v_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":105 +/* "/home/cdyer/cdec/python/src/mteval.pxi":105 * del self.scorer * * def evaluate(self, sentence): # <<<<<<<<<<<<<< @@ -15288,7 +15325,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 int __pyx_clineno = 0; __Pyx_RefNannySetupContext("evaluate", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":107 + /* "/home/cdyer/cdec/python/src/mteval.pxi":107 * def evaluate(self, sentence): * cdef vector[WordID] hyp * cdef SufficientStats sf = SufficientStats() # <<<<<<<<<<<<<< @@ -15300,7 +15337,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 __pyx_v_sf = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":108 + /* "/home/cdyer/cdec/python/src/mteval.pxi":108 * cdef vector[WordID] hyp * cdef SufficientStats sf = SufficientStats() * sf.metric = self.metric # <<<<<<<<<<<<<< @@ -15309,7 +15346,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 */ __pyx_v_sf->metric = __pyx_v_self->metric; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":109 + /* "/home/cdyer/cdec/python/src/mteval.pxi":109 * cdef SufficientStats sf = SufficientStats() * sf.metric = self.metric * sf.stats = new mteval.SufficientStats() # <<<<<<<<<<<<<< @@ -15318,7 +15355,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 */ __pyx_v_sf->stats = new SufficientStats(); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":110 + /* "/home/cdyer/cdec/python/src/mteval.pxi":110 * sf.metric = self.metric * sf.stats = new mteval.SufficientStats() * ConvertSentence(string(as_str(sentence.strip())), &hyp) # <<<<<<<<<<<<<< @@ -15333,7 +15370,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_2, NULL)), (&__pyx_v_hyp)); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":111 + /* "/home/cdyer/cdec/python/src/mteval.pxi":111 * sf.stats = new mteval.SufficientStats() * ConvertSentence(string(as_str(sentence.strip())), &hyp) * self.scorer.get().Evaluate(hyp, sf.stats) # <<<<<<<<<<<<<< @@ -15342,7 +15379,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5 */ __pyx_v_self->scorer->get()->Evaluate(__pyx_v_hyp, __pyx_v_sf->stats); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":112 + /* "/home/cdyer/cdec/python/src/mteval.pxi":112 * ConvertSentence(string(as_str(sentence.strip())), &hyp) * self.scorer.get().Evaluate(hyp, sf.stats) * return sf # <<<<<<<<<<<<<< @@ -15379,7 +15416,7 @@ static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_5candidate_set(PyObject *__p return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":114 +/* "/home/cdyer/cdec/python/src/mteval.pxi":114 * return sf * * def candidate_set(self): # <<<<<<<<<<<<<< @@ -15397,7 +15434,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("candidate_set", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":115 + /* "/home/cdyer/cdec/python/src/mteval.pxi":115 * * def candidate_set(self): * return CandidateSet(self) # <<<<<<<<<<<<<< @@ -15434,14 +15471,14 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_ static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_name = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__name,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__name,0}; PyObject* values[1] = {0}; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":121 + /* "/home/cdyer/cdec/python/src/mteval.pxi":121 * cdef mteval.EvaluationMetric* metric * * def __cinit__(self, bytes name=None): # <<<<<<<<<<<<<< @@ -15505,7 +15542,7 @@ static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":122 + /* "/home/cdyer/cdec/python/src/mteval.pxi":122 * * def __cinit__(self, bytes name=None): * if name: # <<<<<<<<<<<<<< @@ -15515,7 +15552,7 @@ static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__p __pyx_t_1 = (((PyObject *)__pyx_v_name) != Py_None) && (PyBytes_GET_SIZE(((PyObject *)__pyx_v_name)) != 0); if (__pyx_t_1) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":123 + /* "/home/cdyer/cdec/python/src/mteval.pxi":123 * def __cinit__(self, bytes name=None): * if name: * self.name = new string(name) # <<<<<<<<<<<<<< @@ -15525,7 +15562,7 @@ static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__p __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_name)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_self->name = new std::string(__pyx_t_2); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":124 + /* "/home/cdyer/cdec/python/src/mteval.pxi":124 * if name: * self.name = new string(name) * self.metric = mteval.MetricInstance(self.name[0]) # <<<<<<<<<<<<<< @@ -15556,7 +15593,7 @@ static void __pyx_pw_5_cdec_6Scorer_3__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyFinishContext(); } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":126 +/* "/home/cdyer/cdec/python/src/mteval.pxi":126 * self.metric = mteval.MetricInstance(self.name[0]) * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -15568,7 +15605,7 @@ static void __pyx_pf_5_cdec_6Scorer_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_ __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":127 + /* "/home/cdyer/cdec/python/src/mteval.pxi":127 * * def __dealloc__(self): * del self.name # <<<<<<<<<<<<<< @@ -15584,11 +15621,11 @@ static void __pyx_pf_5_cdec_6Scorer_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_refs = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__call__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -15601,7 +15638,8 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObj kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -15627,7 +15665,7 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":129 +/* "/home/cdyer/cdec/python/src/mteval.pxi":129 * del self.name * * def __call__(self, refs): # <<<<<<<<<<<<<< @@ -15656,7 +15694,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score __Pyx_RefNannySetupContext("__call__", 0); __Pyx_INCREF(__pyx_v_refs); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":130 + /* "/home/cdyer/cdec/python/src/mteval.pxi":130 * * def __call__(self, refs): * if isinstance(refs, unicode) or isinstance(refs, str): # <<<<<<<<<<<<<< @@ -15678,7 +15716,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score } if (__pyx_t_4) { - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":131 + /* "/home/cdyer/cdec/python/src/mteval.pxi":131 * def __call__(self, refs): * if isinstance(refs, unicode) or isinstance(refs, str): * refs = [refs] # <<<<<<<<<<<<<< @@ -15697,7 +15735,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score } __pyx_L3:; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":132 + /* "/home/cdyer/cdec/python/src/mteval.pxi":132 * if isinstance(refs, unicode) or isinstance(refs, str): * refs = [refs] * cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]() # <<<<<<<<<<<<<< @@ -15706,7 +15744,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ __pyx_v_refsv = new std::vector >(); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":135 + /* "/home/cdyer/cdec/python/src/mteval.pxi":135 * cdef vector[WordID]* refv * cdef bytes ref_str * for ref in refs: # <<<<<<<<<<<<<< @@ -15724,18 +15762,10 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score for (;;) { if (!__pyx_t_6 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_7 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; - #else - __pyx_t_7 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_6 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++; - #else - __pyx_t_7 = PySequence_ITEM(__pyx_t_1, __pyx_t_5); __pyx_t_5++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_7 = __pyx_t_6(__pyx_t_1); if (unlikely(!__pyx_t_7)) { @@ -15751,7 +15781,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score __pyx_v_ref = __pyx_t_7; __pyx_t_7 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":136 + /* "/home/cdyer/cdec/python/src/mteval.pxi":136 * cdef bytes ref_str * for ref in refs: * refv = new vector[WordID]() # <<<<<<<<<<<<<< @@ -15760,7 +15790,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ __pyx_v_refv = new std::vector(); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":137 + /* "/home/cdyer/cdec/python/src/mteval.pxi":137 * for ref in refs: * refv = new vector[WordID]() * ConvertSentence(string(as_str(ref.strip())), refv) # <<<<<<<<<<<<<< @@ -15775,7 +15805,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_8, NULL)), __pyx_v_refv); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":138 + /* "/home/cdyer/cdec/python/src/mteval.pxi":138 * refv = new vector[WordID]() * ConvertSentence(string(as_str(ref.strip())), refv) * refsv.push_back(refv[0]) # <<<<<<<<<<<<<< @@ -15784,7 +15814,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ __pyx_v_refsv->push_back((__pyx_v_refv[0])); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":139 + /* "/home/cdyer/cdec/python/src/mteval.pxi":139 * ConvertSentence(string(as_str(ref.strip())), refv) * refsv.push_back(refv[0]) * del refv # <<<<<<<<<<<<<< @@ -15795,7 +15825,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":141 + /* "/home/cdyer/cdec/python/src/mteval.pxi":141 * del refv * cdef unsigned i * cdef SegmentEvaluator evaluator = SegmentEvaluator() # <<<<<<<<<<<<<< @@ -15807,7 +15837,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score __pyx_v_evaluator = ((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":142 + /* "/home/cdyer/cdec/python/src/mteval.pxi":142 * cdef unsigned i * cdef SegmentEvaluator evaluator = SegmentEvaluator() * evaluator.metric = self.metric # <<<<<<<<<<<<<< @@ -15816,7 +15846,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ __pyx_v_evaluator->metric = __pyx_v_self->metric; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":143 + /* "/home/cdyer/cdec/python/src/mteval.pxi":143 * cdef SegmentEvaluator evaluator = SegmentEvaluator() * evaluator.metric = self.metric * evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator]( # <<<<<<<<<<<<<< @@ -15825,7 +15855,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ __pyx_v_evaluator->scorer = new boost::shared_ptr(__pyx_v_self->metric->CreateSegmentEvaluator((__pyx_v_refsv[0]))); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":145 + /* "/home/cdyer/cdec/python/src/mteval.pxi":145 * evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator]( * self.metric.CreateSegmentEvaluator(refsv[0])) * del refsv # in theory should not delete but store in SegmentEvaluator # <<<<<<<<<<<<<< @@ -15834,7 +15864,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score */ delete __pyx_v_refsv; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":146 + /* "/home/cdyer/cdec/python/src/mteval.pxi":146 * self.metric.CreateSegmentEvaluator(refsv[0])) * del refsv # in theory should not delete but store in SegmentEvaluator * return evaluator # <<<<<<<<<<<<<< @@ -15874,7 +15904,7 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_7__str__(PyObject *__pyx_v_self) { return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":148 +/* "/home/cdyer/cdec/python/src/mteval.pxi":148 * return evaluator * * def __str__(self): # <<<<<<<<<<<<<< @@ -15891,7 +15921,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_6__str__(struct __pyx_obj_5_cdec_Scorer int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__str__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":149 + /* "/home/cdyer/cdec/python/src/mteval.pxi":149 * * def __str__(self): * return self.name.c_str() # <<<<<<<<<<<<<< @@ -15917,7 +15947,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_6__str__(struct __pyx_obj_5_cdec_Scorer return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":151 +/* "/home/cdyer/cdec/python/src/mteval.pxi":151 * return self.name.c_str() * * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats): # <<<<<<<<<<<<<< @@ -15943,7 +15973,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat int __pyx_clineno = 0; __Pyx_RefNannySetupContext("_compute_score", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":152 + /* "/home/cdyer/cdec/python/src/mteval.pxi":152 * * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats): * cdef Metric metric = metric_ # <<<<<<<<<<<<<< @@ -15953,7 +15983,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat __Pyx_INCREF(((PyObject *)((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_))); __pyx_v_metric = ((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":153 + /* "/home/cdyer/cdec/python/src/mteval.pxi":153 * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats): * cdef Metric metric = metric_ * cdef list ss = [] # <<<<<<<<<<<<<< @@ -15965,7 +15995,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat __pyx_v_ss = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":155 + /* "/home/cdyer/cdec/python/src/mteval.pxi":155 * cdef list ss = [] * cdef unsigned i * for i in range(stats.size()): # <<<<<<<<<<<<<< @@ -15976,7 +16006,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":156 + /* "/home/cdyer/cdec/python/src/mteval.pxi":156 * cdef unsigned i * for i in range(stats.size()): * ss.append(stats[0][i]) # <<<<<<<<<<<<<< @@ -15989,7 +16019,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":157 + /* "/home/cdyer/cdec/python/src/mteval.pxi":157 * for i in range(stats.size()): * ss.append(stats[0][i]) * return metric.score(ss) # <<<<<<<<<<<<<< @@ -16027,7 +16057,7 @@ static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStat return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":159 +/* "/home/cdyer/cdec/python/src/mteval.pxi":159 * return metric.score(ss) * * cdef void _compute_sufficient_stats(void* metric_, # <<<<<<<<<<<<<< @@ -16054,7 +16084,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: int __pyx_clineno = 0; __Pyx_RefNannySetupContext("_compute_sufficient_stats", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":163 + /* "/home/cdyer/cdec/python/src/mteval.pxi":163 * vector[string]* refs, * mteval.SufficientStats* out): * cdef Metric metric = metric_ # <<<<<<<<<<<<<< @@ -16064,7 +16094,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: __Pyx_INCREF(((PyObject *)((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_))); __pyx_v_metric = ((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":164 + /* "/home/cdyer/cdec/python/src/mteval.pxi":164 * mteval.SufficientStats* out): * cdef Metric metric = metric_ * cdef list refs_ = [] # <<<<<<<<<<<<<< @@ -16076,7 +16106,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: __pyx_v_refs_ = __pyx_t_1; __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":166 + /* "/home/cdyer/cdec/python/src/mteval.pxi":166 * cdef list refs_ = [] * cdef unsigned i * for i in range(refs.size()): # <<<<<<<<<<<<<< @@ -16087,7 +16117,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":167 + /* "/home/cdyer/cdec/python/src/mteval.pxi":167 * cdef unsigned i * for i in range(refs.size()): * refs_.append(refs[0][i].c_str()) # <<<<<<<<<<<<<< @@ -16100,7 +16130,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; } - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":168 + /* "/home/cdyer/cdec/python/src/mteval.pxi":168 * for i in range(refs.size()): * refs_.append(refs[0][i].c_str()) * cdef list ss = metric.evaluate(hyp.c_str(), refs_) # <<<<<<<<<<<<<< @@ -16127,7 +16157,7 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: __pyx_v_ss = ((PyObject*)__pyx_t_5); __pyx_t_5 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":169 + /* "/home/cdyer/cdec/python/src/mteval.pxi":169 * refs_.append(refs[0][i].c_str()) * cdef list ss = metric.evaluate(hyp.c_str(), refs_) * out.fields.resize(len(ss)) # <<<<<<<<<<<<<< @@ -16135,13 +16165,12 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: * out.fields[i] = ss[i] */ if (unlikely(((PyObject *)__pyx_v_ss) == Py_None)) { - PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - {__pyx_filename = __pyx_f[5]; __pyx_lineno = 169; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 169; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 169; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); __pyx_v_out->fields.resize(__pyx_t_7); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":170 + /* "/home/cdyer/cdec/python/src/mteval.pxi":170 * cdef list ss = metric.evaluate(hyp.c_str(), refs_) * out.fields.resize(len(ss)) * for i in range(len(ss)): # <<<<<<<<<<<<<< @@ -16149,24 +16178,19 @@ static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std: * */ if (unlikely(((PyObject *)__pyx_v_ss) == Py_None)) { - PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); - {__pyx_filename = __pyx_f[5]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } - __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); if (unlikely(__pyx_t_7 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_7; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":171 + /* "/home/cdyer/cdec/python/src/mteval.pxi":171 * out.fields.resize(len(ss)) * for i in range(len(ss)): * out.fields[i] = ss[i] # <<<<<<<<<<<<<< * * cdef class Metric: */ - if (unlikely(((PyObject *)__pyx_v_ss) == Py_None)) { - PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); - {__pyx_filename = __pyx_f[5]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } __pyx_t_5 = __Pyx_GetItemInt_List(((PyObject *)__pyx_v_ss), __pyx_v_i, sizeof(unsigned int)+1, PyLong_FromUnsignedLong); if (!__pyx_t_5) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __pyx_t_8 = __pyx_PyFloat_AsFloat(__pyx_t_5); if (unlikely((__pyx_t_8 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -16201,7 +16225,7 @@ static int __pyx_pw_5_cdec_6Metric_1__cinit__(PyObject *__pyx_v_self, PyObject * return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":175 +/* "/home/cdyer/cdec/python/src/mteval.pxi":175 * cdef class Metric: * cdef Scorer scorer * def __cinit__(self): # <<<<<<<<<<<<<< @@ -16219,7 +16243,7 @@ static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__p int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":176 + /* "/home/cdyer/cdec/python/src/mteval.pxi":176 * cdef Scorer scorer * def __cinit__(self): * self.scorer = Scorer() # <<<<<<<<<<<<<< @@ -16234,7 +16258,7 @@ static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__p __pyx_v_self->scorer = ((struct __pyx_obj_5_cdec_Scorer *)__pyx_t_1); __pyx_t_1 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":177 + /* "/home/cdyer/cdec/python/src/mteval.pxi":177 * def __cinit__(self): * self.scorer = Scorer() * self.scorer.name = new string(as_str(self.__class__.__name__)) # <<<<<<<<<<<<<< @@ -16249,7 +16273,7 @@ static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__p __pyx_v_self->scorer->name = new std::string(__pyx_f_5_cdec_as_str(__pyx_t_2, NULL)); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":178 + /* "/home/cdyer/cdec/python/src/mteval.pxi":178 * self.scorer = Scorer() * self.scorer.name = new string(as_str(self.__class__.__name__)) * self.scorer.metric = mteval.PyMetricInstance(self.scorer.name[0], # <<<<<<<<<<<<<< @@ -16274,11 +16298,11 @@ static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__p static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_refs = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__call__ (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0}; PyObject* values[1] = {0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -16291,7 +16315,8 @@ static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObj kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; } if (unlikely(kw_args > 0)) { @@ -16317,7 +16342,7 @@ static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":181 +/* "/home/cdyer/cdec/python/src/mteval.pxi":181 * self, _compute_sufficient_stats, _compute_score) * * def __call__(self, refs): # <<<<<<<<<<<<<< @@ -16335,7 +16360,7 @@ static PyObject *__pyx_pf_5_cdec_6Metric_2__call__(struct __pyx_obj_5_cdec_Metri int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__call__", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":182 + /* "/home/cdyer/cdec/python/src/mteval.pxi":182 * * def __call__(self, refs): * return self.scorer(refs) # <<<<<<<<<<<<<< @@ -16379,7 +16404,7 @@ static PyObject *__pyx_pw_5_cdec_6Metric_5score(PyObject *__pyx_v_stats, CYTHON_ return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":184 +/* "/home/cdyer/cdec/python/src/mteval.pxi":184 * return self.scorer(refs) * * def score(SufficientStats stats): # <<<<<<<<<<<<<< @@ -16392,7 +16417,7 @@ static PyObject *__pyx_pf_5_cdec_6Metric_4score(CYTHON_UNUSED struct __pyx_obj_5 __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("score", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":185 + /* "/home/cdyer/cdec/python/src/mteval.pxi":185 * * def score(SufficientStats stats): * return 0 # <<<<<<<<<<<<<< @@ -16416,11 +16441,11 @@ static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObj static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { CYTHON_UNUSED PyObject *__pyx_v_hyp = 0; CYTHON_UNUSED PyObject *__pyx_v_refs = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hyp,&__pyx_n_s__refs,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("evaluate (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hyp,&__pyx_n_s__refs,0}; PyObject* values[2] = {0,0}; if (unlikely(__pyx_kwds)) { Py_ssize_t kw_args; @@ -16434,10 +16459,12 @@ static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObj kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hyp)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hyp); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs)) != 0)) kw_args--; + values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs); + if (likely(values[1])) kw_args--; else { __Pyx_RaiseArgtupleInvalid("evaluate", 1, 2, 2, 1); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } @@ -16467,7 +16494,7 @@ static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObj return __pyx_r; } -/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":187 +/* "/home/cdyer/cdec/python/src/mteval.pxi":187 * return 0 * * def evaluate(self, hyp, refs): # <<<<<<<<<<<<<< @@ -16484,7 +16511,7 @@ static PyObject *__pyx_pf_5_cdec_6Metric_6evaluate(CYTHON_UNUSED struct __pyx_ob int __pyx_clineno = 0; __Pyx_RefNannySetupContext("evaluate", 0); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":188 + /* "/home/cdyer/cdec/python/src/mteval.pxi":188 * * def evaluate(self, hyp, refs): * return [] # <<<<<<<<<<<<<< @@ -16509,7 +16536,7 @@ static PyObject *__pyx_pf_5_cdec_6Metric_6evaluate(CYTHON_UNUSED struct __pyx_ob __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_4generator17(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* Python wrapper */ static PyObject *__pyx_pw_5_cdec_3_make_config(PyObject *__pyx_self, PyObject *__pyx_v_config); /*proto*/ @@ -16518,6 +16545,7 @@ static PyObject *__pyx_pw_5_cdec_3_make_config(PyObject *__pyx_self, PyObject *_ PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("_make_config (wrapper)", 0); + __pyx_self = __pyx_self; __pyx_r = __pyx_pf_5_cdec_2_make_config(__pyx_self, ((PyObject *)__pyx_v_config)); __Pyx_RefNannyFinishContext(); return __pyx_r; @@ -16532,14 +16560,14 @@ static PyObject *__pyx_pw_5_cdec_3_make_config(PyObject *__pyx_self, PyObject *_ */ static PyObject *__pyx_pf_5_cdec_2_make_config(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_config) { - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("_make_config", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)__pyx_ptype_5_cdec___pyx_scope_struct_22__make_config->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_22__make_config, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)__pyx_ptype_5_cdec___pyx_scope_struct_23__make_config->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_23__make_config, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; @@ -16549,7 +16577,7 @@ static PyObject *__pyx_pf_5_cdec_2_make_config(CYTHON_UNUSED PyObject *__pyx_sel __Pyx_INCREF(__pyx_cur_scope->__pyx_v_config); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_config); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_4generator16, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_4generator17, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -16567,9 +16595,9 @@ static PyObject *__pyx_pf_5_cdec_2_make_config(CYTHON_UNUSED PyObject *__pyx_sel return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_4generator17(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)__pyx_generator->closure); PyObject *__pyx_r = NULL; PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; @@ -16621,18 +16649,10 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener for (;;) { if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_3); __Pyx_INCREF(__pyx_t_2); __pyx_t_3++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_2 = __pyx_t_4(__pyx_t_1); if (unlikely(!__pyx_t_2)) { @@ -16646,33 +16666,27 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener } if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { PyObject* sequence = __pyx_t_2; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_6 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyList_GET_ITEM(sequence, 0); __pyx_t_6 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_6); - #else - __pyx_t_5 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_6 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_7 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); @@ -16683,13 +16697,12 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener index = 1; __pyx_t_6 = __pyx_t_8(__pyx_t_7); if (unlikely(!__pyx_t_6)) goto __pyx_L6_unpacking_failed; __Pyx_GOTREF(__pyx_t_6); if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_7), 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_8 = NULL; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L7_unpacking_done; __pyx_L6_unpacking_failed:; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_8 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L7_unpacking_done:; } @@ -16741,18 +16754,10 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener for (;;) { if (!__pyx_t_11 && PyList_CheckExact(__pyx_t_2)) { if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_11 && PyTuple_CheckExact(__pyx_t_2)) { if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_6 = __pyx_t_11(__pyx_t_2); if (unlikely(!__pyx_t_6)) { @@ -16766,33 +16771,27 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener } if ((likely(PyTuple_CheckExact(__pyx_t_6))) || (PyList_CheckExact(__pyx_t_6))) { PyObject* sequence = __pyx_t_6; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_5 = PyList_GET_ITEM(sequence, 0); __pyx_t_7 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_7); - #else - __pyx_t_5 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_12 = PyObject_GetIter(__pyx_t_6); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_12); @@ -16803,13 +16802,12 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener index = 1; __pyx_t_7 = __pyx_t_8(__pyx_t_12); if (unlikely(!__pyx_t_7)) goto __pyx_L11_unpacking_failed; __Pyx_GOTREF(__pyx_t_7); if (__Pyx_IternextUnpackEndCheck(__pyx_t_8(__pyx_t_12), 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_8 = NULL; __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; goto __pyx_L12_unpacking_done; __pyx_L11_unpacking_failed:; __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; - __pyx_t_8 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L12_unpacking_done:; } @@ -16913,18 +16911,10 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener for (;;) { if (!__pyx_t_11 && PyList_CheckExact(__pyx_t_2)) { if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_11 && PyTuple_CheckExact(__pyx_t_2)) { if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_6 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_10); __Pyx_INCREF(__pyx_t_6); __pyx_t_10++; - #else - __pyx_t_6 = PySequence_ITEM(__pyx_t_2, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_6 = __pyx_t_11(__pyx_t_2); if (unlikely(!__pyx_t_6)) { @@ -17048,7 +17038,6 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -17058,13 +17047,13 @@ static int __pyx_pw_5_cdec_7Decoder_1__cinit__(PyObject *__pyx_v_self, PyObject static int __pyx_pw_5_cdec_7Decoder_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_config_str = 0; PyObject *__pyx_v_config = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__config_str,0}; int __pyx_r; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0); __pyx_v_config = PyDict_New(); if (unlikely(!__pyx_v_config)) return -1; __Pyx_GOTREF(__pyx_v_config); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__config_str,0}; PyObject* values[1] = {0}; /* "_cdec.pyx":43 @@ -17117,7 +17106,7 @@ static int __pyx_pw_5_cdec_7Decoder_1__cinit__(PyObject *__pyx_v_self, PyObject __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ +static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator21(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */ /* "_cdec.pyx":54 * 'csplit', 'tagger', 'lexalign'): @@ -17128,24 +17117,24 @@ static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_Generato */ static PyObject *__pyx_pf_5_cdec_7Decoder_9__cinit___genexpr(PyObject *__pyx_self) { - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *__pyx_cur_scope; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations int __pyx_lineno = 0; const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("genexpr", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)__pyx_ptype_5_cdec___pyx_scope_struct_24_genexpr->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_24_genexpr, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)__pyx_ptype_5_cdec___pyx_scope_struct_25_genexpr->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_25_genexpr, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return NULL; } __Pyx_GOTREF(__pyx_cur_scope); - __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *) __pyx_self; + __pyx_cur_scope->__pyx_outer_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *) __pyx_self; __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope)); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope); { - __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Decoder_9__cinit___2generator20, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Decoder_9__cinit___2generator21, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_cur_scope); __Pyx_RefNannyFinishContext(); return (PyObject *) gen; @@ -17163,9 +17152,9 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_9__cinit___genexpr(PyObject *__pyx_sel return __pyx_r; } -static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ +static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator21(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */ { - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)__pyx_generator->closure); + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)__pyx_generator->closure); PyObject *__pyx_r = NULL; PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; @@ -17207,18 +17196,10 @@ static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_Generato for (;;) { if (!__pyx_t_5 && PyList_CheckExact(__pyx_t_2)) { if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_3); __pyx_t_4++; - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_5 && PyTuple_CheckExact(__pyx_t_2)) { if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_2)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_4); __Pyx_INCREF(__pyx_t_3); __pyx_t_4++; - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_2, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_3 = __pyx_t_5(__pyx_t_2); if (unlikely(!__pyx_t_3)) { @@ -17267,7 +17248,6 @@ static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_Generato __pyx_L0:; __Pyx_XDECREF(__pyx_r); __pyx_generator->resume_label = -1; - __Pyx_Generator_clear((PyObject*)__pyx_generator); __Pyx_RefNannyFinishContext(); return NULL; } @@ -17281,7 +17261,7 @@ static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_Generato */ static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *__pyx_v_self, PyObject *__pyx_v_config_str, PyObject *__pyx_v_config) { - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *__pyx_cur_scope; + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *__pyx_cur_scope; PyObject *__pyx_v_formalism = NULL; std::istringstream *__pyx_v_config_stream; int __pyx_r; @@ -17297,7 +17277,7 @@ static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *_ const char *__pyx_filename = NULL; int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__cinit__", 0); - __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)__pyx_ptype_5_cdec___pyx_scope_struct_23___cinit__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_23___cinit__, __pyx_empty_tuple, NULL); + __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)__pyx_ptype_5_cdec___pyx_scope_struct_24___cinit__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_24___cinit__, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __Pyx_RefNannyFinishContext(); return -1; @@ -17732,18 +17712,10 @@ static int __pyx_pf_5_cdec_7Decoder_7weights_2__set__(struct __pyx_obj_5_cdec_De for (;;) { if (!__pyx_t_5 && PyList_CheckExact(__pyx_t_1)) { if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_3); __pyx_t_4++; - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else if (!__pyx_t_5 && PyTuple_CheckExact(__pyx_t_1)) { if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_3); __pyx_t_4++; - #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #endif } else { __pyx_t_3 = __pyx_t_5(__pyx_t_1); if (unlikely(!__pyx_t_3)) { @@ -17757,33 +17729,27 @@ static int __pyx_pf_5_cdec_7Decoder_7weights_2__set__(struct __pyx_obj_5_cdec_De } if ((likely(PyTuple_CheckExact(__pyx_t_3))) || (PyList_CheckExact(__pyx_t_3))) { PyObject* sequence = __pyx_t_3; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + } __pyx_t_6 = PyList_GET_ITEM(sequence, 0); __pyx_t_7 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(__pyx_t_7); - #else - __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - #endif __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else - { + } else { Py_ssize_t index = -1; __pyx_t_8 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_8); @@ -17794,13 +17760,12 @@ static int __pyx_pf_5_cdec_7Decoder_7weights_2__set__(struct __pyx_obj_5_cdec_De index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L6_unpacking_failed; __Pyx_GOTREF(__pyx_t_7); if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_t_9 = NULL; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; goto __pyx_L7_unpacking_done; __pyx_L6_unpacking_failed:; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_9 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_L7_unpacking_done:; } @@ -17881,48 +17846,37 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_9formalism_1__get__(PyObject *__pyx_v_ * property formalism: * def __get__(self): # <<<<<<<<<<<<<< * cdef variables_map* conf = &self.dec.GetConf() - * return conf[0]['formalism'].as_str() + * #return conf[0]['formalism'] */ static PyObject *__pyx_pf_5_cdec_7Decoder_9formalism___get__(struct __pyx_obj_5_cdec_Decoder *__pyx_v_self) { - const boost::program_options::variables_map *__pyx_v_conf; + CYTHON_UNUSED const boost::program_options::variables_map *__pyx_v_conf; PyObject *__pyx_r = NULL; __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; __Pyx_RefNannySetupContext("__get__", 0); /* "_cdec.pyx":84 * property formalism: * def __get__(self): * cdef variables_map* conf = &self.dec.GetConf() # <<<<<<<<<<<<<< - * return conf[0]['formalism'].as_str() - * + * #return conf[0]['formalism'] + * return None */ __pyx_v_conf = (&__pyx_v_self->dec->GetConf()); - /* "_cdec.pyx":85 - * def __get__(self): + /* "_cdec.pyx":86 * cdef variables_map* conf = &self.dec.GetConf() - * return conf[0]['formalism'].as_str() # <<<<<<<<<<<<<< + * #return conf[0]['formalism'] + * return None # <<<<<<<<<<<<<< * * def read_weights(self, weights): */ __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_string_to_py_(((__pyx_v_conf[0])[__pyx_k__formalism]).as()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __pyx_r = ((PyObject *)__pyx_t_1); - __pyx_t_1 = 0; + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; goto __pyx_L0; __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("_cdec.Decoder.formalism.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = NULL; __pyx_L0:; __Pyx_XGIVEREF(__pyx_r); __Pyx_RefNannyFinishContext(); @@ -17940,8 +17894,8 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_5read_weights(PyObject *__pyx_v_self, return __pyx_r; } -/* "_cdec.pyx":87 - * return conf[0]['formalism'].as_str() +/* "_cdec.pyx":88 + * return None * * def read_weights(self, weights): # <<<<<<<<<<<<<< * with open(weights) as fp: @@ -17976,7 +17930,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ int __pyx_clineno = 0; __Pyx_RefNannySetupContext("read_weights", 0); - /* "_cdec.pyx":88 + /* "_cdec.pyx":89 * * def read_weights(self, weights): * with open(weights) as fp: # <<<<<<<<<<<<<< @@ -17984,19 +17938,19 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ * if line.strip().startswith('#'): continue */ /*with:*/ { - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_weights); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_weights); __Pyx_GIVEREF(__pyx_v_weights); - __pyx_t_2 = PyObject_Call(__pyx_builtin_open, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_2 = PyObject_Call(__pyx_builtin_open, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - __pyx_t_3 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s____exit__); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_3 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s____exit__); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s____enter__); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_t_2, __pyx_n_s____enter__); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __pyx_t_4 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -18011,7 +17965,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __pyx_v_fp = __pyx_t_4; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "_cdec.pyx":89 + /* "_cdec.pyx":90 * def read_weights(self, weights): * with open(weights) as fp: * for line in fp: # <<<<<<<<<<<<<< @@ -18022,31 +17976,23 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __pyx_t_4 = __pyx_v_fp; __Pyx_INCREF(__pyx_t_4); __pyx_t_8 = 0; __pyx_t_9 = NULL; } else { - __pyx_t_8 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_v_fp); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_8 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_v_fp); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_4); __pyx_t_9 = Py_TYPE(__pyx_t_4)->tp_iternext; } for (;;) { if (!__pyx_t_9 && PyList_CheckExact(__pyx_t_4)) { if (__pyx_t_8 >= PyList_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_t_8++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_4, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L7_error;}; - #endif } else if (!__pyx_t_9 && PyTuple_CheckExact(__pyx_t_4)) { if (__pyx_t_8 >= PyTuple_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_t_8++; - #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_4, __pyx_t_8); __pyx_t_8++; if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L7_error;}; - #endif } else { __pyx_t_2 = __pyx_t_9(__pyx_t_4); if (unlikely(!__pyx_t_2)) { if (PyErr_Occurred()) { if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear(); - else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + else {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} } break; } @@ -18056,25 +18002,25 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __pyx_v_line = __pyx_t_2; __pyx_t_2 = 0; - /* "_cdec.pyx":90 + /* "_cdec.pyx":91 * with open(weights) as fp: * for line in fp: * if line.strip().startswith('#'): continue # <<<<<<<<<<<<<< * fname, value = line.split() * self.weights[fname.strip()] = float(value) */ - __pyx_t_2 = PyObject_GetAttr(__pyx_v_line, __pyx_n_s__strip); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_v_line, __pyx_n_s__strip); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__startswith); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__startswith); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_51), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_51), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_10) { goto __pyx_L16_continue; @@ -18082,49 +18028,43 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ } __pyx_L18:; - /* "_cdec.pyx":91 + /* "_cdec.pyx":92 * for line in fp: * if line.strip().startswith('#'): continue * fname, value = line.split() # <<<<<<<<<<<<<< * self.weights[fname.strip()] = float(value) * */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_line, __pyx_n_s__split); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_line, __pyx_n_s__split); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if ((likely(PyTuple_CheckExact(__pyx_t_2))) || (PyList_CheckExact(__pyx_t_2))) { PyObject* sequence = __pyx_t_2; - #if CYTHON_COMPILING_IN_CPYTHON - Py_ssize_t size = Py_SIZE(sequence); - #else - Py_ssize_t size = PySequence_Size(sequence); - #endif - if (unlikely(size != 2)) { - if (size > 2) __Pyx_RaiseTooManyValuesError(2); - else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} - } - #if CYTHON_COMPILING_IN_CPYTHON if (likely(PyTuple_CheckExact(sequence))) { + if (unlikely(PyTuple_GET_SIZE(sequence) != 2)) { + if (PyTuple_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyTuple_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + } __pyx_t_1 = PyTuple_GET_ITEM(sequence, 0); __pyx_t_11 = PyTuple_GET_ITEM(sequence, 1); } else { + if (unlikely(PyList_GET_SIZE(sequence) != 2)) { + if (PyList_GET_SIZE(sequence) > 2) __Pyx_RaiseTooManyValuesError(2); + else __Pyx_RaiseNeedMoreValuesError(PyList_GET_SIZE(sequence)); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + } __pyx_t_1 = PyList_GET_ITEM(sequence, 0); __pyx_t_11 = PyList_GET_ITEM(sequence, 1); } __Pyx_INCREF(__pyx_t_1); __Pyx_INCREF(__pyx_t_11); - #else - __pyx_t_1 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} - __pyx_t_11 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} - #endif __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else - { + } else { Py_ssize_t index = -1; - __pyx_t_12 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_12 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_12)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_13 = Py_TYPE(__pyx_t_12)->tp_iternext; @@ -18132,15 +18072,14 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __Pyx_GOTREF(__pyx_t_1); index = 1; __pyx_t_11 = __pyx_t_13(__pyx_t_12); if (unlikely(!__pyx_t_11)) goto __pyx_L19_unpacking_failed; __Pyx_GOTREF(__pyx_t_11); - if (__Pyx_IternextUnpackEndCheck(__pyx_t_13(__pyx_t_12), 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} - __pyx_t_13 = NULL; + if (__Pyx_IternextUnpackEndCheck(__pyx_t_13(__pyx_t_12), 2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; goto __pyx_L20_unpacking_done; __pyx_L19_unpacking_failed:; __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; - __pyx_t_13 = NULL; - if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); + if (!PyErr_Occurred()) __Pyx_RaiseNeedMoreValuesError(index); + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __pyx_L20_unpacking_done:; } __Pyx_XDECREF(__pyx_v_fname); @@ -18150,22 +18089,22 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __pyx_v_value = __pyx_t_11; __pyx_t_11 = 0; - /* "_cdec.pyx":92 + /* "_cdec.pyx":93 * if line.strip().startswith('#'): continue * fname, value = line.split() * self.weights[fname.strip()] = float(value) # <<<<<<<<<<<<<< * * def translate(self, sentence, grammar=None): */ - __pyx_t_14 = __Pyx_PyObject_AsDouble(__pyx_v_value); if (unlikely(__pyx_t_14 == ((double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} - __pyx_t_2 = PyFloat_FromDouble(__pyx_t_14); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_14 = __Pyx_PyObject_AsDouble(__pyx_v_value); if (unlikely(__pyx_t_14 == ((double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_2 = PyFloat_FromDouble(__pyx_t_14); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_2); - __pyx_t_11 = PyObject_GetAttr(__pyx_v_fname, __pyx_n_s__strip); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_11 = PyObject_GetAttr(__pyx_v_fname, __pyx_n_s__strip); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_11); - __pyx_t_1 = PyObject_Call(__pyx_t_11, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + __pyx_t_1 = PyObject_Call(__pyx_t_11, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; - if (PyObject_SetItem(((PyObject *)__pyx_v_self->weights), __pyx_t_1, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L7_error;} + if (PyObject_SetItem(((PyObject *)__pyx_v_self->weights), __pyx_t_1, __pyx_t_2) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L7_error;} __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_L16_continue:; @@ -18183,7 +18122,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "_cdec.pyx":88 + /* "_cdec.pyx":89 * * def read_weights(self, weights): * with open(weights) as fp: # <<<<<<<<<<<<<< @@ -18192,11 +18131,11 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ */ /*except:*/ { __Pyx_AddTraceback("_cdec.Decoder.read_weights", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_2, &__pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} + if (__Pyx_GetException(&__pyx_t_4, &__pyx_t_2, &__pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} __Pyx_GOTREF(__pyx_t_4); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_1); - __pyx_t_11 = PyTuple_New(3); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} + __pyx_t_11 = PyTuple_New(3); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} __Pyx_GOTREF(__pyx_t_11); __Pyx_INCREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_4); @@ -18209,11 +18148,11 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __Pyx_GIVEREF(__pyx_t_1); __pyx_t_15 = PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (unlikely(!__pyx_t_15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} + if (unlikely(!__pyx_t_15)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} __Pyx_GOTREF(__pyx_t_15); __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_15); __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; - if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} + if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} __pyx_t_16 = (!__pyx_t_10); if (__pyx_t_16) { __Pyx_GIVEREF(__pyx_t_4); @@ -18221,7 +18160,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ __Pyx_GIVEREF(__pyx_t_1); __Pyx_ErrRestore(__pyx_t_4, __pyx_t_2, __pyx_t_1); __pyx_t_4 = 0; __pyx_t_2 = 0; __pyx_t_1 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L9_except_error;} goto __pyx_L23; } __pyx_L23:; @@ -18249,11 +18188,11 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_ if (__pyx_t_3) { __pyx_t_7 = PyObject_Call(__pyx_t_3, __pyx_k_tuple_52, NULL); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_7); __pyx_t_16 = __Pyx_PyObject_IsTrue(__pyx_t_7); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(__pyx_t_16 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (unlikely(__pyx_t_16 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } } goto __pyx_L24; @@ -18288,14 +18227,14 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_7translate(PyObject *__pyx_v_self, PyO static PyObject *__pyx_pw_5_cdec_7Decoder_7translate(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_sentence = 0; PyObject *__pyx_v_grammar = 0; + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__sentence,&__pyx_n_s__grammar,0}; PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("translate (wrapper)", 0); { - static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__sentence,&__pyx_n_s__grammar,0}; PyObject* values[2] = {0,0}; - /* "_cdec.pyx":94 + /* "_cdec.pyx":95 * self.weights[fname.strip()] = float(value) * * def translate(self, sentence, grammar=None): # <<<<<<<<<<<<<< @@ -18315,7 +18254,8 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_7translate(PyObject *__pyx_v_self, PyO kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__sentence)) != 0)) kw_args--; + values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__sentence); + if (likely(values[0])) kw_args--; else goto __pyx_L5_argtuple_error; case 1: if (kw_args > 0) { @@ -18324,7 +18264,7 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_7translate(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "translate") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "translate") < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;} } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -18339,7 +18279,7 @@ static PyObject *__pyx_pw_5_cdec_7Decoder_7translate(PyObject *__pyx_v_self, PyO } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("translate", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L3_error;} + __Pyx_RaiseArgtupleInvalid("translate", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[0]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;} __pyx_L3_error:; __Pyx_AddTraceback("_cdec.Decoder.translate", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -18367,7 +18307,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec int __pyx_clineno = 0; __Pyx_RefNannySetupContext("translate", 0); - /* "_cdec.pyx":96 + /* "_cdec.pyx":97 * def translate(self, sentence, grammar=None): * cdef bytes input_str * if isinstance(sentence, unicode) or isinstance(sentence, str): # <<<<<<<<<<<<<< @@ -18389,19 +18329,19 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec } if (__pyx_t_4) { - /* "_cdec.pyx":97 + /* "_cdec.pyx":98 * cdef bytes input_str * if isinstance(sentence, unicode) or isinstance(sentence, str): * input_str = as_str(sentence.strip()) # <<<<<<<<<<<<<< * elif isinstance(sentence, Lattice): * input_str = str(sentence) # PLF format */ - __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__strip); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__strip); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = PyBytes_FromString(__pyx_f_5_cdec_as_str(__pyx_t_5, NULL)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 97; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyBytes_FromString(__pyx_f_5_cdec_as_str(__pyx_t_5, NULL)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_1)); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_input_str = __pyx_t_1; @@ -18409,7 +18349,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec goto __pyx_L3; } - /* "_cdec.pyx":98 + /* "_cdec.pyx":99 * if isinstance(sentence, unicode) or isinstance(sentence, str): * input_str = as_str(sentence.strip()) * elif isinstance(sentence, Lattice): # <<<<<<<<<<<<<< @@ -18422,62 +18362,62 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_4) { - /* "_cdec.pyx":99 + /* "_cdec.pyx":100 * input_str = as_str(sentence.strip()) * elif isinstance(sentence, Lattice): * input_str = str(sentence) # PLF format # <<<<<<<<<<<<<< * else: * raise TypeError('Cannot translate input type %s' % type(sentence)) */ - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_sentence); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_sentence); __Pyx_GIVEREF(__pyx_v_sentence); - __pyx_t_5 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; - if (!(likely(PyBytes_CheckExact(__pyx_t_5))||((__pyx_t_5) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_5)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (!(likely(PyBytes_CheckExact(__pyx_t_5))||((__pyx_t_5) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_5)->tp_name), 0))) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_input_str = ((PyObject*)__pyx_t_5); __pyx_t_5 = 0; goto __pyx_L3; } /*else*/ { - /* "_cdec.pyx":101 + /* "_cdec.pyx":102 * input_str = str(sentence) # PLF format * else: * raise TypeError('Cannot translate input type %s' % type(sentence)) # <<<<<<<<<<<<<< * if grammar: * if isinstance(grammar, str) or isinstance(grammar, unicode): */ - __pyx_t_5 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_53), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_53), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(((PyObject *)__pyx_t_5)); - __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_t_5)); __Pyx_GIVEREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; - __pyx_t_5 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 101; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} } __pyx_L3:; - /* "_cdec.pyx":102 + /* "_cdec.pyx":103 * else: * raise TypeError('Cannot translate input type %s' % type(sentence)) * if grammar: # <<<<<<<<<<<<<< * if isinstance(grammar, str) or isinstance(grammar, unicode): * self.dec.AddSupplementalGrammarFromString(string(as_str(grammar))) */ - __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_grammar); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_v_grammar); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__pyx_t_4) { - /* "_cdec.pyx":103 + /* "_cdec.pyx":104 * raise TypeError('Cannot translate input type %s' % type(sentence)) * if grammar: * if isinstance(grammar, str) or isinstance(grammar, unicode): # <<<<<<<<<<<<<< @@ -18499,7 +18439,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec } if (__pyx_t_3) { - /* "_cdec.pyx":104 + /* "_cdec.pyx":105 * if grammar: * if isinstance(grammar, str) or isinstance(grammar, unicode): * self.dec.AddSupplementalGrammarFromString(string(as_str(grammar))) # <<<<<<<<<<<<<< @@ -18511,19 +18451,19 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec } /*else*/ { - /* "_cdec.pyx":106 + /* "_cdec.pyx":107 * self.dec.AddSupplementalGrammarFromString(string(as_str(grammar))) * else: * self.dec.AddSupplementalGrammar(TextGrammar(grammar).grammar[0]) # <<<<<<<<<<<<<< * cdef decoder.BasicObserver observer = decoder.BasicObserver() * self.dec.Decode(string(input_str), &observer) */ - __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_v_grammar); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_v_grammar); __Pyx_GIVEREF(__pyx_v_grammar); - __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_TextGrammar)), ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_TextGrammar)), ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0; __pyx_v_self->dec->AddSupplementalGrammar((((struct __pyx_obj_5_cdec_TextGrammar *)__pyx_t_1)->__pyx_base.grammar[0])); @@ -18534,7 +18474,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec } __pyx_L4:; - /* "_cdec.pyx":107 + /* "_cdec.pyx":108 * else: * self.dec.AddSupplementalGrammar(TextGrammar(grammar).grammar[0]) * cdef decoder.BasicObserver observer = decoder.BasicObserver() # <<<<<<<<<<<<<< @@ -18543,17 +18483,17 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec */ __pyx_v_observer = BasicObserver(); - /* "_cdec.pyx":108 + /* "_cdec.pyx":109 * self.dec.AddSupplementalGrammar(TextGrammar(grammar).grammar[0]) * cdef decoder.BasicObserver observer = decoder.BasicObserver() * self.dec.Decode(string(input_str), &observer) # <<<<<<<<<<<<<< * if observer.hypergraph == NULL: * raise ParseFailed() */ - __pyx_t_6 = PyBytes_AsString(((PyObject *)__pyx_v_input_str)); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_6 = PyBytes_AsString(((PyObject *)__pyx_v_input_str)); if (unlikely((!__pyx_t_6) && PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 109; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_v_self->dec->Decode(std::string(__pyx_t_6), (&__pyx_v_observer)); - /* "_cdec.pyx":109 + /* "_cdec.pyx":110 * cdef decoder.BasicObserver observer = decoder.BasicObserver() * self.dec.Decode(string(input_str), &observer) * if observer.hypergraph == NULL: # <<<<<<<<<<<<<< @@ -18563,38 +18503,38 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec __pyx_t_3 = (__pyx_v_observer.hypergraph == NULL); if (__pyx_t_3) { - /* "_cdec.pyx":110 + /* "_cdec.pyx":111 * self.dec.Decode(string(input_str), &observer) * if observer.hypergraph == NULL: * raise ParseFailed() # <<<<<<<<<<<<<< * cdef Hypergraph hg = Hypergraph() * hg.hg = new hypergraph.Hypergraph(observer.hypergraph[0]) */ - __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__ParseFailed); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__ParseFailed); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_1); - __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - {__pyx_filename = __pyx_f[0]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} goto __pyx_L6; } __pyx_L6:; - /* "_cdec.pyx":111 + /* "_cdec.pyx":112 * if observer.hypergraph == NULL: * raise ParseFailed() * cdef Hypergraph hg = Hypergraph() # <<<<<<<<<<<<<< * hg.hg = new hypergraph.Hypergraph(observer.hypergraph[0]) * return hg */ - __pyx_t_5 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Hypergraph)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 111; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_t_5 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Hypergraph)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_t_5); __pyx_v_hg = ((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_t_5); __pyx_t_5 = 0; - /* "_cdec.pyx":112 + /* "_cdec.pyx":113 * raise ParseFailed() * cdef Hypergraph hg = Hypergraph() * hg.hg = new hypergraph.Hypergraph(observer.hypergraph[0]) # <<<<<<<<<<<<<< @@ -18602,7 +18542,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec */ __pyx_v_hg->hg = new Hypergraph((__pyx_v_observer.hypergraph[0])); - /* "_cdec.pyx":113 + /* "_cdec.pyx":114 * cdef Hypergraph hg = Hypergraph() * hg.hg = new hypergraph.Hypergraph(observer.hypergraph[0]) * return hg # <<<<<<<<<<<<<< @@ -18627,50 +18567,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec return __pyx_r; } -/* "string.to_py":25 - * - * @cname("__pyx_convert_string_to_py_") - * cdef object __pyx_convert_string_to_py_(string& s): # <<<<<<<<<<<<<< - * return s.data()[:s.size()] - * - */ - -static PyObject *__pyx_convert_string_to_py_(const std::string &__pyx_v_s) { - PyObject *__pyx_r = NULL; - __Pyx_RefNannyDeclarations - PyObject *__pyx_t_1 = NULL; - int __pyx_lineno = 0; - const char *__pyx_filename = NULL; - int __pyx_clineno = 0; - __Pyx_RefNannySetupContext("__pyx_convert_string_to_py_", 0); - - /* "string.to_py":26 - * @cname("__pyx_convert_string_to_py_") - * cdef object __pyx_convert_string_to_py_(string& s): - * return s.data()[:s.size()] # <<<<<<<<<<<<<< - * - * - */ - __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyBytes_FromStringAndSize(__pyx_v_s.data() + 0, __pyx_v_s.size() - 0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[6]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __Pyx_GOTREF(((PyObject *)__pyx_t_1)); - __pyx_r = ((PyObject *)__pyx_t_1); - __pyx_t_1 = 0; - goto __pyx_L0; - - __pyx_r = Py_None; __Pyx_INCREF(Py_None); - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_AddTraceback("string.to_py.__pyx_convert_string_to_py_", __pyx_clineno, __pyx_lineno, __pyx_filename); - __pyx_r = 0; - __pyx_L0:; - __Pyx_XGIVEREF(__pyx_r); - __Pyx_RefNannyFinishContext(); - return __pyx_r; -} - -static PyObject *__pyx_tp_new_5_cdec_DenseVector(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_DenseVector(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -18867,7 +18764,7 @@ static PyTypeObject __pyx_type_5_cdec_DenseVector = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_SparseVector(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_SparseVector(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -19064,7 +18961,7 @@ static PyTypeObject __pyx_type_5_cdec_SparseVector = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_NT(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_NT(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec_NT *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -19097,11 +18994,11 @@ static int __pyx_tp_clear_5_cdec_NT(PyObject *o) { return 0; } -static PyObject *__pyx_getprop_5_cdec_2NT_cat(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_2NT_cat(PyObject *o, void *x) { return __pyx_pw_5_cdec_2NT_3cat_1__get__(o); } -static int __pyx_setprop_5_cdec_2NT_cat(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_2NT_cat(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_2NT_3cat_3__set__(o, v); } @@ -19110,11 +19007,11 @@ static int __pyx_setprop_5_cdec_2NT_cat(PyObject *o, PyObject *v, CYTHON_UNUSED } } -static PyObject *__pyx_getprop_5_cdec_2NT_ref(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_2NT_ref(PyObject *o, void *x) { return __pyx_pw_5_cdec_2NT_3ref_1__get__(o); } -static int __pyx_setprop_5_cdec_2NT_ref(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_2NT_ref(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_2NT_3ref_3__set__(o, v); } @@ -19288,7 +19185,7 @@ static PyTypeObject __pyx_type_5_cdec_NT = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_NTRef(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_NTRef(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -19298,11 +19195,11 @@ static void __pyx_tp_dealloc_5_cdec_NTRef(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_5NTRef_ref(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5NTRef_ref(PyObject *o, void *x) { return __pyx_pw_5_cdec_5NTRef_3ref_1__get__(o); } -static int __pyx_setprop_5_cdec_5NTRef_ref(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5NTRef_ref(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5NTRef_3ref_3__set__(o, v); } @@ -19475,7 +19372,7 @@ static PyTypeObject __pyx_type_5_cdec_NTRef = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_TRule(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_TRule(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -19494,15 +19391,15 @@ static void __pyx_tp_dealloc_5_cdec_TRule(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_5TRule_arity(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_arity(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_5arity_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_5TRule_f(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_f(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_1f_1__get__(o); } -static int __pyx_setprop_5_cdec_5TRule_f(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5TRule_f(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5TRule_1f_3__set__(o, v); } @@ -19512,11 +19409,11 @@ static int __pyx_setprop_5_cdec_5TRule_f(PyObject *o, PyObject *v, CYTHON_UNUSED } } -static PyObject *__pyx_getprop_5_cdec_5TRule_e(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_e(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_1e_1__get__(o); } -static int __pyx_setprop_5_cdec_5TRule_e(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5TRule_e(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5TRule_1e_3__set__(o, v); } @@ -19526,11 +19423,11 @@ static int __pyx_setprop_5_cdec_5TRule_e(PyObject *o, PyObject *v, CYTHON_UNUSED } } -static PyObject *__pyx_getprop_5_cdec_5TRule_a(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_a(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_1a_1__get__(o); } -static int __pyx_setprop_5_cdec_5TRule_a(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5TRule_a(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5TRule_1a_4__set__(o, v); } @@ -19540,11 +19437,11 @@ static int __pyx_setprop_5_cdec_5TRule_a(PyObject *o, PyObject *v, CYTHON_UNUSED } } -static PyObject *__pyx_getprop_5_cdec_5TRule_scores(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_scores(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_6scores_1__get__(o); } -static int __pyx_setprop_5_cdec_5TRule_scores(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5TRule_scores(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5TRule_6scores_3__set__(o, v); } @@ -19554,11 +19451,11 @@ static int __pyx_setprop_5_cdec_5TRule_scores(PyObject *o, PyObject *v, CYTHON_U } } -static PyObject *__pyx_getprop_5_cdec_5TRule_lhs(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_5TRule_lhs(PyObject *o, void *x) { return __pyx_pw_5_cdec_5TRule_3lhs_1__get__(o); } -static int __pyx_setprop_5_cdec_5TRule_lhs(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_5TRule_lhs(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_5TRule_3lhs_3__set__(o, v); } @@ -19864,11 +19761,7 @@ static PyTypeObject __pyx_type_5_cdec_MRule = { &__pyx_tp_as_mapping_MRule, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ - #if CYTHON_COMPILING_IN_PYPY - __pyx_pw_5_cdec_5TRule_5__str__, /*tp_str*/ - #else 0, /*tp_str*/ - #endif 0, /*tp_getattro*/ 0, /*tp_setattro*/ &__pyx_tp_as_buffer_MRule, /*tp_as_buffer*/ @@ -19904,7 +19797,7 @@ static PyTypeObject __pyx_type_5_cdec_MRule = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_Grammar(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_Grammar(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -19923,11 +19816,11 @@ static void __pyx_tp_dealloc_5_cdec_Grammar(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_7Grammar_name(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_7Grammar_name(PyObject *o, void *x) { return __pyx_pw_5_cdec_7Grammar_4name_1__get__(o); } -static int __pyx_setprop_5_cdec_7Grammar_name(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_7Grammar_name(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_7Grammar_4name_3__set__(o, v); } @@ -20241,11 +20134,7 @@ static PyTypeObject __pyx_type_5_cdec_TextGrammar = { 0, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ - #if CYTHON_COMPILING_IN_PYPY - __pyx_pw_5_cdec_7Grammar_3__iter__, /*tp_iter*/ - #else 0, /*tp_iter*/ - #endif 0, /*tp_iternext*/ __pyx_methods_5_cdec_TextGrammar, /*tp_methods*/ 0, /*tp_members*/ @@ -20271,7 +20160,7 @@ static PyTypeObject __pyx_type_5_cdec_TextGrammar = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_Hypergraph(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_Hypergraph(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -20290,19 +20179,19 @@ static void __pyx_tp_dealloc_5_cdec_Hypergraph(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_10Hypergraph_edges(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_10Hypergraph_edges(PyObject *o, void *x) { return __pyx_pw_5_cdec_10Hypergraph_5edges_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_10Hypergraph_nodes(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_10Hypergraph_nodes(PyObject *o, void *x) { return __pyx_pw_5_cdec_10Hypergraph_5nodes_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_10Hypergraph_goal(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_10Hypergraph_goal(PyObject *o, void *x) { return __pyx_pw_5_cdec_10Hypergraph_4goal_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_10Hypergraph_npaths(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_10Hypergraph_npaths(PyObject *o, void *x) { return __pyx_pw_5_cdec_10Hypergraph_6npaths_1__get__(o); } @@ -20315,11 +20204,12 @@ static PyMethodDef __pyx_methods_5_cdec_Hypergraph[] = { {__Pyx_NAMESTR("kbest_trees"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_14kbest_trees, METH_O, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("kbest_features"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_17kbest_features, METH_O, __Pyx_DOCSTR(0)}, {__Pyx_NAMESTR("sample"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_20sample, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("intersect"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_23intersect, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("prune"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_25prune, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("lattice"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_27lattice, METH_NOARGS, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("reweight"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_29reweight, METH_O, __Pyx_DOCSTR(0)}, - {__Pyx_NAMESTR("inside_outside"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_31inside_outside, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("sample_trees"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_23sample_trees, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("intersect"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_26intersect, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("prune"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_28prune, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("lattice"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_30lattice, METH_NOARGS, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("reweight"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_32reweight, METH_O, __Pyx_DOCSTR(0)}, + {__Pyx_NAMESTR("inside_outside"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_34inside_outside, METH_NOARGS, __Pyx_DOCSTR(0)}, {0, 0, 0, 0} }; @@ -20486,7 +20376,7 @@ static PyTypeObject __pyx_type_5_cdec_Hypergraph = { }; static struct __pyx_vtabstruct_5_cdec_HypergraphEdge __pyx_vtable_5_cdec_HypergraphEdge; -static PyObject *__pyx_tp_new_5_cdec_HypergraphEdge(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_HypergraphEdge(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec_HypergraphEdge *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -20520,31 +20410,31 @@ static int __pyx_tp_clear_5_cdec_HypergraphEdge(PyObject *o) { return 0; } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_head_node(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_head_node(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_9head_node_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_tail_nodes(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_tail_nodes(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_10tail_nodes_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_span(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_span(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_4span_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_feature_values(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_feature_values(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_14feature_values_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_prob(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_prob(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_4prob_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_trule(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphEdge_trule(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphEdge_5trule_1__get__(o); } -static int __pyx_setprop_5_cdec_14HypergraphEdge_trule(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_14HypergraphEdge_trule(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_14HypergraphEdge_5trule_3__set__(o, v); } @@ -20722,7 +20612,7 @@ static PyTypeObject __pyx_type_5_cdec_HypergraphEdge = { }; static struct __pyx_vtabstruct_5_cdec_HypergraphNode __pyx_vtable_5_cdec_HypergraphNode; -static PyObject *__pyx_tp_new_5_cdec_HypergraphNode(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_HypergraphNode(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec_HypergraphNode *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -20735,19 +20625,19 @@ static void __pyx_tp_dealloc_5_cdec_HypergraphNode(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_in_edges(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_in_edges(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphNode_8in_edges_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_out_edges(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_out_edges(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphNode_9out_edges_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_span(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_span(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphNode_4span_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_cat(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_14HypergraphNode_cat(PyObject *o, void *x) { return __pyx_pw_5_cdec_14HypergraphNode_3cat_1__get__(o); } @@ -21116,7 +21006,7 @@ static PyTypeObject __pyx_type_5_cdec_Lattice = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_Candidate(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_Candidate(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -21126,19 +21016,19 @@ static void __pyx_tp_dealloc_5_cdec_Candidate(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static PyObject *__pyx_getprop_5_cdec_9Candidate_words(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_9Candidate_words(PyObject *o, void *x) { return __pyx_pw_5_cdec_9Candidate_5words_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_9Candidate_fmap(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_9Candidate_fmap(PyObject *o, void *x) { return __pyx_pw_5_cdec_9Candidate_4fmap_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_9Candidate_score(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_9Candidate_score(PyObject *o, void *x) { return __pyx_pw_5_cdec_9Candidate_5score_1__get__(o); } -static int __pyx_setprop_5_cdec_9Candidate_score(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_9Candidate_score(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_9Candidate_5score_3__set__(o, v); } @@ -21313,7 +21203,7 @@ static PyTypeObject __pyx_type_5_cdec_Candidate = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_SufficientStats(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_SufficientStats(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -21339,11 +21229,11 @@ static PyObject *__pyx_sq_item_5_cdec_SufficientStats(PyObject *o, Py_ssize_t i) return r; } -static PyObject *__pyx_getprop_5_cdec_15SufficientStats_score(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_15SufficientStats_score(PyObject *o, void *x) { return __pyx_pw_5_cdec_15SufficientStats_5score_1__get__(o); } -static PyObject *__pyx_getprop_5_cdec_15SufficientStats_detail(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_15SufficientStats_detail(PyObject *o, void *x) { return __pyx_pw_5_cdec_15SufficientStats_6detail_1__get__(o); } @@ -21699,7 +21589,7 @@ static PyTypeObject __pyx_type_5_cdec_CandidateSet = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_SegmentEvaluator(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_SegmentEvaluator(PyTypeObject *t, PyObject *a, PyObject *k) { PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; return o; @@ -22058,7 +21948,7 @@ static PyTypeObject __pyx_type_5_cdec_Scorer = { #endif }; -static PyObject *__pyx_tp_new_5_cdec_Metric(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec_Metric(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec_Metric *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -22299,11 +22189,11 @@ static int __pyx_tp_clear_5_cdec_Decoder(PyObject *o) { return 0; } -static PyObject *__pyx_getprop_5_cdec_7Decoder_weights(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_7Decoder_weights(PyObject *o, void *x) { return __pyx_pw_5_cdec_7Decoder_7weights_1__get__(o); } -static int __pyx_setprop_5_cdec_7Decoder_weights(PyObject *o, PyObject *v, CYTHON_UNUSED void *x) { +static int __pyx_setprop_5_cdec_7Decoder_weights(PyObject *o, PyObject *v, void *x) { if (v) { return __pyx_pw_5_cdec_7Decoder_7weights_3__set__(o, v); } @@ -22313,7 +22203,7 @@ static int __pyx_setprop_5_cdec_7Decoder_weights(PyObject *o, PyObject *v, CYTHO } } -static PyObject *__pyx_getprop_5_cdec_7Decoder_formalism(PyObject *o, CYTHON_UNUSED void *x) { +static PyObject *__pyx_getprop_5_cdec_7Decoder_formalism(PyObject *o, void *x) { return __pyx_pw_5_cdec_7Decoder_9formalism_1__get__(o); } @@ -22483,7 +22373,7 @@ static PyTypeObject __pyx_type_5_cdec_Decoder = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct____iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct____iter__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct____iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -22674,7 +22564,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct____iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_1___iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_1___iter__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_1___iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -22865,7 +22755,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_1___iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_2__phrase(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_2__phrase(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_2__phrase *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -23056,7 +22946,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_2__phrase = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_3_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_3_genexpr(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_3_genexpr *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -23263,7 +23153,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_3_genexpr = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_4___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_4___get__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_4___get__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -23454,7 +23344,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_5___str__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_5___str__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_5___str__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -23645,7 +23535,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_5___str__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_6_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_6_genexpr(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_6_genexpr *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -23852,7 +23742,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_6_genexpr = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_7___iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_7___iter__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -24051,7 +23941,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_7___iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_8_kbest(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_8_kbest(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_8_kbest *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -24250,7 +24140,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_8_kbest = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_9_kbest_trees(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_9_kbest_trees(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_9_kbest_trees *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -24465,7 +24355,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_9_kbest_trees = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_10_kbest_features(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_10_kbest_features(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_10_kbest_features *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -24672,7 +24562,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_10_kbest_features = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_11_sample(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_11_sample(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_11_sample *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -24863,32 +24753,32 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_11_sample = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_12___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_12_sample_trees(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)o); p->__pyx_v_self = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_12___get__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_12_sample_trees(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)o; Py_XDECREF(((PyObject *)p->__pyx_v_self)); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_12___get__(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_12_sample_trees(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)o; if (p->__pyx_v_self) { e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; } return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_12___get__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_12_sample_trees(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Hypergraph *)Py_None); Py_INCREF(Py_None); @@ -24896,11 +24786,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_12___get__(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_12___get__[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_12_sample_trees[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_12___get__ = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_12_sample_trees = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -24958,7 +24848,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_12___get__ = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_12___get__ = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_12_sample_trees = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -24971,13 +24861,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_12___get__ = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_12___get__ = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_12_sample_trees = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_12___get__ = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_12_sample_trees = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -24998,12 +24888,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_12___get__ = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_12___get__ = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_12_sample_trees = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_12___get__"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_12_sample_trees"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_12_sample_trees), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_12___get__, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_12_sample_trees, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -25013,24 +24903,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_12___get__ = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_12___get__, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_12___get__, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_12___get__, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_12_sample_trees, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_12_sample_trees, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_12_sample_trees, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_12___get__, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_12_sample_trees, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_12___get__, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_12___get__, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_12_sample_trees, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_12_sample_trees, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_12___get__, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_12_sample_trees, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -25040,7 +24930,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_12___get__ = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_12___get__, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_12_sample_trees, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -25054,7 +24944,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_12___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_13___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_13___get__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_13___get__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -25245,7 +25135,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_13___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_14___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_14___get__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -25273,7 +25163,7 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_14___get__(PyObject *o) { struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_14___get__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5_cdec_HypergraphEdge *)Py_None); Py_INCREF(Py_None); + p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Hypergraph *)Py_None); Py_INCREF(Py_None); Py_XDECREF(tmp); return 0; } @@ -25436,7 +25326,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_14___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_15___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_15___get__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -25464,7 +25354,7 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_15___get__(PyObject *o) { struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_15___get__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); - p->__pyx_v_self = ((struct __pyx_obj_5_cdec_HypergraphNode *)Py_None); Py_INCREF(Py_None); + p->__pyx_v_self = ((struct __pyx_obj_5_cdec_HypergraphEdge *)Py_None); Py_INCREF(Py_None); Py_XDECREF(tmp); return 0; } @@ -25627,7 +25517,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_15___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_16___get__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_16___get__(PyTypeObject *t, PyObject *a, PyObject *k) { struct __pyx_obj_5_cdec___pyx_scope_struct_16___get__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; @@ -25818,32 +25708,223 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_16___get__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_17___iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_17___get__(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *p; + PyObject *o = (*t->tp_alloc)(t, 0); + if (!o) return 0; + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)o); + p->__pyx_v_self = 0; + return o; +} + +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_17___get__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)o; + Py_XDECREF(((PyObject *)p->__pyx_v_self)); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_17___get__(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)o; + if (p->__pyx_v_self) { + e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_17___get__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__ *)o; + PyObject* tmp; + tmp = ((PyObject*)p->__pyx_v_self); + p->__pyx_v_self = ((struct __pyx_obj_5_cdec_HypergraphNode *)Py_None); Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_17___get__[] = { + {0, 0, 0, 0} +}; + +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_17___get__ = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_divide*/ + #endif + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_coerce*/ + #endif + 0, /*nb_int*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_long*/ + #else + 0, /*reserved*/ + #endif + 0, /*nb_float*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_oct*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*nb_hex*/ + #endif + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + #if PY_MAJOR_VERSION < 3 + 0, /*nb_inplace_divide*/ + #endif + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + 0, /*nb_inplace_and*/ + 0, /*nb_inplace_xor*/ + 0, /*nb_inplace_or*/ + 0, /*nb_floor_divide*/ + 0, /*nb_true_divide*/ + 0, /*nb_inplace_floor_divide*/ + 0, /*nb_inplace_true_divide*/ + #if PY_VERSION_HEX >= 0x02050000 + 0, /*nb_index*/ + #endif +}; + +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_17___get__ = { + 0, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_17___get__ = { + 0, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_17___get__ = { + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getreadbuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getwritebuffer*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getsegcount*/ + #endif + #if PY_MAJOR_VERSION < 3 + 0, /*bf_getcharbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_getbuffer*/ + #endif + #if PY_VERSION_HEX >= 0x02060000 + 0, /*bf_releasebuffer*/ + #endif +}; + +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_17___get__ = { + PyVarObject_HEAD_INIT(0, 0) + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_17___get__"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_17___get__), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_17___get__, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #else + 0, /*reserved*/ + #endif + 0, /*tp_repr*/ + &__pyx_tp_as_number___pyx_scope_struct_17___get__, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_17___get__, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_17___get__, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + &__pyx_tp_as_buffer___pyx_scope_struct_17___get__, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + 0, /*tp_doc*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_17___get__, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_17___get__, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5_cdec___pyx_scope_struct_17___get__, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_17___get__, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + #if PY_VERSION_HEX >= 0x02060000 + 0, /*tp_version_tag*/ + #endif +}; + +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_18___iter__(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)o); p->__pyx_v_self = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_17___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_18___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)o; Py_XDECREF(((PyObject *)p->__pyx_v_self)); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_17___iter__(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_18___iter__(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)o; if (p->__pyx_v_self) { e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; } return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_17___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__ *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_18___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Lattice *)Py_None); Py_INCREF(Py_None); @@ -25851,11 +25932,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_17___iter__(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_17___iter__[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_18___iter__[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_17___iter__ = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_18___iter__ = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -25913,7 +25994,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_17___iter__ = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_17___iter__ = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_18___iter__ = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -25926,13 +26007,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_17___iter__ = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_17___iter__ = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_18___iter__ = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_17___iter__ = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_18___iter__ = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -25953,12 +26034,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_17___iter__ = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_17___iter__ = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_18___iter__ = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_17___iter__"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_17___iter__), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_18___iter__"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_18___iter__), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_17___iter__, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_18___iter__, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -25968,24 +26049,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_17___iter__ = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_17___iter__, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_17___iter__, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_17___iter__, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_18___iter__, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_18___iter__, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_18___iter__, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_17___iter__, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_18___iter__, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_17___iter__, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_17___iter__, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_18___iter__, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_18___iter__, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_17___iter__, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_18___iter__, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -25995,7 +26076,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_17___iter__ = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_17___iter__, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_18___iter__, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -26009,32 +26090,32 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_17___iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_18_todot(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_19_todot(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)o); p->__pyx_v_self = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_18_todot(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_19_todot(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)o; Py_XDECREF(((PyObject *)p->__pyx_v_self)); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_18_todot(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_19_todot(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)o; if (p->__pyx_v_self) { e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; } return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_18_todot(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_19_todot(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Lattice *)Py_None); Py_INCREF(Py_None); @@ -26042,11 +26123,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_18_todot(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_18_todot[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_19_todot[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_18_todot = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_19_todot = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -26104,7 +26185,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_18_todot = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_18_todot = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_19_todot = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -26117,13 +26198,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_18_todot = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_18_todot = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_19_todot = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_18_todot = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_19_todot = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -26144,12 +26225,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_18_todot = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_18_todot = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_19_todot = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_18_todot"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_19_todot"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_18_todot, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_19_todot, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -26159,24 +26240,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_18_todot = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_18_todot, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_18_todot, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_18_todot, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_19_todot, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_19_todot, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_19_todot, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_18_todot, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_19_todot, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_18_todot, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_18_todot, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_19_todot, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_19_todot, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_18_todot, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_19_todot, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -26186,7 +26267,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_18_todot = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_18_todot, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_19_todot, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -26200,11 +26281,11 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_18_todot = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_19_lines(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_20_lines(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)o); p->__pyx_outer_scope = 0; p->__pyx_v_delta = 0; p->__pyx_v_i = 0; @@ -26215,8 +26296,8 @@ static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_19_lines(PyTypeObject *t return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_19_lines(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_20_lines(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)o; Py_XDECREF(((PyObject *)p->__pyx_outer_scope)); Py_XDECREF(p->__pyx_v_delta); Py_XDECREF(p->__pyx_v_i); @@ -26227,9 +26308,9 @@ static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_19_lines(PyObject *o) { (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_19_lines(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_20_lines(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)o; if (p->__pyx_outer_scope) { e = (*v)(((PyObject*)p->__pyx_outer_scope), a); if (e) return e; } @@ -26254,11 +26335,11 @@ static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_19_lines(PyObject *o, vis return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_19_lines(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_20_lines(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_outer_scope); - p->__pyx_outer_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_18_todot *)Py_None); Py_INCREF(Py_None); + p->__pyx_outer_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_19_todot *)Py_None); Py_INCREF(Py_None); Py_XDECREF(tmp); tmp = ((PyObject*)p->__pyx_v_delta); p->__pyx_v_delta = Py_None; Py_INCREF(Py_None); @@ -26281,11 +26362,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_19_lines(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_19_lines[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_20_lines[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_19_lines = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_20_lines = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -26343,7 +26424,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_19_lines = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_19_lines = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_20_lines = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -26356,13 +26437,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_19_lines = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_19_lines = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_20_lines = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_19_lines = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_20_lines = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -26383,12 +26464,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_19_lines = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_19_lines = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_20_lines = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_19_lines"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_19_lines), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_20_lines"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_20_lines), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_19_lines, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_20_lines, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -26398,24 +26479,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_19_lines = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_19_lines, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_19_lines, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_19_lines, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_20_lines, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_20_lines, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_20_lines, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_19_lines, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_20_lines, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_19_lines, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_19_lines, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_20_lines, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_20_lines, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_19_lines, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_20_lines, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -26425,7 +26506,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_19_lines = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_19_lines, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_20_lines, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -26439,28 +26520,28 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_19_lines = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_20___iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_21___iter__(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o); p->__pyx_v_i = 0; p->__pyx_v_self = 0; p->__pyx_t_1 = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_20___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_21___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; Py_XDECREF(p->__pyx_v_i); Py_XDECREF(((PyObject *)p->__pyx_v_self)); Py_XDECREF(p->__pyx_t_1); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_20___iter__(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_21___iter__(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; if (p->__pyx_v_i) { e = (*v)(p->__pyx_v_i, a); if (e) return e; } @@ -26473,8 +26554,8 @@ static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_20___iter__(PyObject *o, return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_20___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_21___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_i); p->__pyx_v_i = Py_None; Py_INCREF(Py_None); @@ -26488,11 +26569,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_20___iter__(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_20___iter__[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_21___iter__[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_20___iter__ = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_21___iter__ = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -26550,7 +26631,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_20___iter__ = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_20___iter__ = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_21___iter__ = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -26563,13 +26644,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_20___iter__ = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_20___iter__ = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_21___iter__ = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_20___iter__ = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_21___iter__ = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -26590,12 +26671,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_20___iter__ = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_20___iter__ = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_21___iter__ = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_20___iter__"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_21___iter__"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_20___iter__, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_21___iter__, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -26605,24 +26686,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_20___iter__ = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_20___iter__, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_20___iter__, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_20___iter__, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_21___iter__, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_21___iter__, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_21___iter__, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_20___iter__, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_21___iter__, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_20___iter__, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_20___iter__, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_21___iter__, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_21___iter__, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_20___iter__, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_21___iter__, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -26632,7 +26713,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_20___iter__ = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_20___iter__, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_21___iter__, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -26646,32 +26727,32 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_20___iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_21___iter__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_22___iter__(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)o); p->__pyx_v_self = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_21___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_22___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)o; Py_XDECREF(((PyObject *)p->__pyx_v_self)); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_21___iter__(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_22___iter__(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)o; if (p->__pyx_v_self) { e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e; } return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_21___iter__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__ *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_22___iter__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_self); p->__pyx_v_self = ((struct __pyx_obj_5_cdec_CandidateSet *)Py_None); Py_INCREF(Py_None); @@ -26679,11 +26760,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_21___iter__(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_21___iter__[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_22___iter__[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_21___iter__ = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_22___iter__ = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -26741,7 +26822,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_21___iter__ = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_21___iter__ = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_22___iter__ = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -26754,13 +26835,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_21___iter__ = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_21___iter__ = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_22___iter__ = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_21___iter__ = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_22___iter__ = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -26781,12 +26862,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_21___iter__ = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_21___iter__ = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_22___iter__ = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_21___iter__"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_22___iter__"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_22___iter__), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_21___iter__, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_22___iter__, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -26796,24 +26877,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_21___iter__ = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_21___iter__, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_21___iter__, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_21___iter__, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_22___iter__, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_22___iter__, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_22___iter__, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_21___iter__, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_22___iter__, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_21___iter__, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_21___iter__, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_22___iter__, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_22___iter__, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_21___iter__, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_22___iter__, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -26823,7 +26904,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_21___iter__ = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_21___iter__, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_22___iter__, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -26837,11 +26918,11 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_21___iter__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_22__make_config(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_23__make_config(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)o); p->__pyx_v_config = 0; p->__pyx_v_info = 0; p->__pyx_v_key = 0; @@ -26852,8 +26933,8 @@ static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_22__make_config(PyTypeOb return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_22__make_config(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_23__make_config(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)o; Py_XDECREF(p->__pyx_v_config); Py_XDECREF(p->__pyx_v_info); Py_XDECREF(p->__pyx_v_key); @@ -26864,9 +26945,9 @@ static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_22__make_config(PyObject (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_22__make_config(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_23__make_config(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)o; if (p->__pyx_v_config) { e = (*v)(p->__pyx_v_config, a); if (e) return e; } @@ -26891,8 +26972,8 @@ static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_22__make_config(PyObject return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_22__make_config(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_23__make_config(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_config); p->__pyx_v_config = Py_None; Py_INCREF(Py_None); @@ -26918,11 +26999,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_22__make_config(PyObject *o) return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_22__make_config[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_23__make_config[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_22__make_config = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_23__make_config = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -26980,7 +27061,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_22__make_config = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_22__make_config = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_23__make_config = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -26993,13 +27074,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_22__make_config 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_22__make_config = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_23__make_config = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_22__make_config = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_23__make_config = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -27020,12 +27101,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_22__make_config = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_22__make_config = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_23__make_config = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_22__make_config"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_22__make_config), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_23__make_config"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_23__make_config), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_22__make_config, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_23__make_config, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -27035,24 +27116,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_22__make_config = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_22__make_config, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_22__make_config, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_22__make_config, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_23__make_config, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_23__make_config, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_23__make_config, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_22__make_config, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_23__make_config, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_22__make_config, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_22__make_config, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_23__make_config, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_23__make_config, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_22__make_config, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_23__make_config, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -27062,7 +27143,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_22__make_config = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_22__make_config, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_23__make_config, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -27076,32 +27157,32 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_22__make_config = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_23___cinit__(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_24___cinit__(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)o); p->__pyx_v_config = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_23___cinit__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_24___cinit__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)o; Py_XDECREF(p->__pyx_v_config); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_23___cinit__(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_24___cinit__(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)o; if (p->__pyx_v_config) { e = (*v)(p->__pyx_v_config, a); if (e) return e; } return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_23___cinit__(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_24___cinit__(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_v_config); p->__pyx_v_config = Py_None; Py_INCREF(Py_None); @@ -27109,11 +27190,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_23___cinit__(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_23___cinit__[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_24___cinit__[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_23___cinit__ = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_24___cinit__ = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -27171,7 +27252,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_23___cinit__ = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_23___cinit__ = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_24___cinit__ = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -27184,13 +27265,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_23___cinit__ = 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_23___cinit__ = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_24___cinit__ = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_23___cinit__ = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_24___cinit__ = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -27211,12 +27292,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_23___cinit__ = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_23___cinit__ = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_24___cinit__ = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_23___cinit__"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_24___cinit__"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_23___cinit__, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_24___cinit__, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -27226,24 +27307,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_23___cinit__ = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_23___cinit__, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_23___cinit__, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_23___cinit__, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_24___cinit__, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_24___cinit__, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_24___cinit__, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_23___cinit__, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_24___cinit__, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_23___cinit__, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_23___cinit__, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_24___cinit__, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_24___cinit__, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_23___cinit__, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_24___cinit__, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -27253,7 +27334,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_23___cinit__ = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_23___cinit__, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_24___cinit__, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -27267,28 +27348,28 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_23___cinit__ = { #endif }; -static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_24_genexpr(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *p; +static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_25_genexpr(PyTypeObject *t, PyObject *a, PyObject *k) { + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *p; PyObject *o = (*t->tp_alloc)(t, 0); if (!o) return 0; - p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)o); + p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)o); p->__pyx_outer_scope = 0; p->__pyx_v_kv = 0; p->__pyx_t_0 = 0; return o; } -static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_24_genexpr(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)o; +static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_25_genexpr(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)o; Py_XDECREF(((PyObject *)p->__pyx_outer_scope)); Py_XDECREF(p->__pyx_v_kv); Py_XDECREF(p->__pyx_t_0); (*Py_TYPE(o)->tp_free)(o); } -static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_24_genexpr(PyObject *o, visitproc v, void *a) { +static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_25_genexpr(PyObject *o, visitproc v, void *a) { int e; - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)o; + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)o; if (p->__pyx_outer_scope) { e = (*v)(((PyObject*)p->__pyx_outer_scope), a); if (e) return e; } @@ -27301,11 +27382,11 @@ static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_24_genexpr(PyObject *o, v return 0; } -static int __pyx_tp_clear_5_cdec___pyx_scope_struct_24_genexpr(PyObject *o) { - struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr *)o; +static int __pyx_tp_clear_5_cdec___pyx_scope_struct_25_genexpr(PyObject *o) { + struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr *)o; PyObject* tmp; tmp = ((PyObject*)p->__pyx_outer_scope); - p->__pyx_outer_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_23___cinit__ *)Py_None); Py_INCREF(Py_None); + p->__pyx_outer_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_24___cinit__ *)Py_None); Py_INCREF(Py_None); Py_XDECREF(tmp); tmp = ((PyObject*)p->__pyx_v_kv); p->__pyx_v_kv = Py_None; Py_INCREF(Py_None); @@ -27316,11 +27397,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_24_genexpr(PyObject *o) { return 0; } -static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_24_genexpr[] = { +static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_25_genexpr[] = { {0, 0, 0, 0} }; -static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_24_genexpr = { +static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_25_genexpr = { 0, /*nb_add*/ 0, /*nb_subtract*/ 0, /*nb_multiply*/ @@ -27378,7 +27459,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_24_genexpr = { #endif }; -static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_24_genexpr = { +static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_25_genexpr = { 0, /*sq_length*/ 0, /*sq_concat*/ 0, /*sq_repeat*/ @@ -27391,13 +27472,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_24_genexpr = { 0, /*sq_inplace_repeat*/ }; -static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_24_genexpr = { +static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_25_genexpr = { 0, /*mp_length*/ 0, /*mp_subscript*/ 0, /*mp_ass_subscript*/ }; -static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_24_genexpr = { +static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_25_genexpr = { #if PY_MAJOR_VERSION < 3 0, /*bf_getreadbuffer*/ #endif @@ -27418,12 +27499,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_24_genexpr = { #endif }; -static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_24_genexpr = { +static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_25_genexpr = { PyVarObject_HEAD_INIT(0, 0) - __Pyx_NAMESTR("_cdec.__pyx_scope_struct_24_genexpr"), /*tp_name*/ - sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_24_genexpr), /*tp_basicsize*/ + __Pyx_NAMESTR("_cdec.__pyx_scope_struct_25_genexpr"), /*tp_name*/ + sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_25_genexpr), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc_5_cdec___pyx_scope_struct_24_genexpr, /*tp_dealloc*/ + __pyx_tp_dealloc_5_cdec___pyx_scope_struct_25_genexpr, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -27433,24 +27514,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_24_genexpr = { 0, /*reserved*/ #endif 0, /*tp_repr*/ - &__pyx_tp_as_number___pyx_scope_struct_24_genexpr, /*tp_as_number*/ - &__pyx_tp_as_sequence___pyx_scope_struct_24_genexpr, /*tp_as_sequence*/ - &__pyx_tp_as_mapping___pyx_scope_struct_24_genexpr, /*tp_as_mapping*/ + &__pyx_tp_as_number___pyx_scope_struct_25_genexpr, /*tp_as_number*/ + &__pyx_tp_as_sequence___pyx_scope_struct_25_genexpr, /*tp_as_sequence*/ + &__pyx_tp_as_mapping___pyx_scope_struct_25_genexpr, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ - &__pyx_tp_as_buffer___pyx_scope_struct_24_genexpr, /*tp_as_buffer*/ + &__pyx_tp_as_buffer___pyx_scope_struct_25_genexpr, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - __pyx_tp_traverse_5_cdec___pyx_scope_struct_24_genexpr, /*tp_traverse*/ - __pyx_tp_clear_5_cdec___pyx_scope_struct_24_genexpr, /*tp_clear*/ + __pyx_tp_traverse_5_cdec___pyx_scope_struct_25_genexpr, /*tp_traverse*/ + __pyx_tp_clear_5_cdec___pyx_scope_struct_25_genexpr, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ 0, /*tp_iternext*/ - __pyx_methods_5_cdec___pyx_scope_struct_24_genexpr, /*tp_methods*/ + __pyx_methods_5_cdec___pyx_scope_struct_25_genexpr, /*tp_methods*/ 0, /*tp_members*/ 0, /*tp_getset*/ 0, /*tp_base*/ @@ -27460,7 +27541,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_24_genexpr = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new_5_cdec___pyx_scope_struct_24_genexpr, /*tp_new*/ + __pyx_tp_new_5_cdec___pyx_scope_struct_25_genexpr, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -27627,10 +27708,10 @@ static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_NotImplemented = __Pyx_GetName(__pyx_b, __pyx_n_s__NotImplemented); if (!__pyx_builtin_NotImplemented) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_super = __Pyx_GetName(__pyx_b, __pyx_n_s__super); if (!__pyx_builtin_super) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 179; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_ValueError = __Pyx_GetName(__pyx_b, __pyx_n_s__ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 212; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_eval = __Pyx_GetName(__pyx_b, __pyx_n_s__eval); if (!__pyx_builtin_eval) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_eval = __Pyx_GetName(__pyx_b, __pyx_n_s__eval); if (!__pyx_builtin_eval) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 114; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_enumerate = __Pyx_GetName(__pyx_b, __pyx_n_s__enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_builtin_IndexError = __Pyx_GetName(__pyx_b, __pyx_n_s__IndexError); if (!__pyx_builtin_IndexError) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_builtin_open = __Pyx_GetName(__pyx_b, __pyx_n_s__open); if (!__pyx_builtin_open) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_builtin_open = __Pyx_GetName(__pyx_b, __pyx_n_s__open); if (!__pyx_builtin_open) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} return 0; __pyx_L1_error:; return -1; @@ -27654,7 +27735,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_2)); - /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":89 + /* "/home/cdyer/cdec/python/src/vectors.pxi":89 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for SparseVector') # <<<<<<<<<<<<<< @@ -27668,7 +27749,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_4)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_5)); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":6 + /* "/home/cdyer/cdec/python/src/grammar.pxi":6 * * def _phrase(phrase): * return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase) # <<<<<<<<<<<<<< @@ -27682,7 +27763,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_6)); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":212 + /* "/home/cdyer/cdec/python/src/grammar.pxi":212 * trule = convert_rule(trule) * elif not isinstance(trule, TRule): * raise ValueError('the grammar should contain TRule objects') # <<<<<<<<<<<<<< @@ -27695,33 +27776,33 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_13)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_14)); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":191 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":203 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for HypergraphEdge') # <<<<<<<<<<<<<< * * cdef class HypergraphNode: */ - __pyx_k_tuple_18 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_18)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_18 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_18)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_18); __Pyx_INCREF(((PyObject *)__pyx_kp_s_17)); PyTuple_SET_ITEM(__pyx_k_tuple_18, 0, ((PyObject *)__pyx_kp_s_17)); __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_17)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_18)); - /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":228 + /* "/home/cdyer/cdec/python/src/hypergraph.pxi":240 * elif op == 3: # != * return not (x == y) * raise NotImplemented('comparison not implemented for HypergraphNode') # <<<<<<<<<<<<<< */ - __pyx_k_tuple_20 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_20)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 228; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_20 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_20)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 240; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_20); __Pyx_INCREF(((PyObject *)__pyx_kp_s_19)); PyTuple_SET_ITEM(__pyx_k_tuple_20, 0, ((PyObject *)__pyx_kp_s_19)); __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_19)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_20)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":13 + /* "/home/cdyer/cdec/python/src/lattice.pxi":13 * else: * if isinstance(inp, unicode): * inp = inp.encode('utf8') # <<<<<<<<<<<<<< @@ -27735,7 +27816,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_21)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":24 + /* "/home/cdyer/cdec/python/src/lattice.pxi":24 * def __getitem__(self, int index): * if not 0 <= index < len(self): * raise IndexError('lattice index out of range') # <<<<<<<<<<<<<< @@ -27749,7 +27830,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_23)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_24)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":37 + /* "/home/cdyer/cdec/python/src/lattice.pxi":37 * def __setitem__(self, int index, tuple arcs): * if not 0 <= index < len(self): * raise IndexError('lattice index out of range') # <<<<<<<<<<<<<< @@ -27763,7 +27844,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_23)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_25)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":41 + /* "/home/cdyer/cdec/python/src/lattice.pxi":41 * for (label, cost, dist2next) in arcs: * if isinstance(label, unicode): * label = label.encode('utf8') # <<<<<<<<<<<<<< @@ -27777,7 +27858,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_26)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":64 + /* "/home/cdyer/cdec/python/src/lattice.pxi":64 * for i in range(len(self)): * for label, weight, delta in self[i]: * yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"')) # <<<<<<<<<<<<<< @@ -27794,7 +27875,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_32)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_33)); - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":58 + /* "/home/cdyer/cdec/python/src/lattice.pxi":58 * * def todot(self): * def lines(): # <<<<<<<<<<<<<< @@ -27818,7 +27899,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_36)); __pyx_k_codeobj_37 = (PyObject*)__Pyx_PyCode_New(0, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_36, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_38, __pyx_n_s__lines, 58, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_37)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":67 + /* "/home/cdyer/cdec/python/src/lattice.pxi":67 * yield '%d [shape=doublecircle]' % len(self) * yield '}' * return '\n'.join(lines()).encode('utf8') # <<<<<<<<<<<<<< @@ -27830,7 +27911,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_40)); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50 + /* "/home/cdyer/cdec/python/src/mteval.pxi":50 * def __getitem__(self, int index): * if not 0 <= index < len(self): * raise IndexError('sufficient stats vector index out of range') # <<<<<<<<<<<<<< @@ -27844,7 +27925,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_41)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_42)); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":84 + /* "/home/cdyer/cdec/python/src/mteval.pxi":84 * def __getitem__(self,int k): * if not 0 <= k < self.cs.size(): * raise IndexError('candidate set index out of range') # <<<<<<<<<<<<<< @@ -27875,28 +27956,28 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(Py_None); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_47)); - /* "_cdec.pyx":90 + /* "_cdec.pyx":91 * with open(weights) as fp: * for line in fp: * if line.strip().startswith('#'): continue # <<<<<<<<<<<<<< * fname, value = line.split() * self.weights[fname.strip()] = float(value) */ - __pyx_k_tuple_51 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_51)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_51 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_51)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_51); __Pyx_INCREF(((PyObject *)__pyx_kp_s_50)); PyTuple_SET_ITEM(__pyx_k_tuple_51, 0, ((PyObject *)__pyx_kp_s_50)); __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_50)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_51)); - /* "_cdec.pyx":88 + /* "_cdec.pyx":89 * * def read_weights(self, weights): * with open(weights) as fp: # <<<<<<<<<<<<<< * for line in fp: * if line.strip().startswith('#'): continue */ - __pyx_k_tuple_52 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_52)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_k_tuple_52 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_52)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 89; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_GOTREF(__pyx_k_tuple_52); __Pyx_INCREF(Py_None); PyTuple_SET_ITEM(__pyx_k_tuple_52, 0, Py_None); @@ -27909,7 +27990,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(Py_None); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_52)); - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":5 + /* "/home/cdyer/cdec/python/src/grammar.pxi":5 * import cdec.sa._sa as _sa * * def _phrase(phrase): # <<<<<<<<<<<<<< @@ -27930,7 +28011,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_56)); __pyx_k_codeobj_57 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_56, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_58, __pyx_n_s___phrase, 5, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_57)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":190 + /* "/home/cdyer/cdec/python/src/mteval.pxi":190 * return [] * * BLEU = Scorer('IBM_BLEU') # <<<<<<<<<<<<<< @@ -27944,7 +28025,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__IBM_BLEU)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_59)); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":191 + /* "/home/cdyer/cdec/python/src/mteval.pxi":191 * * BLEU = Scorer('IBM_BLEU') * TER = Scorer('TER') # <<<<<<<<<<<<<< @@ -27957,7 +28038,7 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(((PyObject *)__pyx_n_s__TER)); __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_60)); - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":192 + /* "/home/cdyer/cdec/python/src/mteval.pxi":192 * BLEU = Scorer('IBM_BLEU') * TER = Scorer('TER') * CER = Scorer('CER') # <<<<<<<<<<<<<< @@ -28066,9 +28147,6 @@ PyMODINIT_FUNC PyInit__cdec(void) #endif __pyx_b = PyImport_AddModule(__Pyx_NAMESTR(__Pyx_BUILTIN_MODULE_NAME)); if (!__pyx_b) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; - #if CYTHON_COMPILING_IN_PYPY - Py_INCREF(__pyx_b); - #endif if (__Pyx_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;}; /*--- Initialize various global constants etc. ---*/ if (unlikely(__Pyx_InitGlobals() < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -28114,15 +28192,15 @@ PyMODINIT_FUNC PyInit__cdec(void) __pyx_ptype_5_cdec_Hypergraph = &__pyx_type_5_cdec_Hypergraph; __pyx_vtabptr_5_cdec_HypergraphEdge = &__pyx_vtable_5_cdec_HypergraphEdge; __pyx_vtable_5_cdec_HypergraphEdge.init = (PyObject *(*)(struct __pyx_obj_5_cdec_HypergraphEdge *, Hypergraph *, unsigned int))__pyx_f_5_cdec_14HypergraphEdge_init; - if (PyType_Ready(&__pyx_type_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_5_cdec_HypergraphEdge.tp_dict, __pyx_vtabptr_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "HypergraphEdge", (PyObject *)&__pyx_type_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 147; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_5_cdec_HypergraphEdge.tp_dict, __pyx_vtabptr_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "HypergraphEdge", (PyObject *)&__pyx_type_5_cdec_HypergraphEdge) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 159; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec_HypergraphEdge = &__pyx_type_5_cdec_HypergraphEdge; __pyx_vtabptr_5_cdec_HypergraphNode = &__pyx_vtable_5_cdec_HypergraphNode; __pyx_vtable_5_cdec_HypergraphNode.init = (PyObject *(*)(struct __pyx_obj_5_cdec_HypergraphNode *, Hypergraph *, unsigned int))__pyx_f_5_cdec_14HypergraphNode_init; - if (PyType_Ready(&__pyx_type_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetVtable(__pyx_type_5_cdec_HypergraphNode.tp_dict, __pyx_vtabptr_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - if (__Pyx_SetAttrString(__pyx_m, "HypergraphNode", (PyObject *)&__pyx_type_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 205; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetVtable(__pyx_type_5_cdec_HypergraphNode.tp_dict, __pyx_vtabptr_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 205; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (__Pyx_SetAttrString(__pyx_m, "HypergraphNode", (PyObject *)&__pyx_type_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 205; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec_HypergraphNode = &__pyx_type_5_cdec_HypergraphNode; if (PyType_Ready(&__pyx_type_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} if (__Pyx_SetAttrString(__pyx_m, "Lattice", (PyObject *)&__pyx_type_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -28172,36 +28250,38 @@ PyMODINIT_FUNC PyInit__cdec(void) __pyx_ptype_5_cdec___pyx_scope_struct_10_kbest_features = &__pyx_type_5_cdec___pyx_scope_struct_10_kbest_features; if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_11_sample) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_11_sample = &__pyx_type_5_cdec___pyx_scope_struct_11_sample; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_12___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_12___get__ = &__pyx_type_5_cdec___pyx_scope_struct_12___get__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_13___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 119; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_12_sample_trees) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_12_sample_trees = &__pyx_type_5_cdec___pyx_scope_struct_12_sample_trees; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_13___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 125; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_13___get__ = &__pyx_type_5_cdec___pyx_scope_struct_13___get__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_14___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_14___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_14___get__ = &__pyx_type_5_cdec___pyx_scope_struct_14___get__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_15___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 203; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_15___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 179; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_15___get__ = &__pyx_type_5_cdec___pyx_scope_struct_15___get__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_16___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 209; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_16___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 215; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_16___get__ = &__pyx_type_5_cdec___pyx_scope_struct_16___get__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_17___iter__) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_17___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_17___iter__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_18_todot) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_18_todot = &__pyx_type_5_cdec___pyx_scope_struct_18_todot; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_19_lines) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_19_lines = &__pyx_type_5_cdec___pyx_scope_struct_19_lines; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_20___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_20___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_20___iter__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_21___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_17___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_17___get__ = &__pyx_type_5_cdec___pyx_scope_struct_17___get__; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_18___iter__) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 52; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_18___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_18___iter__; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_19_todot) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_19_todot = &__pyx_type_5_cdec___pyx_scope_struct_19_todot; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_20_lines) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_20_lines = &__pyx_type_5_cdec___pyx_scope_struct_20_lines; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_21___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __pyx_ptype_5_cdec___pyx_scope_struct_21___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_21___iter__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_22__make_config) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_22__make_config = &__pyx_type_5_cdec___pyx_scope_struct_22__make_config; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_23___cinit__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_23___cinit__ = &__pyx_type_5_cdec___pyx_scope_struct_23___cinit__; - if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_24_genexpr) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_5_cdec___pyx_scope_struct_24_genexpr = &__pyx_type_5_cdec___pyx_scope_struct_24_genexpr; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_22___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_22___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_22___iter__; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_23__make_config) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_23__make_config = &__pyx_type_5_cdec___pyx_scope_struct_23__make_config; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_24___cinit__) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_24___cinit__ = &__pyx_type_5_cdec___pyx_scope_struct_24___cinit__; + if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_25_genexpr) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_5_cdec___pyx_scope_struct_25_genexpr = &__pyx_type_5_cdec___pyx_scope_struct_25_genexpr; /*--- Type import code ---*/ - __pyx_ptype_4cdec_2sa_3_sa_Phrase = __Pyx_ImportType("cdec.sa._sa", "Phrase", sizeof(struct __pyx_obj_4cdec_2sa_3_sa_Phrase), 1); if (unlikely(!__pyx_ptype_4cdec_2sa_3_sa_Phrase)) {__pyx_filename = __pyx_f[7]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_vtabptr_4cdec_2sa_3_sa_Phrase = (struct __pyx_vtabstruct_4cdec_2sa_3_sa_Phrase*)__Pyx_GetVtable(__pyx_ptype_4cdec_2sa_3_sa_Phrase->tp_dict); if (unlikely(!__pyx_vtabptr_4cdec_2sa_3_sa_Phrase)) {__pyx_filename = __pyx_f[7]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} - __pyx_ptype_4cdec_2sa_3_sa_Rule = __Pyx_ImportType("cdec.sa._sa", "Rule", sizeof(struct __pyx_obj_4cdec_2sa_3_sa_Rule), 1); if (unlikely(!__pyx_ptype_4cdec_2sa_3_sa_Rule)) {__pyx_filename = __pyx_f[7]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_4cdec_2sa_3_sa_Phrase = __Pyx_ImportType("cdec.sa._sa", "Phrase", sizeof(struct __pyx_obj_4cdec_2sa_3_sa_Phrase), 1); if (unlikely(!__pyx_ptype_4cdec_2sa_3_sa_Phrase)) {__pyx_filename = __pyx_f[6]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_vtabptr_4cdec_2sa_3_sa_Phrase = (struct __pyx_vtabstruct_4cdec_2sa_3_sa_Phrase*)__Pyx_GetVtable(__pyx_ptype_4cdec_2sa_3_sa_Phrase->tp_dict); if (unlikely(!__pyx_vtabptr_4cdec_2sa_3_sa_Phrase)) {__pyx_filename = __pyx_f[6]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} + __pyx_ptype_4cdec_2sa_3_sa_Rule = __Pyx_ImportType("cdec.sa._sa", "Rule", sizeof(struct __pyx_obj_4cdec_2sa_3_sa_Rule), 1); if (unlikely(!__pyx_ptype_4cdec_2sa_3_sa_Rule)) {__pyx_filename = __pyx_f[6]; __pyx_lineno = 7; __pyx_clineno = __LINE__; goto __pyx_L1_error;} /*--- Variable import code ---*/ /*--- Function import code ---*/ __pyx_t_1 = __Pyx_ImportModule("cdec.sa._sa"); if (!__pyx_t_1) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} @@ -28212,7 +28292,7 @@ PyMODINIT_FUNC PyInit__cdec(void) Py_DECREF(__pyx_t_1); __pyx_t_1 = 0; /*--- Execution code ---*/ - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":3 + /* "/home/cdyer/cdec/python/src/grammar.pxi":3 * cimport grammar * cimport cdec.sa._sa as _sa * import cdec.sa._sa as _sa # <<<<<<<<<<<<<< @@ -28230,7 +28310,7 @@ PyMODINIT_FUNC PyInit__cdec(void) if (PyObject_SetAttr(__pyx_m, __pyx_n_s___sa, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":5 + /* "/home/cdyer/cdec/python/src/grammar.pxi":5 * import cdec.sa._sa as _sa * * def _phrase(phrase): # <<<<<<<<<<<<<< @@ -28242,7 +28322,7 @@ PyMODINIT_FUNC PyInit__cdec(void) if (PyObject_SetAttr(__pyx_m, __pyx_n_s___phrase, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":190 + /* "/home/cdyer/cdec/python/src/mteval.pxi":190 * return [] * * BLEU = Scorer('IBM_BLEU') # <<<<<<<<<<<<<< @@ -28254,7 +28334,7 @@ PyMODINIT_FUNC PyInit__cdec(void) if (PyObject_SetAttr(__pyx_m, __pyx_n_s__BLEU, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":191 + /* "/home/cdyer/cdec/python/src/mteval.pxi":191 * * BLEU = Scorer('IBM_BLEU') * TER = Scorer('TER') # <<<<<<<<<<<<<< @@ -28265,7 +28345,7 @@ PyMODINIT_FUNC PyInit__cdec(void) if (PyObject_SetAttr(__pyx_m, __pyx_n_s__TER, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":192 + /* "/home/cdyer/cdec/python/src/mteval.pxi":192 * BLEU = Scorer('IBM_BLEU') * TER = Scorer('TER') * CER = Scorer('CER') # <<<<<<<<<<<<<< @@ -28356,14 +28436,6 @@ PyMODINIT_FUNC PyInit__cdec(void) __Pyx_GOTREF(((PyObject *)__pyx_t_3)); if (PyObject_SetAttr(__pyx_m, __pyx_n_s____test__, ((PyObject *)__pyx_t_3)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 1; __pyx_clineno = __LINE__; goto __pyx_L1_error;} __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0; - - /* "string.to_py":25 - * - * @cname("__pyx_convert_string_to_py_") - * cdef object __pyx_convert_string_to_py_(string& s): # <<<<<<<<<<<<<< - * return s.data()[:s.size()] - * - */ goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); @@ -28590,7 +28662,7 @@ static void __Pyx_RaiseArgtupleInvalid( more_or_less = "exactly"; } PyErr_Format(PyExc_TypeError, - "%s() takes %s %" PY_FORMAT_SIZE_T "d positional argument%s (%" PY_FORMAT_SIZE_T "d given)", + "%s() takes %s %"PY_FORMAT_SIZE_T"d positional argument%s (%"PY_FORMAT_SIZE_T"d given)", func_name, more_or_less, num_expected, (num_expected == 1) ? "" : "s", num_found); } @@ -28602,11 +28674,6 @@ static CYTHON_INLINE int __Pyx_CheckKeywordStrings( { PyObject* key = 0; Py_ssize_t pos = 0; -#if CPYTHON_COMPILING_IN_PYPY - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) - goto invalid_keyword; - return 1; -#else while (PyDict_Next(kwdict, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) @@ -28622,7 +28689,6 @@ invalid_keyword_type: PyErr_Format(PyExc_TypeError, "%s() keywords must be strings", function_name); return 0; -#endif invalid_keyword: PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 @@ -28761,49 +28827,17 @@ bad: return -1; } -static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { - PyErr_Format(PyExc_ValueError, - "too many values to unpack (expected %" PY_FORMAT_SIZE_T "d)", expected); -} + static CYTHON_INLINE void __Pyx_RaiseNeedMoreValuesError(Py_ssize_t index) { PyErr_Format(PyExc_ValueError, - "need more than %" PY_FORMAT_SIZE_T "d value%s to unpack", + "need more than %"PY_FORMAT_SIZE_T"d value%s to unpack", index, (index == 1) ? "" : "s"); } -static CYTHON_INLINE int __Pyx_IterFinish(void) { -#if CYTHON_COMPILING_IN_CPYTHON - PyThreadState *tstate = PyThreadState_GET(); - PyObject* exc_type = tstate->curexc_type; - if (unlikely(exc_type)) { - if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { - PyObject *exc_value, *exc_tb; - exc_value = tstate->curexc_value; - exc_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; - Py_DECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - return 0; - } else { - return -1; - } - } - return 0; -#else - if (unlikely(PyErr_Occurred())) { - if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { - PyErr_Clear(); - return 0; - } else { - return -1; - } - } - return 0; -#endif +static CYTHON_INLINE void __Pyx_RaiseTooManyValuesError(Py_ssize_t expected) { + PyErr_Format(PyExc_ValueError, + "too many values to unpack (expected %"PY_FORMAT_SIZE_T"d)", expected); } static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { @@ -28811,54 +28845,45 @@ static int __Pyx_IternextUnpackEndCheck(PyObject *retval, Py_ssize_t expected) { Py_DECREF(retval); __Pyx_RaiseTooManyValuesError(expected); return -1; - } else { - return __Pyx_IterFinish(); + } else if (PyErr_Occurred()) { + if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) { + PyErr_Clear(); + return 0; + } else { + return -1; + } } return 0; } static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* defval) { PyObject* next; - iternextfunc iternext = Py_TYPE(iterator)->tp_iternext; -#if CYTHON_COMPILING_IN_CPYTHON - if (unlikely(!iternext)) { -#else - if (unlikely(!iternext) || unlikely(!PyIter_Check(iterator))) { -#endif + if (unlikely(!PyIter_Check(iterator))) { PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", Py_TYPE(iterator)->tp_name); + "%.200s object is not an iterator", iterator->ob_type->tp_name); return NULL; } - next = iternext(iterator); - if (likely(next)) + next = (*(Py_TYPE(iterator)->tp_iternext))(iterator); + if (likely(next)) { return next; -#if CYTHON_COMPILING_IN_CPYTHON -#if PY_VERSION_HEX >= 0x03010000 || (PY_MAJOR_VERSION < 3 && PY_VERSION_HEX >= 0x02070000) - if (unlikely(iternext == &_PyObject_NextNotImplemented)) - return NULL; -#endif -#endif - if (defval) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (unlikely(exc_type != PyExc_StopIteration) && - !PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) + } else if (defval) { + if (PyErr_Occurred()) { + if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } Py_INCREF(defval); return defval; - } - if (!PyErr_Occurred()) + } else if (PyErr_Occurred()) { + return NULL; + } else { PyErr_SetNone(PyExc_StopIteration); - return NULL; + return NULL; + } } static double __Pyx__PyObject_AsDouble(PyObject* obj) { PyObject* float_value; -#if CYTHON_COMPILING_IN_PYPY - float_value = PyNumber_Float(obj); -#else if (Py_TYPE(obj)->tp_as_number && Py_TYPE(obj)->tp_as_number->nb_float) { return PyFloat_AsDouble(obj); } else if (PyUnicode_CheckExact(obj) || PyBytes_CheckExact(obj)) { @@ -28875,7 +28900,6 @@ static double __Pyx__PyObject_AsDouble(PyObject* obj) { PyTuple_SET_ITEM(args, 0, 0); Py_DECREF(args); } -#endif if (likely(float_value)) { double value = PyFloat_AS_DOUBLE(float_value); Py_DECREF(float_value); @@ -28887,7 +28911,6 @@ bad: static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { PyObject *local_type, *local_value, *local_tb; -#if CYTHON_COMPILING_IN_CPYTHON PyObject *tmp_type, *tmp_value, *tmp_tb; PyThreadState *tstate = PyThreadState_GET(); local_type = tstate->curexc_type; @@ -28896,27 +28919,19 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) tstate->curexc_type = 0; tstate->curexc_value = 0; tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_COMPILING_IN_CPYTHON if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif goto bad; #if PY_MAJOR_VERSION >= 3 if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) goto bad; #endif - Py_INCREF(local_type); - Py_INCREF(local_value); - Py_INCREF(local_tb); *type = local_type; *value = local_value; *tb = local_tb; -#if CYTHON_COMPILING_IN_CPYTHON + Py_INCREF(local_type); + Py_INCREF(local_value); + Py_INCREF(local_tb); tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; @@ -28924,13 +28939,10 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) tstate->exc_value = local_value; tstate->exc_traceback = local_tb; /* Make sure tstate is in a consistent state when we XDECREF - these objects (DECREF may run arbitrary code). */ + these objects (XDECREF may run arbitrary code). */ Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif return 0; bad: *type = 0; @@ -28943,7 +28955,6 @@ bad: } static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) { -#if CYTHON_COMPILING_IN_CPYTHON PyThreadState *tstate = PyThreadState_GET(); *type = tstate->exc_type; *value = tstate->exc_value; @@ -28951,12 +28962,8 @@ static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, Py_XINCREF(*type); Py_XINCREF(*value); Py_XINCREF(*tb); -#else - PyErr_GetExcInfo(type, value, tb); -#endif } static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) { -#if CYTHON_COMPILING_IN_CPYTHON PyObject *tmp_type, *tmp_value, *tmp_tb; PyThreadState *tstate = PyThreadState_GET(); tmp_type = tstate->exc_type; @@ -28968,9 +28975,6 @@ static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb) Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(type, value, tb); -#endif } static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, long level) { @@ -29044,6 +29048,7 @@ bad: static PyObject *__Pyx_FindPy2Metaclass(PyObject *bases) { PyObject *metaclass; + /* Default metaclass */ #if PY_MAJOR_VERSION < 3 if (PyTuple_Check(bases) && PyTuple_GET_SIZE(bases) > 0) { PyObject *base = PyTuple_GET_ITEM(bases, 0); @@ -29073,6 +29078,7 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na PyObject *metaclass; if (PyDict_SetItemString(dict, "__module__", modname) < 0) return NULL; + /* Python2 __metaclass__ */ metaclass = PyDict_GetItemString(dict, "__metaclass__"); if (metaclass) { Py_INCREF(metaclass); @@ -29454,56 +29460,6 @@ __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) PyString_AsString(func_name), (void *)op); #endif } -#if CYTHON_COMPILING_IN_PYPY -static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - PyCFunctionObject* f = (PyCFunctionObject*)func; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - Py_ssize_t size; - switch (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)) { - case METH_VARARGS: - if (likely(kw == NULL) || PyDict_Size(kw) == 0) - return (*meth)(self, arg); - break; - case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); - case METH_NOARGS: - if (likely(kw == NULL) || PyDict_Size(kw) == 0) { - size = PyTuple_GET_SIZE(arg); - if (size == 0) - return (*meth)(self, NULL); - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - case METH_O: - if (likely(kw == NULL) || PyDict_Size(kw) == 0) { - size = PyTuple_GET_SIZE(arg); - if (size == 1) - return (*meth)(self, PyTuple_GET_ITEM(arg, 0)); - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - f->m_ml->ml_name, size); - return NULL; - } - break; - default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - return NULL; - } - PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - f->m_ml->ml_name); - return NULL; -} -#else -static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - return PyCFunction_Call(func, arg, kw); -} -#endif static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("cython_function_or_method"), /*tp_name*/ @@ -29523,7 +29479,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash*/ - __Pyx_CyFunction_Call, /*tp_call*/ + __Pyx_PyCFunction_Call, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ @@ -29559,16 +29515,15 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_version_tag*/ #endif }; -static int __Pyx_CyFunction_init(void) { -#if !CYTHON_COMPILING_IN_PYPY - __pyx_CyFunctionType_type.tp_call = PyCFunction_Call; -#endif +static int __Pyx_CyFunction_init(void) +{ if (PyType_Ready(&__pyx_CyFunctionType_type) < 0) return -1; __pyx_CyFunctionType = &__pyx_CyFunctionType_type; return 0; } -static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { +void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) +{ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; m->defaults = PyMem_Malloc(size); if (!m->defaults) @@ -29577,16 +29532,14 @@ static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t m->defaults_pyobjects = pyobjects; return m->defaults; } -static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { +static void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) +{ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; m->defaults_tuple = tuple; Py_INCREF(tuple); } static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY - return PyObject_RichCompareBool(s1, s2, equals); -#else if (s1 == s2) { return (equals == Py_EQ); } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { @@ -29614,13 +29567,9 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq Py_DECREF(py_result); return result; } -#endif } static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY - return PyObject_RichCompareBool(s1, s2, equals); -#else if (s1 == s2) { return (equals == Py_EQ); } else if (PyUnicode_CheckExact(s1) & PyUnicode_CheckExact(s2)) { @@ -29660,7 +29609,6 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int Py_DECREF(py_result); return result; } -#endif } static CYTHON_INLINE unsigned char __Pyx_PyInt_AsUnsignedChar(PyObject* x) { @@ -30063,8 +30011,8 @@ static CYTHON_INLINE signed PY_LONG_LONG __Pyx_PyInt_AsSignedLongLong(PyObject* } } -static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename) { +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename) { PyObject *old_exc, *old_val, *old_tb; PyObject *ctx; __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); @@ -30084,7 +30032,6 @@ static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; -#if CYTHON_COMPILING_IN_CPYTHON PyThreadState *tstate = PyThreadState_GET(); tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; @@ -30092,10 +30039,6 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, tstate->exc_type = *type; tstate->exc_value = *value; tstate->exc_traceback = *tb; -#else - PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); - PyErr_SetExcInfo(*type, *value, *tb); -#endif *type = tmp_type; *value = tmp_value; *tb = tmp_tb; @@ -30105,70 +30048,9 @@ static PyObject *__Pyx_Generator_Next(PyObject *self); static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value); static PyObject *__Pyx_Generator_Close(PyObject *self); static PyObject *__Pyx_Generator_Throw(PyObject *gen, PyObject *args); -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) -#define __Pyx_Generator_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; - PyObject *value = NULL; - __Pyx_ErrFetch(&et, &ev, &tb); - if (!et) { - Py_XDECREF(tb); - Py_XDECREF(ev); - Py_INCREF(Py_None); - *pvalue = Py_None; - return 0; - } - if (unlikely(et != PyExc_StopIteration) && - unlikely(!PyErr_GivenExceptionMatches(et, PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - if (likely(et == PyExc_StopIteration)) { - if (likely(!ev) || !PyObject_IsInstance(ev, PyExc_StopIteration)) { - if (!ev) { - Py_INCREF(Py_None); - ev = Py_None; - } - Py_XDECREF(tb); - Py_DECREF(et); - *pvalue = ev; - return 0; - } - } - PyErr_NormalizeException(&et, &ev, &tb); - if (unlikely(!PyObject_IsInstance(ev, PyExc_StopIteration))) { - __Pyx_ErrRestore(et, ev, tb); - return -1; - } - Py_XDECREF(tb); - Py_DECREF(et); -#if PY_VERSION_HEX >= 0x030300A0 - value = ((PyStopIterationObject *)ev)->value; - Py_INCREF(value); - Py_DECREF(ev); -#else - { - PyObject* args = PyObject_GetAttrString(ev, "args"); - Py_DECREF(ev); - if (likely(args)) { - value = PyObject_GetItem(args, 0); - Py_DECREF(args); - } - if (unlikely(!value)) { - __Pyx_ErrRestore(NULL, NULL, NULL); - Py_INCREF(Py_None); - value = Py_None; - } - } -#endif - *pvalue = value; - return 0; -} -#endif static CYTHON_INLINE -void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self) { +void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self) +{ PyObject *exc_type = self->exc_type; PyObject *exc_value = self->exc_value; PyObject *exc_traceback = self->exc_traceback; @@ -30180,18 +30062,14 @@ void __Pyx_Generator_ExceptionClear(__pyx_GeneratorObject *self) { Py_XDECREF(exc_traceback); } static CYTHON_INLINE -int __Pyx_Generator_CheckRunning(__pyx_GeneratorObject *gen) { - if (unlikely(gen->is_running)) { +PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value) +{ + PyObject *retval; + if (unlikely(self->is_running)) { PyErr_SetString(PyExc_ValueError, "generator already executing"); - return 1; + return NULL; } - return 0; -} -static CYTHON_INLINE -PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value) { - PyObject *retval; - assert(!self->is_running); if (unlikely(self->resume_label == 0)) { if (unlikely(value && value != Py_None)) { PyErr_SetString(PyExc_TypeError, @@ -30204,240 +30082,81 @@ PyObject *__Pyx_Generator_SendEx(__pyx_GeneratorObject *self, PyObject *value) { PyErr_SetNone(PyExc_StopIteration); return NULL; } - if (value) { -#if CYTHON_COMPILING_IN_PYPY -#else - /* Generators always return to their most recent caller, not - * necessarily their creator. */ - if (self->exc_traceback) { - PyThreadState *tstate = PyThreadState_GET(); - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_XINCREF(tstate->frame); - assert(f->f_back == NULL); - f->f_back = tstate->frame; - } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); - } else { + if (value) + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback); + else __Pyx_Generator_ExceptionClear(self); - } self->is_running = 1; retval = self->body((PyObject *) self, value); self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY -#else - /* Don't keep the reference to f_back any longer than necessary. It - * may keep a chain of frames alive or it could create a reference - * cycle. */ - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } -#endif - } else { + if (retval) + __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, &self->exc_traceback); + else __Pyx_Generator_ExceptionClear(self); - } return retval; } -static CYTHON_INLINE -PyObject *__Pyx_Generator_FinishDelegation(__pyx_GeneratorObject *gen) { - PyObject *ret; - PyObject *val = NULL; - __Pyx_Generator_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); - ret = __Pyx_Generator_SendEx(gen, val); - Py_XDECREF(val); - return ret; -} -static PyObject *__Pyx_Generator_Next(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - ret = Py_TYPE(yf)->tp_iternext(yf); - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Generator_FinishDelegation(gen); - } - return __Pyx_Generator_SendEx(gen, Py_None); +static PyObject *__Pyx_Generator_Next(PyObject *self) +{ + return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, Py_None); } -static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject*) self; - PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - gen->is_running = 1; - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Send(yf, value); - } else { - if (value == Py_None) - ret = PyIter_Next(yf); - else - ret = PyObject_CallMethod(yf, (char*)"send", (char*)"O", value); - } - gen->is_running = 0; - if (likely(ret)) { - return ret; - } - return __Pyx_Generator_FinishDelegation(gen); - } - return __Pyx_Generator_SendEx(gen, value); -} -static int __Pyx_Generator_CloseIter(__pyx_GeneratorObject *gen, PyObject *yf) { - PyObject *retval = NULL; - int err = 0; - if (__Pyx_Generator_CheckExact(yf)) { - retval = __Pyx_Generator_Close(yf); - if (!retval) - return -1; - } else { - PyObject *meth; - gen->is_running = 1; - meth = PyObject_GetAttrString(yf, "close"); - if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_WriteUnraisable(yf); - } - PyErr_Clear(); - } else { - retval = PyObject_CallFunction(meth, NULL); - Py_DECREF(meth); - if (!retval) - err = -1; - } - gen->is_running = 0; - } - Py_XDECREF(retval); - return err; +static PyObject *__Pyx_Generator_Send(PyObject *self, PyObject *value) +{ + return __Pyx_Generator_SendEx((__pyx_GeneratorObject *) self, value); } -static PyObject *__Pyx_Generator_Close(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - PyObject *retval, *raised_exception; - PyObject *yf = gen->yieldfrom; - int err = 0; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - Py_INCREF(yf); - err = __Pyx_Generator_CloseIter(gen, yf); - __Pyx_Generator_Undelegate(gen); - Py_DECREF(yf); - } - if (err == 0) +static PyObject *__Pyx_Generator_Close(PyObject *self) +{ + __pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self; + PyObject *retval; #if PY_VERSION_HEX < 0x02050000 - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetNone(PyExc_StopIteration); #else - PyErr_SetNone(PyExc_GeneratorExit); + PyErr_SetNone(PyExc_GeneratorExit); #endif - retval = __Pyx_Generator_SendEx(gen, NULL); + retval = __Pyx_Generator_SendEx(generator, NULL); if (retval) { Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, "generator ignored GeneratorExit"); return NULL; } - raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration -#if PY_VERSION_HEX >= 0x02050000 - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) +#if PY_VERSION_HEX < 0x02050000 + if (PyErr_ExceptionMatches(PyExc_StopIteration)) +#else + if (PyErr_ExceptionMatches(PyExc_StopIteration) + || PyErr_ExceptionMatches(PyExc_GeneratorExit)) #endif - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) { - if (raised_exception) PyErr_Clear(); /* ignore these errors */ + PyErr_Clear(); /* ignore these errors */ Py_INCREF(Py_None); return Py_None; } return NULL; } -static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; +static PyObject *__Pyx_Generator_Throw(PyObject *self, PyObject *args) +{ + __pyx_GeneratorObject *generator = (__pyx_GeneratorObject *) self; PyObject *typ; PyObject *tb = NULL; PyObject *val = NULL; - PyObject *yf = gen->yieldfrom; if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) return NULL; - if (unlikely(__Pyx_Generator_CheckRunning(gen))) - return NULL; - if (yf) { - PyObject *ret; - Py_INCREF(yf); -#if PY_VERSION_HEX >= 0x02050000 - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { - int err = __Pyx_Generator_CloseIter(gen, yf); - Py_DECREF(yf); - __Pyx_Generator_Undelegate(gen); - if (err < 0) - return __Pyx_Generator_SendEx(gen, NULL); - goto throw_here; - } -#endif - gen->is_running = 1; - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Generator_Throw(yf, args); - } else { - PyObject *meth = PyObject_GetAttrString(yf, "throw"); - if (unlikely(!meth)) { - Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - gen->is_running = 0; - return NULL; - } - PyErr_Clear(); - __Pyx_Generator_Undelegate(gen); - gen->is_running = 0; - goto throw_here; - } - ret = PyObject_CallObject(meth, args); - Py_DECREF(meth); - } - gen->is_running = 0; - Py_DECREF(yf); - if (!ret) { - ret = __Pyx_Generator_FinishDelegation(gen); - } - return ret; - } -throw_here: __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Generator_SendEx(gen, NULL); + return __Pyx_Generator_SendEx(generator, NULL); } -static int __Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg) { +static int +__Pyx_Generator_traverse(PyObject *self, visitproc visit, void *arg) +{ __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; Py_VISIT(gen->closure); Py_VISIT(gen->classobj); - Py_VISIT(gen->yieldfrom); Py_VISIT(gen->exc_type); Py_VISIT(gen->exc_value); Py_VISIT(gen->exc_traceback); return 0; } -static int __Pyx_Generator_clear(PyObject *self) { - __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; - Py_CLEAR(gen->closure); - Py_CLEAR(gen->classobj); - Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); - return 0; -} -static void __Pyx_Generator_dealloc(PyObject *self) { +static void +__Pyx_Generator_dealloc(PyObject *self) +{ __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; PyObject_GC_UnTrack(gen); if (gen->gi_weakreflist != NULL) @@ -30449,10 +30168,16 @@ static void __Pyx_Generator_dealloc(PyObject *self) { return; /* resurrected. :( */ } PyObject_GC_UnTrack(self); - __Pyx_Generator_clear(self); + Py_CLEAR(gen->closure); + Py_CLEAR(gen->classobj); + Py_CLEAR(gen->exc_type); + Py_CLEAR(gen->exc_value); + Py_CLEAR(gen->exc_traceback); PyObject_GC_Del(gen); } -static void __Pyx_Generator_del(PyObject *self) { +static void +__Pyx_Generator_del(PyObject *self) +{ PyObject *res; PyObject *error_type, *error_value, *error_traceback; __pyx_GeneratorObject *gen = (__pyx_GeneratorObject *) self; @@ -30481,13 +30206,11 @@ static void __Pyx_Generator_del(PyObject *self) { _Py_NewReference(self); self->ob_refcnt = refcnt; } -#if CYTHON_COMPILING_FOR_CPYTHON assert(PyType_IS_GC(self->ob_type) && _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so * we need to undo that. */ _Py_DEC_REFTOTAL; -#endif /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object * chain, so no more to do there. * If COUNT_ALLOCS, the original decref bumped tp_frees, and @@ -30495,17 +30218,13 @@ static void __Pyx_Generator_del(PyObject *self) { * undone. */ #ifdef COUNT_ALLOCS - --Py_TYPE(self)->tp_frees; - --Py_TYPE(self)->tp_allocs; + --self->ob_type->tp_frees; + --self->ob_type->tp_allocs; #endif } static PyMemberDef __pyx_Generator_memberlist[] = { {(char *) "gi_running", -#if PY_VERSION_HEX >= 0x02060000 - T_BOOL, -#else T_INT, -#endif offsetof(__pyx_GeneratorObject, is_running), READONLY, NULL}, @@ -30517,7 +30236,7 @@ static PyMethodDef __pyx_Generator_methods[] = { {__Pyx_NAMESTR("close"), (PyCFunction) __Pyx_Generator_Close, METH_NOARGS, 0}, {0, 0, 0, 0} }; -static PyTypeObject __pyx_GeneratorType_type = { +static PyTypeObject __pyx_GeneratorType = { PyVarObject_HEAD_INIT(0, 0) __Pyx_NAMESTR("generator"), /*tp_name*/ sizeof(__pyx_GeneratorObject), /*tp_basicsize*/ @@ -30538,7 +30257,7 @@ static PyTypeObject __pyx_GeneratorType_type = { 0, /*tp_hash*/ 0, /*tp_call*/ 0, /*tp_str*/ - 0, /*tp_getattro*/ + PyObject_GenericGetAttr, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags*/ @@ -30547,7 +30266,7 @@ static PyTypeObject __pyx_GeneratorType_type = { 0, /*tp_clear*/ 0, /*tp_richcompare*/ offsetof(__pyx_GeneratorObject, gi_weakreflist), /* tp_weaklistoffse */ - 0, /*tp_iter*/ + PyObject_SelfIter, /*tp_iter*/ (iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/ __pyx_Generator_methods, /*tp_methods*/ __pyx_Generator_memberlist, /*tp_members*/ @@ -30572,10 +30291,12 @@ static PyTypeObject __pyx_GeneratorType_type = { 0, /*tp_version_tag*/ #endif }; -static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, - PyObject *closure) { +static +__pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, + PyObject *closure) +{ __pyx_GeneratorObject *gen = - PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType_type); + PyObject_GC_New(__pyx_GeneratorObject, &__pyx_GeneratorType); if (gen == NULL) return NULL; gen->body = body; @@ -30584,7 +30305,6 @@ static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, gen->is_running = 0; gen->resume_label = 0; gen->classobj = NULL; - gen->yieldfrom = NULL; gen->exc_type = NULL; gen->exc_value = NULL; gen->exc_traceback = NULL; @@ -30592,14 +30312,9 @@ static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body, PyObject_GC_Track(gen); return gen; } -static int __pyx_Generator_init(void) { - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; - __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - if (PyType_Ready(&__pyx_GeneratorType_type)) { - return -1; - } - __pyx_GeneratorType = &__pyx_GeneratorType_type; - return 0; +static int __pyx_Generator_init(void) +{ + return PyType_Ready(&__pyx_GeneratorType); } static int __Pyx_check_binary_version(void) { -- cgit v1.2.3 From 0ec6eab158320dc87054057a5a6aaa3536d2fc91 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Thu, 9 Aug 2012 01:36:36 -0400 Subject: fixes for new word api --- decoder/ff_charset.cc | 6 +++--- decoder/ff_csplit.cc | 2 +- decoder/ff_wordalign.cc | 2 +- decoder/hg_intersect.cc | 2 +- decoder/hg_io.cc | 2 +- gi/pf/base_distributions.cc | 2 +- gi/pf/cfg_wfst_composer.cc | 2 +- mteval/ns.cc | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) (limited to 'decoder') diff --git a/decoder/ff_charset.cc b/decoder/ff_charset.cc index 33afc1a7..472de82b 100644 --- a/decoder/ff_charset.cc +++ b/decoder/ff_charset.cc @@ -7,9 +7,9 @@ using namespace std; NonLatinCount::NonLatinCount(const string& param) : FeatureFunction(), fid_(FD::Convert("NonLatinCount")) {} -bool ContainsNonLatin(const char* word) { - int cur = 0; - while(word[cur]) { +bool ContainsNonLatin(const string& word) { + unsigned cur = 0; + while(cur < word.size()) { const int size = UTF8Len(word[cur]); if (size > 1) return true; cur += size; diff --git a/decoder/ff_csplit.cc b/decoder/ff_csplit.cc index c9ed996c..252dbf8c 100644 --- a/decoder/ff_csplit.cc +++ b/decoder/ff_csplit.cc @@ -88,7 +88,7 @@ void BasicCSplitFeaturesImpl::TraversalFeaturesImpl( features->set_value(letters_sq_, (edge.j_ - edge.i_) * (edge.j_ - edge.i_)); features->set_value(letters_sqrt_, sqrt(edge.j_ - edge.i_)); const WordID word = edge.rule_->e_[1]; - const char* sword = TD::Convert(word); + const char* sword = TD::Convert(word).c_str(); const int len = strlen(sword); int cur = 0; int chars = 0; diff --git a/decoder/ff_wordalign.cc b/decoder/ff_wordalign.cc index decdf9bc..1491819d 100644 --- a/decoder/ff_wordalign.cc +++ b/decoder/ff_wordalign.cc @@ -549,7 +549,7 @@ void IdentityCycleDetector::TraversalFeaturesImpl(const SentenceMetadata& smeta, static map big_enough; map::iterator it = big_enough_.find(word); if (it == big_enough_.end()) { - out_is_identity = big_enough_[word] = strlen(TD::Convert(word)) >= length_min_; + out_is_identity = big_enough_[word] = TD::Convert(word).size() >= length_min_; } else { out_is_identity = it->second; } diff --git a/decoder/hg_intersect.cc b/decoder/hg_intersect.cc index 6e3bfee6..e9a91061 100644 --- a/decoder/hg_intersect.cc +++ b/decoder/hg_intersect.cc @@ -101,7 +101,7 @@ bool HG::Intersect(const Lattice& target, Hypergraph* hg) { // grammar, create the labels here const string kSEP = "_"; for (unsigned i = 0; i < nnodes; ++i) { - const char* pstr = "CAT"; + string pstr = "CAT"; if (hg->nodes_[i].cat_ < 0) pstr = TD::Convert(-hg->nodes_[i].cat_); cats[i] = TD::Convert(pstr + kSEP + lexical_cast(i)) * -1; diff --git a/decoder/hg_io.cc b/decoder/hg_io.cc index 8bd40387..3a68a429 100644 --- a/decoder/hg_io.cc +++ b/decoder/hg_io.cc @@ -600,7 +600,7 @@ void HypergraphIO::WriteAsCFG(const Hypergraph& hg) { // grammar, create the labels here const string kSEP = "_"; for (int i = 0; i < hg.nodes_.size(); ++i) { - const char* pstr = "CAT"; + string pstr = "CAT"; if (hg.nodes_[i].cat_ < 0) pstr = TD::Convert(-hg.nodes_[i].cat_); cats[i] = TD::Convert(pstr + kSEP + boost::lexical_cast(i)) * -1; diff --git a/gi/pf/base_distributions.cc b/gi/pf/base_distributions.cc index d9761005..57e0bbe1 100644 --- a/gi/pf/base_distributions.cc +++ b/gi/pf/base_distributions.cc @@ -37,7 +37,7 @@ TableLookupBase::TableLookupBase(const string& fname) { } else if (cc == 1) { x.e_.push_back(cur); } else if (cc == 2) { - table[x].logeq(atof(TD::Convert(cur))); + table[x].logeq(atof(TD::Convert(cur).c_str())); ++cc; } else { if (flag) cerr << endl; diff --git a/gi/pf/cfg_wfst_composer.cc b/gi/pf/cfg_wfst_composer.cc index 19c0875d..21d5ec5b 100644 --- a/gi/pf/cfg_wfst_composer.cc +++ b/gi/pf/cfg_wfst_composer.cc @@ -410,7 +410,7 @@ class CFG_WFSTComposerImpl { std::cerr << "TERMINAL SYMBOL: " << TD::Convert(git->first) << endl; abort(); } - std::vector > extensions = r->ExtendInput(atoi(TD::Convert(git->first))); + std::vector > extensions = r->ExtendInput(atoi(TD::Convert(git->first).c_str())); for (unsigned nsi = 0; nsi < extensions.size(); ++nsi) { const WFSTNode* next_r = extensions[nsi].first; const EGrammarNode* next_dot = &git->second; diff --git a/mteval/ns.cc b/mteval/ns.cc index 33952da7..3af7cc63 100644 --- a/mteval/ns.cc +++ b/mteval/ns.cc @@ -252,7 +252,7 @@ EvaluationMetric* EvaluationMetric::Instance(const string& imetric_id) { } else if (metric_id == "TER") { m = new TERMetric; } else if (metric_id == "METEOR") { - m = new ExternalMetric("METEOR", "java -Xmx1536m -jar /Users/cdyer/software/meteor/meteor-1.3.jar - - -mira -lower -t tune -l en"); + m = new ExternalMetric("METEOR", "java -Xmx1536m -jar /cab0/tools/meteor-1.3/meteor-1.3.jar - - -mira -lower -t tune -l en"); } else if (metric_id.find("COMB:") == 0) { m = new CombinationMetric(metric_id); } else if (metric_id == "CER") { -- cgit v1.2.3 From c5d87ccefa6e94857bf6de7f1e25c46c2c50979e Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 12 Aug 2012 23:15:14 -0400 Subject: nasty unit test failure --- decoder/hg_test.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'decoder') diff --git a/decoder/hg_test.h b/decoder/hg_test.h index 2e308c37..c098961d 100644 --- a/decoder/hg_test.h +++ b/decoder/hg_test.h @@ -64,12 +64,20 @@ Name HGjsons[]= { } +void AddNullEdge(Hypergraph* hg) { + TRule x; + x.arity_ = 0; + hg->nodes_[0].in_edges_.push_back(hg->AddEdge(TRulePtr(new TRule(x)), Hypergraph::TailNodeVector())->id_); +} + void HGSetup::CreateTinyLatticeHG(Hypergraph* hg) { Json(hg,HGjsons[TinyLatticeHG]); + AddNullEdge(hg); } void HGSetup::CreateLatticeHG(Hypergraph* hg) { Json(hg,HGjsons[LatticeHG]); + AddNullEdge(hg); } void HGSetup::CreateHG_tiny(Hypergraph* hg) { -- cgit v1.2.3 From b6e70b420ed993ee73f71058d04b382147896068 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 12 Aug 2012 23:33:21 -0400 Subject: use new union api --- decoder/Makefile.am | 1 + decoder/decoder.cc | 5 +- decoder/hg_test.cc | 7 +- decoder/hg_union.cc | 58 +++ decoder/hg_union.h | 9 + extools/Makefile.am | 30 -- extools/README | 32 -- extools/coarsen_grammar.pl | 133 ------- extools/extract.cc | 336 ----------------- extools/extract.h | 94 ----- extools/extractor.cc | 439 ----------------------- extools/extractor_monolingual.cc | 256 ------------- extools/featurize_grammar.cc | 716 ------------------------------------- extools/filter_grammar.cc | 135 ------- extools/lex_trans_tbl.h | 25 -- extools/merge_lines.pl | 43 --- extools/mr_stripe_rule_reduce.cc | 172 --------- extools/score_grammar.cc | 352 ------------------ extools/sentence_pair.cc | 198 ---------- extools/sentence_pair.h | 43 --- extools/sg_lexer.l | 294 --------------- extools/simple-extract-context.sh | 9 - extools/simple-extract.sh | 11 - extools/striped_grammar.cc | 67 ---- extools/striped_grammar.h | 56 --- extools/suffix_tree.h | 46 --- extools/test_data/README | 10 - extools/test_data/corpus.aligned | 5 - extools/test_data/corpus.en | 5 - extools/test_data/corpus.fr | 5 - extools/test_data/corpus.len_cats | 5 - extools/test_data/fr-en.al.len | 5 - extools/test_data/make_len_cats.pl | 23 -- 33 files changed, 75 insertions(+), 3550 deletions(-) create mode 100644 decoder/hg_union.cc create mode 100644 decoder/hg_union.h delete mode 100644 extools/Makefile.am delete mode 100644 extools/README delete mode 100755 extools/coarsen_grammar.pl delete mode 100644 extools/extract.cc delete mode 100644 extools/extract.h delete mode 100644 extools/extractor.cc delete mode 100644 extools/extractor_monolingual.cc delete mode 100644 extools/featurize_grammar.cc delete mode 100644 extools/filter_grammar.cc delete mode 100644 extools/lex_trans_tbl.h delete mode 100755 extools/merge_lines.pl delete mode 100644 extools/mr_stripe_rule_reduce.cc delete mode 100644 extools/score_grammar.cc delete mode 100644 extools/sentence_pair.cc delete mode 100644 extools/sentence_pair.h delete mode 100644 extools/sg_lexer.l delete mode 100755 extools/simple-extract-context.sh delete mode 100755 extools/simple-extract.sh delete mode 100644 extools/striped_grammar.cc delete mode 100644 extools/striped_grammar.h delete mode 100644 extools/suffix_tree.h delete mode 100644 extools/test_data/README delete mode 100644 extools/test_data/corpus.aligned delete mode 100644 extools/test_data/corpus.en delete mode 100644 extools/test_data/corpus.fr delete mode 100644 extools/test_data/corpus.len_cats delete mode 100644 extools/test_data/fr-en.al.len delete mode 100755 extools/test_data/make_len_cats.pl (limited to 'decoder') diff --git a/decoder/Makefile.am b/decoder/Makefile.am index 0a792549..4a98a4f1 100644 --- a/decoder/Makefile.am +++ b/decoder/Makefile.am @@ -44,6 +44,7 @@ libcdec_a_SOURCES = \ hg_remove_eps.cc \ decoder.cc \ hg_intersect.cc \ + hg_union.cc \ hg_sampler.cc \ factored_lexicon_helper.cc \ viterbi.cc \ diff --git a/decoder/decoder.cc b/decoder/decoder.cc index a6f7b1ce..a69a6d05 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -24,6 +24,7 @@ #include "hg.h" #include "sentence_metadata.h" #include "hg_intersect.h" +#include "hg_union.h" #include "oracle_bleu.h" #include "apply_models.h" @@ -980,7 +981,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { bool succeeded = HypergraphIO::ReadFromJSON(rf.stream(), &new_hg); if (!succeeded) abort(); } - new_hg.Union(forest); + HG::Union(forest, &new_hg); bool succeeded = writer.Write(new_hg, false); if (!succeeded) abort(); } else { @@ -1067,7 +1068,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { bool succeeded = HypergraphIO::ReadFromJSON(rf.stream(), &new_hg); if (!succeeded) abort(); } - new_hg.Union(forest); + HG::Union(forest, &new_hg); bool succeeded = writer.Write(new_hg, false); if (!succeeded) abort(); } else { diff --git a/decoder/hg_test.cc b/decoder/hg_test.cc index 92ed98b2..c47af850 100644 --- a/decoder/hg_test.cc +++ b/decoder/hg_test.cc @@ -6,6 +6,7 @@ #include "json_parse.h" #include "hg_intersect.h" +#include "hg_union.h" #include "viterbi.h" #include "kbest.h" #include "inside_outside.h" @@ -52,7 +53,7 @@ BOOST_AUTO_TEST_CASE(Union) { int l2 = ViterbiPathLength(hg2); cerr << c1 << "\t" << TD::GetString(t1) << endl; cerr << c2 << "\t" << TD::GetString(t2) << endl; - hg1.Union(hg2); + HG::Union(hg2, &hg1); hg1.Reweight(wts); c3 = ViterbiESentence(hg1, &t3); int l3 = ViterbiPathLength(hg1); @@ -121,8 +122,8 @@ BOOST_AUTO_TEST_CASE(InsideScore) { vector post; inside = hg.ComputeBestPathThroughEdges(&post); BOOST_CHECK_CLOSE(-0.3, log(inside), 1e-4); // computed by hand - BOOST_CHECK_EQUAL(post.size(), 4); - for (int i = 0; i < 4; ++i) { + BOOST_CHECK_EQUAL(post.size(), 5); + for (int i = 0; i < 5; ++i) { cerr << "edge post: " << log(post[i]) << '\t' << hg.edges_[i].rule_->AsString() << endl; } } diff --git a/decoder/hg_union.cc b/decoder/hg_union.cc new file mode 100644 index 00000000..37082976 --- /dev/null +++ b/decoder/hg_union.cc @@ -0,0 +1,58 @@ +#include "hg_union.h" + +#include "hg.h" + +using namespace std; + +namespace HG { + +void Union(const Hypergraph& in, Hypergraph* out) { + if (&in == out) return; + if (out->nodes_.empty()) { + out->nodes_ = in.nodes_; + out->edges_ = in.edges_; return; + } + unsigned noff = out->nodes_.size(); + unsigned eoff = out->edges_.size(); + int ogoal = in.nodes_.size() - 1; + int cgoal = noff - 1; + // keep a single goal node, so add nodes.size - 1 + out->nodes_.resize(out->nodes_.size() + ogoal); + // add all edges + out->edges_.resize(out->edges_.size() + in.edges_.size()); + + for (int i = 0; i < ogoal; ++i) { + const Hypergraph::Node& on = in.nodes_[i]; + Hypergraph::Node& cn = out->nodes_[i + noff]; + cn.id_ = i + noff; + cn.in_edges_.resize(on.in_edges_.size()); + for (unsigned j = 0; j < on.in_edges_.size(); ++j) + cn.in_edges_[j] = on.in_edges_[j] + eoff; + + cn.out_edges_.resize(on.out_edges_.size()); + for (unsigned j = 0; j < on.out_edges_.size(); ++j) + cn.out_edges_[j] = on.out_edges_[j] + eoff; + } + + for (unsigned i = 0; i < in.edges_.size(); ++i) { + const Hypergraph::Edge& oe = in.edges_[i]; + Hypergraph::Edge& ce = out->edges_[i + eoff]; + ce.id_ = i + eoff; + ce.rule_ = oe.rule_; + ce.feature_values_ = oe.feature_values_; + if (oe.head_node_ == ogoal) { + ce.head_node_ = cgoal; + out->nodes_[cgoal].in_edges_.push_back(ce.id_); + } else { + ce.head_node_ = oe.head_node_ + noff; + } + ce.tail_nodes_.resize(oe.tail_nodes_.size()); + for (unsigned j = 0; j < oe.tail_nodes_.size(); ++j) + ce.tail_nodes_[j] = oe.tail_nodes_[j] + noff; + } + + out->TopologicallySortNodesAndEdges(cgoal); +} + +} + diff --git a/decoder/hg_union.h b/decoder/hg_union.h new file mode 100644 index 00000000..34624246 --- /dev/null +++ b/decoder/hg_union.h @@ -0,0 +1,9 @@ +#ifndef _HG_UNION_H_ +#define _HG_UNION_H_ + +class Hypergraph; +namespace HG { + void Union(const Hypergraph& in, Hypergraph* out); +}; + +#endif diff --git a/extools/Makefile.am b/extools/Makefile.am deleted file mode 100644 index ee363264..00000000 --- a/extools/Makefile.am +++ /dev/null @@ -1,30 +0,0 @@ -bin_PROGRAMS = \ - extractor \ - mr_stripe_rule_reduce \ - filter_grammar \ - featurize_grammar \ - extractor_monolingual - -noinst_PROGRAMS = - -sg_lexer.cc: sg_lexer.l - $(LEX) -s -CF -8 -o$@ $< - -filter_grammar_SOURCES = filter_grammar.cc extract.cc sentence_pair.cc striped_grammar.cc sg_lexer.cc -filter_grammar_LDADD = $(top_srcdir)/utils/libutils.a -lz -#filter_grammar_LDFLAGS = -all-static - -featurize_grammar_SOURCES = featurize_grammar.cc extract.cc sentence_pair.cc sg_lexer.cc striped_grammar.cc -featurize_grammar_LDADD = $(top_srcdir)/utils/libutils.a -lz - -mr_stripe_rule_reduce_SOURCES = mr_stripe_rule_reduce.cc extract.cc sentence_pair.cc striped_grammar.cc sg_lexer.cc -mr_stripe_rule_reduce_LDADD = $(top_srcdir)/utils/libutils.a -lz - -extractor_SOURCES = sentence_pair.cc extract.cc extractor.cc striped_grammar.cc -extractor_LDADD = $(top_srcdir)/utils/libutils.a -lz - -extractor_monolingual_SOURCES = extractor_monolingual.cc -extractor_monolingual_LDADD = $(top_srcdir)/utils/libutils.a -lz - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils - diff --git a/extools/README b/extools/README deleted file mode 100644 index af91ce79..00000000 --- a/extools/README +++ /dev/null @@ -1,32 +0,0 @@ - -Categories have the format i-j:CAT where i and j are the indices of the spaces -between words in the TARGET language. For example, slash categories can be written: - - the blue house - 0-1:DT 1-2:JJ 2-3:NN 1-3:NBAR 0-2:NP/NN 0-3:NP - - -You may multiply label each span, e.g. - - NP - | - NBAR - | - NN - | - John - 0-1:NP 0-1:NBAR 0-1:NP - -However, this may result in a very large number of rules being extracted. - - -**** -* Filtering and Scoring of Unscored and Unfiltered Grammars -**** - -Take the unfiltered grammar, and a test set, and run: -./filter_grammar < unfiltered.grammar > filter.grammar - -Then, to score the new filtered grammar, run: -./score_grammar < filtered.grammar > scored.grammar - diff --git a/extools/coarsen_grammar.pl b/extools/coarsen_grammar.pl deleted file mode 100755 index f2dd6689..00000000 --- a/extools/coarsen_grammar.pl +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/perl - -# dumb grammar coarsener that maps every nonterminal to X (except S). - -use strict; - -unless (@ARGV > 1){ - die "Usage: $0 [ ... ] \n"; -} -my $weight_file = shift @ARGV; - -$ENV{"LC_ALL"} = "C"; -local(*GRAMMAR, *OUT_GRAMMAR, *WEIGHTS); - -my %weights; -unless (open(WEIGHTS, $weight_file)) {die "Could not open weight file $weight_file\n" } -while (){ - if (/(.+) (.+)$/){ - $weights{$1} = $2; - } -} -close(WEIGHTS); -unless (keys(%weights)){ - die "Could not find any PhraseModel features in weight file (perhaps you specified the wrong file?)\n\n". - "Usage: $0 [ ... ] \n"; -} - -sub cleanup_and_die; -$SIG{INT} = "cleanup_and_die"; -$SIG{TERM} = "cleanup_and_die"; -$SIG{HUP} = "cleanup_and_die"; - -open(OUT_GRAMMAR, ">grammar.tmp"); -while (my $grammar_file = shift @ARGV){ - unless (open(GRAMMAR, $grammar_file)) {die "Could not open grammar file $grammar_file\n"} - while (){ - if (/^((.*\|{3}){3})(.*)$/){ - my $rule = $1; - my $rest = $3; - my $coarse_rule = $rule; - $coarse_rule =~ s/\[X[^\],]*/[X/g; - print OUT_GRAMMAR "$coarse_rule $rule $rest\n"; - } else { - die "Unrecognized rule format: $_\n"; - } - } - close(GRAMMAR); -} -close(OUT_GRAMMAR); - -`sort grammar.tmp > grammar.tmp.sorted`; -sub dump_rules; -sub compute_score; -unless (open(GRAMMAR, "grammar.tmp.sorted")){ die "Something went wrong; could not open intermediate file grammar.tmp.sorted\n"}; -my $prev_coarse_rule = ""; -my $best_features = ""; -my $best_score = 0; -my @rules = (); -while (){ - if (/^\s*((\S.*\|{3}\s*){3})((\S.*\|{3}\s*){3})(.*)$/){ - my $coarse_rule = $1; - my $fine_rule = $3; - my $features = $5; # This code does not correctly handle rules with other info (e.g. alignments) - if ($coarse_rule eq $prev_coarse_rule){ - my $score = compute_score($features, %weights); - if ($score > $best_score){ - $best_score = $score; - $best_features = $features; - } - } else { - dump_rules($prev_coarse_rule, $best_features, @rules); - $prev_coarse_rule = $coarse_rule; - $best_features = $features; - $best_score = compute_score($features, %weights); - @rules = (); - } - push(@rules, "$fine_rule$features\n"); - } else { - die "Something went wrong during grammar projection: $_\n"; - } -} -dump_rules($prev_coarse_rule, $best_features, @rules); -close(GRAMMAR); -cleanup(); - -sub compute_score { - my($features, %weights) = @_; - my $score = 0; - if ($features =~ s/^\s*(\S.*\S)\s*$/$1/) { - my @features = split(/\s+/, $features); - my $pm=0; - for my $feature (@features) { - my $feature_name; - my $feature_val; - if ($feature =~ /(.*)=(.*)/){ - $feature_name = $1; - $feature_val= $2; - } else { - $feature_name = "PhraseModel_" . $pm; - $feature_val= $feature; - } - $pm++; - if ($weights{$feature_name}){ - $score += $weights{$feature_name} * $feature_val; - } - } - } else { - die "Unexpected feature value format: $features\n"; - } - return $score; -} - -sub dump_rules { - my($coarse_rule, $coarse_rule_scores, @fine_rules) = @_; - unless($coarse_rule){ return; } - print "$coarse_rule $coarse_rule_scores\n"; - for my $rule (@fine_rules){ - print "\t$rule"; - } -} - -sub cleanup_and_die { - cleanup(); - die "\n"; -} - -sub cleanup { - `rm -rf grammar.tmp grammar.tmp.sorted`; -} - - - - diff --git a/extools/extract.cc b/extools/extract.cc deleted file mode 100644 index 49542fed..00000000 --- a/extools/extract.cc +++ /dev/null @@ -1,336 +0,0 @@ -#include "extract.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "sentence_pair.h" -#include "tdict.h" -#include "wordid.h" -#include "array2d.h" - -using namespace std; -using namespace boost; -using std::tr1::unordered_map; -using boost::tuple; - -namespace { - inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; } - - inline void SkipWhitespace(const char* buf, int* ptr) { - while (buf[*ptr] && IsWhitespace(buf[*ptr])) { ++(*ptr); } - } -} - -Extract::RuleObserver::~RuleObserver() { - cerr << "Rules extracted: " << count << endl; -} - -void Extract::ExtractBasePhrases(const int max_base_phrase_size, - const AnnotatedParallelSentence& sentence, - vector* phrases) { - phrases->clear(); - - vector > f_spans(sentence.f_len, pair(sentence.e_len, 0)); - vector > e_spans(sentence.e_len, pair(sentence.f_len, 0)); - // for each alignment point in e, precompute the minimal consistent phrases in f - // for each alignment point in f, precompute the minimal consistent phrases in e - for (int i = 0; i < sentence.f_len; ++i) { - for (int j = 0; j < sentence.e_len; ++j) { - if (sentence.aligned(i,j)) { - if (j < f_spans[i].first) f_spans[i].first = j; - f_spans[i].second = j+1; - if (i < e_spans[j].first) e_spans[j].first = i; - e_spans[j].second = i+1; - } - } - } - - for (int i1 = 0; i1 < sentence.f_len; ++i1) { - if (sentence.f_aligned[i1] == 0) continue; - int j1 = sentence.e_len; - int j2 = 0; - const int i_limit = min(sentence.f_len, i1 + max_base_phrase_size); - for (int i2 = i1 + 1; i2 <= i_limit; ++i2) { - if (sentence.f_aligned[i2-1] == 0) continue; - // cerr << "F has aligned span " << i1 << " to " << i2 << endl; - j1 = min(j1, f_spans[i2-1].first); - j2 = max(j2, f_spans[i2-1].second); - if (j1 >= j2) continue; - if (j2 - j1 > max_base_phrase_size) continue; - int condition = 0; - for (int j = j1; j < j2; ++j) { - if (e_spans[j].first < i1) { condition = 1; break; } - if (e_spans[j].second > i2) { condition = 2; break; } - } - if (condition == 1) break; - if (condition == 2) continue; - // category types added later! - phrases->push_back(ParallelSpan(i1, i2, j1, j2)); - // cerr << i1 << " " << i2 << " : " << j1 << " " << j2 << endl; - } - } -} - -void Extract::LoosenPhraseBounds(const AnnotatedParallelSentence& sentence, - const int max_base_phrase_size, - vector* phrases) { - const int num_phrases = phrases->size(); - map > > > marker; - for (int i = 0; i < num_phrases; ++i) { - const ParallelSpan& cur = (*phrases)[i]; - marker[cur.i1][cur.i2][cur.j1][cur.j2] = true; - } - for (int i = 0; i < num_phrases; ++i) { - const ParallelSpan& cur = (*phrases)[i]; - const int i1_max = cur.i1; - const int i2_min = cur.i2; - const int j1_max = cur.j1; - const int j2_min = cur.j2; - int i1_min = i1_max; - while (i1_min > 0 && sentence.f_aligned[i1_min-1] == 0) { --i1_min; } - int j1_min = j1_max; - while (j1_min > 0 && sentence.e_aligned[j1_min-1] == 0) { --j1_min; } - int i2_max = i2_min; - while (i2_max < sentence.f_len && sentence.f_aligned[i2_max] == 0) { ++i2_max; } - int j2_max = j2_min; - while (j2_max < sentence.e_len && sentence.e_aligned[j2_max] == 0) { ++j2_max; } - for (int i1 = i1_min; i1 <= i1_max; ++i1) { - const int ilim = min(i2_max, i1 + max_base_phrase_size); - for (int i2 = max(i1+1,i2_min); i2 <= ilim; ++i2) { - for (int j1 = j1_min; j1 <= j1_max; ++j1) { - const int jlim = std::min(j2_max, j1 + max_base_phrase_size); - for (int j2 = std::max(j1+1, j2_min); j2 <= jlim; ++j2) { - bool& seen = marker[i1][i2][j1][j2]; - if (!seen) - phrases->push_back(ParallelSpan(i1,i2,j1,j2)); - seen = true; - } - } - } - } - } -} - -template -void -lookup_and_append(const map &dict, const K &key, V &output) -{ - typename map::const_iterator found = dict.find(key); - if (found != dict.end()) - copy(found->second.begin(), found->second.end(), back_inserter(output)); -} - -// this uses the TARGET span (i,j) to annotate phrases, will copy -// phrases if there is more than one annotation. -// TODO: support source annotation -void Extract::AnnotatePhrasesWithCategoryTypes(const WordID default_cat, - const map< boost::tuple, vector > &types, - vector* phrases) { - const int num_unannotated_phrases = phrases->size(); - // have to use num_unannotated_phrases since we may grow the vector - for (int i = 0; i < num_unannotated_phrases; ++i) { - ParallelSpan& phrase = (*phrases)[i]; - vector cats; - lookup_and_append(types, boost::make_tuple(phrase.i1, phrase.i2, phrase.j1, phrase.j2), cats); - lookup_and_append(types, boost::make_tuple((short)-1, (short)-1, phrase.j1, phrase.j2), cats); - lookup_and_append(types, boost::make_tuple(phrase.i1, phrase.i2, (short)-1, (short)-1), cats); - if (cats.empty() && default_cat != 0) { - cats = vector(1, default_cat); - } - if (cats.empty()) { - cerr << "ERROR span " << phrase.i1 << "," << phrase.i2 << "-" - << phrase.j1 << "," << phrase.j2 << " has no type. " - "Did you forget --default_category?\n"; - } - phrase.cat = cats[0]; - for (int ci = 1; ci < cats.size(); ++ci) { - ParallelSpan new_phrase = phrase; - new_phrase.cat = cats[ci]; - phrases->push_back(new_phrase); - } - } -} - -// a partially complete (f-side) of a rule -struct RuleItem { - vector f; - int i,j,syms,vars; - explicit RuleItem(int pi) : i(pi), j(pi), syms(), vars() {} - void Extend(const WordID& fword) { - f.push_back(ParallelSpan(fword)); - ++j; - ++syms; - } - void Extend(const ParallelSpan& subphrase) { - f.push_back(subphrase); - j += subphrase.i2 - subphrase.i1; - ++vars; - ++syms; - } - bool RuleFEndsInVariable() const { - if (f.size() > 0) { - return f.back().IsVariable(); - } else { return false; } - } -}; - -void Extract::ExtractConsistentRules(const AnnotatedParallelSentence& sentence, - const vector& phrases, - const int max_vars, - const int max_syms, - const bool permit_adjacent_nonterminals, - const bool require_aligned_terminal, - RuleObserver* observer, - vector* all_cats) { - const char bkoff_mrkr = '_'; - queue q; // agenda for BFS - int max_len = -1; - unordered_map, vector, boost::hash > > fspans; - vector > spans_by_start(sentence.f_len); - set starts; - WordID bkoff; - for (int i = 0; i < phrases.size(); ++i) { - fspans[make_pair(phrases[i].i1,phrases[i].i2)].push_back(phrases[i]); - max_len = max(max_len, phrases[i].i2 - phrases[i].i1); - // have we already added a rule item starting at phrases[i].i1? - if (starts.insert(phrases[i].i1).second) - q.push(RuleItem(phrases[i].i1)); - spans_by_start[phrases[i].i1].push_back(phrases[i]); - } - starts.clear(); - vector > next_e(sentence.e_len); - vector cur_rhs_f, cur_rhs_e; - vector > cur_terminal_align; - vector cur_es, cur_fs; - while(!q.empty()) { - const RuleItem& rule = q.front(); - - // extend the partial rule - if (rule.j < sentence.f_len && (rule.j - rule.i) < max_len && rule.syms < max_syms) { - RuleItem ew = rule; - - // extend with a word - ew.Extend(sentence.f[ew.j]); - q.push(ew); - - // with variables - if (rule.vars < max_vars && - !spans_by_start[rule.j].empty() && - ((!rule.RuleFEndsInVariable()) || permit_adjacent_nonterminals)) { - const vector& sub_phrases = spans_by_start[rule.j]; - for (int it = 0; it < sub_phrases.size(); ++it) { - if (sub_phrases[it].i2 - sub_phrases[it].i1 + rule.j - rule.i <= max_len) { - RuleItem ev = rule; - ev.Extend(sub_phrases[it]); - q.push(ev); - assert(ev.j <= sentence.f_len); - } - } - } - } - // determine if rule is consistent - if (rule.syms > 0 && - fspans.count(make_pair(rule.i,rule.j)) && - (!rule.RuleFEndsInVariable() || rule.syms > 1)) { - const vector& orig_spans = fspans[make_pair(rule.i,rule.j)]; - for (int s = 0; s < orig_spans.size(); ++s) { - const ParallelSpan& orig_span = orig_spans[s]; - const WordID lhs = orig_span.cat; - for (int j = orig_span.j1; j < orig_span.j2; ++j) next_e[j].first = -1; - int nt_index_e = 0; - for (int i = 0; i < rule.f.size(); ++i) { - const ParallelSpan& cur = rule.f[i]; - if (cur.IsVariable()) - next_e[cur.j1] = pair(cur.j2, ++nt_index_e); - } - cur_rhs_f.clear(); - cur_rhs_e.clear(); - cur_terminal_align.clear(); - cur_fs.clear(); - cur_es.clear(); - - const int elen = orig_span.j2 - orig_span.j1; - vector isvar(elen, 0); - int fbias = rule.i; - bool bad_rule = false; - bool has_aligned_terminal = false; - for (int i = 0; i < rule.f.size(); ++i) { - const ParallelSpan& cur = rule.f[i]; - cur_rhs_f.push_back(cur.cat); - if (cur.cat > 0) { // terminal - if (sentence.f_aligned[fbias + i]) has_aligned_terminal = true; - cur_fs.push_back(fbias + i); - } else { // non-terminal - int subj1 = cur.j1 - orig_span.j1; - int subj2 = cur.j2 - orig_span.j1; - if (subj1 < 0 || subj2 > elen) { bad_rule = true; break; } - for (int j = subj1; j < subj2 && !bad_rule; ++j) { - int& isvarj = isvar[j]; - isvarj = true; - } - if (bad_rule) break; - cur_fs.push_back(-1); - fbias += cur.i2 - cur.i1 - 1; - } - } - if (require_aligned_terminal && !has_aligned_terminal) bad_rule = true; - if (!bad_rule) { - for (int j = orig_span.j1; j < orig_span.j2; ++j) { - if (next_e[j].first < 0) { - cur_rhs_e.push_back(sentence.e[j]); - cur_es.push_back(j); - } else { - cur_rhs_e.push_back(1 - next_e[j].second); // next_e[j].second is NT gap index - cur_es.push_back(-1); - j = next_e[j].first - 1; - } - } - for (short i = 0; i < cur_fs.size(); ++i) - if (cur_fs[i] >= 0) - for (short j = 0; j < cur_es.size(); ++j) - if (cur_es[j] >= 0 && sentence.aligned(cur_fs[i],cur_es[j])) - cur_terminal_align.push_back(make_pair(i,j)); - //observer->CountRule(lhs, cur_rhs_f, cur_rhs_e, cur_terminal_align); - - if(!all_cats->empty()) { - //produce the backoff grammar if the category wordIDs are available - for (int i = 0; i < cur_rhs_f.size(); ++i) { - if(cur_rhs_f[i] < 0) { - //cerr << cur_rhs_f[i] << ": (cats,f) |" << TD::Convert(-cur_rhs_f[i]) << endl; - string nonterm = TD::Convert(-cur_rhs_f[i]); - nonterm+=bkoff_mrkr; - bkoff = -TD::Convert(nonterm); - cur_rhs_f[i]=bkoff; - /*vector rhs_f_bkoff; - vector rhs_e_bkoff; - vector > bkoff_align; - bkoff_align.clear(); - bkoff_align.push_back(make_pair(0,0)); - - for (int cat = 0; cat < all_cats->size(); ++cat) { - rhs_f_bkoff.clear(); - rhs_e_bkoff.clear(); - rhs_f_bkoff.push_back(-(*all_cats)[cat]); - rhs_e_bkoff.push_back(0); - observer->CountRule(bkoff,rhs_f_bkoff,rhs_e_bkoff,bkoff_align); - - }*/ - } - } - - } - observer->CountRule(lhs, cur_rhs_f, cur_rhs_e, cur_terminal_align); - } - } - } - q.pop(); - } -} - diff --git a/extools/extract.h b/extools/extract.h deleted file mode 100644 index e9ea5e65..00000000 --- a/extools/extract.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef _EXTRACT_H_ -#define _EXTRACT_H_ - -#include -#include -#include -#include -#include "array2d.h" -#include "wordid.h" -#include "sparse_vector.h" - -struct AnnotatedParallelSentence; - -// usually represents a consistent phrase, which may -// be annotated with a type (cat) -// inside the rule extractor, this class is also used to represent a word -// in a partial rule. -struct ParallelSpan { - // i1 = i of f side - // i2 = j of f side - // j1 = i of e side - // j2 = j of e side - short i1,i2,j1,j2; - // cat is set by AnnotatePhrasesWithCategoryTypes, otherwise it's 0 - WordID cat; // category type of span (also overloaded by RuleItem class - // to be a word ID) - ParallelSpan() : i1(-1), i2(-1), j1(-1), j2(-1), cat() {} - // used by Rule class to represent a terminal symbol: - explicit ParallelSpan(WordID w) : i1(-1), i2(-1), j1(-1), j2(-1), cat(w) {} - ParallelSpan(int pi1, int pi2, int pj1, int pj2) : i1(pi1), i2(pi2), j1(pj1), j2(pj2), cat() {} - ParallelSpan(int pi1, int pi2, int pj1, int pj2, WordID c) : i1(pi1), i2(pi2), j1(pj1), j2(pj2), cat(c) {} - - // ParallelSpan is used in the Rule class where it is - // overloaded to also represent terminal symbols - inline bool IsVariable() const { return i1 != -1; } -}; - -// rule extraction logic lives here. this has no data, it's just got -// static member functions. -struct Extract { - // RuleObserver's CountRule is called for each rule extracted - // implement CountRuleImpl to do things like count the rules, - // write them to a file, etc. - struct RuleObserver { - RuleObserver() : count() {} - virtual void CountRule(WordID lhs, - const std::vector& rhs_f, - const std::vector& rhs_e, - const std::vector >& fe_terminal_alignments) { - ++count; - CountRuleImpl(lhs, rhs_f, rhs_e, fe_terminal_alignments); - } - virtual ~RuleObserver(); - - protected: - virtual void CountRuleImpl(WordID lhs, - const std::vector& rhs_f, - const std::vector& rhs_e, - const std::vector >& fe_terminal_alignments) = 0; - private: - int count; - }; - - // given a set of "tight" phrases and the aligned sentence they were - // extracted from, "loosen" them - static void LoosenPhraseBounds(const AnnotatedParallelSentence& sentence, - const int max_base_phrase_size, - std::vector* phrases); - - // extract all consistent phrase pairs, up to size max_base_phrase_size - // (on the source side). these phrases will be "tight". - static void ExtractBasePhrases(const int max_base_phrase_size, - const AnnotatedParallelSentence& sentence, - std::vector* phrases); - - // this uses the TARGET span (i,j) to annotate phrases, will copy - // phrases if there is more than one annotation. - static void AnnotatePhrasesWithCategoryTypes(const WordID default_cat, - const std::map< boost::tuple, std::vector > &types, - std::vector* phrases); - - // use the Chiang (2007) extraction logic to extract consistent subphrases - // observer->CountRule is called once for each rule extracted - static void ExtractConsistentRules(const AnnotatedParallelSentence& sentence, - const std::vector& phrases, - const int max_vars, - const int max_syms, - const bool permit_adjacent_nonterminals, - const bool require_aligned_terminal, - RuleObserver* observer, - std::vector* all_cats); -}; - -#endif diff --git a/extools/extractor.cc b/extools/extractor.cc deleted file mode 100644 index 1e4154ef..00000000 --- a/extools/extractor.cc +++ /dev/null @@ -1,439 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "sparse_vector.h" -#include "sentence_pair.h" -#include "extract.h" -#include "tdict.h" -#include "fdict.h" -#include "wordid.h" -#include "array2d.h" -#include "filelib.h" -#include "striped_grammar.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static const size_t MAX_LINE_LENGTH = 100000; -WordID kBOS, kEOS, kDIVIDER, kGAP, kSPLIT; -int kCOUNT; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("input,i", po::value()->default_value("-"), "Input file") - ("default_category,d", po::value(), "Default span type (use X for 'Hiero')") - ("x_cdyer_pos,x", "Extract monolingual POS contexts (cdyer experimental)") - ("loose", "Use loose phrase extraction heuristic for base phrases") - ("base_phrase,B", "Write base phrases") - ("base_phrase_spans", "Write base sentences and phrase spans") - ("phrase_language", po::value()->default_value("target"), "Extract phrase strings in source, target or both languages") - ("context_language", po::value()->default_value("target"), "Extract context strings in source, target or both languages") - ("bidir,b", "Extract bidirectional rules (for computing p(f|e) in addition to p(e|f))") - ("combiner_size,c", po::value()->default_value(800000), "Number of unique items to store in cache before writing rule counts. Set to 1 to disable cache. Set to 0 for no limit.") - ("silent", "Write nothing to stderr except errors") - ("phrase_context,C", "Write base phrase contexts") - ("phrase_context_size,S", po::value()->default_value(2), "Use this many words of context on left and write when writing base phrase contexts") - ("max_base_phrase_size,L", po::value()->default_value(10), "Maximum starting phrase size") - ("max_syms,l", po::value()->default_value(5), "Maximum number of symbols in final phrase size") - ("max_vars,v", po::value()->default_value(2), "Maximum number of nonterminal variables in final phrase size") - ("permit_adjacent_nonterminals,A", "Permit adjacent nonterminals in source side of rules") - ("no_required_aligned_terminal,n", "Do not require an aligned terminal") - ("topics,t", po::value()->default_value(50), "Number of categories assigned during clustering") - ("backoff,g","Produce a backoff grammar") - ("help,h", "Print this help message and exit"); - po::options_description clo("Command line options"); - po::options_description dcmdline_options; - dcmdline_options.add(opts); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - po::notify(*conf); - - if (conf->count("help") || conf->count("input") == 0) { - cerr << "\nUsage: extractor [-options]\n"; - cerr << dcmdline_options << endl; - exit(1); - } -} - -// TODO how to handle alignment information? -void WriteBasePhrases(const AnnotatedParallelSentence& sentence, - const vector& phrases) { - vector e,f; - for (int it = 0; it < phrases.size(); ++it) { - const ParallelSpan& phrase = phrases[it]; - e.clear(); - f.clear(); - for (int i = phrase.i1; i < phrase.i2; ++i) - f.push_back(sentence.f[i]); - for (int j = phrase.j1; j < phrase.j2; ++j) - e.push_back(sentence.e[j]); - cout << TD::GetString(f) << " ||| " << TD::GetString(e) << endl; - } -} - -void WriteBasePhraseSpans(const AnnotatedParallelSentence& sentence, - const vector& phrases) { - cout << TD::GetString(sentence.f) << " ||| " << TD::GetString(sentence.e) << " |||"; - for (int it = 0; it < phrases.size(); ++it) { - const ParallelSpan& phrase = phrases[it]; - cout << " " << phrase.i1 << "-" << phrase.i2 - << "-" << phrase.j1 << "-" << phrase.j2; - } - cout << endl; -} - -struct CountCombiner { - CountCombiner(const size_t& csize) : combiner_size(csize) { - if (csize == 0) { cerr << "Using unlimited combiner cache.\n"; } - } - ~CountCombiner() { - if (!cache.empty()) WriteAndClearCache(); - } - - void Count(const vector& key, - const vector& val, - const int count_type, - const vector >& aligns) { - if (combiner_size != 1) { - RuleStatistics& v = cache[key][val]; - float newcount = v.counts.add_value(count_type, 1.0f); - // hack for adding alignments - if (newcount < 7.0f && aligns.size() > v.aligns.size()) - v.aligns = aligns; - if (combiner_size > 1 && cache.size() > combiner_size) - WriteAndClearCache(); - } else { - cout << TD::GetString(key) << '\t' << TD::GetString(val) << " ||| "; - cout << RuleStatistics(count_type, 1.0f, aligns) << endl; - } - } - - private: - void WriteAndClearCache() { - for (unordered_map, Vec2PhraseCount, boost::hash > >::iterator it = cache.begin(); - it != cache.end(); ++it) { - cout << TD::GetString(it->first) << '\t'; - const Vec2PhraseCount& vals = it->second; - bool needdiv = false; - for (Vec2PhraseCount::const_iterator vi = vals.begin(); vi != vals.end(); ++vi) { - if (needdiv) cout << " ||| "; else needdiv = true; - cout << TD::GetString(vi->first) << " ||| " << vi->second; - } - cout << endl; - } - cache.clear(); - } - - const size_t combiner_size; - typedef unordered_map, RuleStatistics, boost::hash > > Vec2PhraseCount; - unordered_map, Vec2PhraseCount, boost::hash > > cache; -}; - -// TODO optional source context -// output : k = phrase "document" v = context "term" -void WritePhraseContexts(const AnnotatedParallelSentence& sentence, - const vector& phrases, - const int ctx_size, - bool phrase_s, bool phrase_t, - bool context_s, bool context_t, - CountCombiner* o) { - vector context, context_f; - if (context_t) - { - context.resize(ctx_size * 2 + 1); - context[ctx_size] = kGAP; - } - if (context_s) - { - context_f.resize(ctx_size * 2 + 1); - context_f[ctx_size] = kGAP; - } - vector key, key_f; - if (phrase_t) key.reserve(100); - if (phrase_s) key_f.reserve(100); - - for (int it = 0; it < phrases.size(); ++it) { - const ParallelSpan& phrase = phrases[it]; - - key.clear(); - for (int j = phrase.j1; j < phrase.j2 && phrase_t; ++j) - key.push_back(sentence.e[j]); - - if (context_t) - { - context.resize(ctx_size * 2 + 1); - for (int i = 0; i < ctx_size && context_t; ++i) { - int epos = phrase.j1 - 1 - i; - const WordID left_ctx = (epos < 0) ? kBOS : sentence.e[epos]; - context[ctx_size - i - 1] = left_ctx; - epos = phrase.j2 + i; - const WordID right_ctx = (epos >= sentence.e_len) ? kEOS : sentence.e[epos]; - context[ctx_size + i + 1] = right_ctx; - } - } - else - context.clear(); - - if (phrase_s) - { - key_f.clear(); - for (int i = phrase.i1; i < phrase.i2; ++i) - key_f.push_back(sentence.f[i]); - if (phrase_t) key.push_back(kSPLIT); - copy(key_f.begin(), key_f.end(), back_inserter(key)); - } - - if (context_s) - { - for (int i = 0; i < ctx_size; ++i) { - int fpos = phrase.i1 - 1 - i; - const WordID left_ctx = (fpos < 0) ? kBOS : sentence.f[fpos]; - context_f[ctx_size - i - 1] = left_ctx; - fpos = phrase.i2 + i; - const WordID right_ctx = (fpos >= sentence.f_len) ? kEOS : sentence.f[fpos]; - context_f[ctx_size + i + 1] = right_ctx; - } - if (context_t) context.push_back(kSPLIT); - copy(context_f.begin(), context_f.end(), back_inserter(context)); - } - - o->Count(key, context, kCOUNT, vector >()); - } -} - -struct SimpleRuleWriter : public Extract::RuleObserver { - protected: - virtual void CountRuleImpl(WordID lhs, - const vector& rhs_f, - const vector& rhs_e, - const vector >& fe_terminal_alignments) { - cout << "[" << TD::Convert(-lhs) << "] |||"; - for (int i = 0; i < rhs_f.size(); ++i) { - if (rhs_f[i] < 0) cout << " [" << TD::Convert(-rhs_f[i]) << ']'; - else cout << ' ' << TD::Convert(rhs_f[i]); - } - cout << " |||"; - for (int i = 0; i < rhs_e.size(); ++i) { - if (rhs_e[i] <= 0) cout << " [" << (1-rhs_e[i]) << ']'; - else cout << ' ' << TD::Convert(rhs_e[i]); - } - cout << " |||"; - for (int i = 0; i < fe_terminal_alignments.size(); ++i) { - cout << ' ' << fe_terminal_alignments[i].first << '-' << fe_terminal_alignments[i].second; - } - cout << endl; - } -}; - -struct HadoopStreamingRuleObserver : public Extract::RuleObserver { - HadoopStreamingRuleObserver(CountCombiner* cc, bool bidir_flag) : - bidir(bidir_flag), - kF(TD::Convert("F")), - kE(TD::Convert("E")), - kDIVIDER(TD::Convert("|||")), - kLB("["), kRB("]"), - combiner(*cc), - kEMPTY(), - kCFE(FD::Convert("CFE")) { - for (int i=1; i < 50; ++i) - index2sym[1-i] = TD::Convert(kLB + boost::lexical_cast(i) + kRB); - fmajor_key.resize(10, kF); - emajor_key.resize(10, kE); - if (bidir) - fmajor_key[2] = emajor_key[2] = kDIVIDER; - else - fmajor_key[1] = kDIVIDER; - } - - protected: - virtual void CountRuleImpl(WordID lhs, - const vector& rhs_f, - const vector& rhs_e, - const vector >& fe_terminal_alignments) { - if (bidir) { // extract rules in "both directions" E->F and F->E - fmajor_key.resize(3 + rhs_f.size()); - emajor_key.resize(3 + rhs_e.size()); - fmajor_val.resize(rhs_e.size()); - emajor_val.resize(rhs_f.size()); - emajor_key[1] = fmajor_key[1] = MapSym(lhs); - int nt = 1; - for (int i = 0; i < rhs_f.size(); ++i) { - const WordID id = rhs_f[i]; - if (id < 0) { - fmajor_key[3 + i] = MapSym(id, nt); - emajor_val[i] = MapSym(id, nt); - ++nt; - } else { - fmajor_key[3 + i] = id; - emajor_val[i] = id; - } - } - for (int i = 0; i < rhs_e.size(); ++i) { - WordID id = rhs_e[i]; - if (id <= 0) { - fmajor_val[i] = index2sym[id]; - emajor_key[3 + i] = index2sym[id]; - } else { - fmajor_val[i] = id; - emajor_key[3 + i] = id; - } - } - combiner.Count(fmajor_key, fmajor_val, kCFE, fe_terminal_alignments); - combiner.Count(emajor_key, emajor_val, kCFE, kEMPTY); - } else { // extract rules only in F->E - fmajor_key.resize(2 + rhs_f.size()); - fmajor_val.resize(rhs_e.size()); - fmajor_key[0] = MapSym(lhs); - int nt = 1; - for (int i = 0; i < rhs_f.size(); ++i) { - const WordID id = rhs_f[i]; - if (id < 0) - fmajor_key[2 + i] = MapSym(id, nt++); - else - fmajor_key[2 + i] = id; - } - for (int i = 0; i < rhs_e.size(); ++i) { - const WordID id = rhs_e[i]; - if (id <= 0) - fmajor_val[i] = index2sym[id]; - else - fmajor_val[i] = id; - } - combiner.Count(fmajor_key, fmajor_val, kCFE, fe_terminal_alignments); - } - } - - private: - WordID MapSym(WordID sym, int ind = 0) { - WordID& r = cat2ind2sym[sym][ind]; - if (!r) { - if (ind == 0) - r = TD::Convert(kLB + TD::Convert(-sym) + kRB); - else - r = TD::Convert(kLB + TD::Convert(-sym) + "," + boost::lexical_cast(ind) + kRB); - } - return r; - } - - const bool bidir; - const WordID kF, kE, kDIVIDER; - const string kLB, kRB; - CountCombiner& combiner; - const vector > kEMPTY; - const int kCFE; - map > cat2ind2sym; - map index2sym; - vector emajor_key, emajor_val, fmajor_key, fmajor_val; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kBOS = TD::Convert(""); - kEOS = TD::Convert(""); - kDIVIDER = TD::Convert("|||"); - kGAP = TD::Convert(""); - kCOUNT = FD::Convert("C"); - kSPLIT = TD::Convert(""); - - WordID default_cat = 0; // 0 means no default- extraction will - // fail if a phrase is extracted without a - // category - const bool backoff = (conf.count("backoff") ? true : false); - if (conf.count("default_category")) { - string sdefault_cat = conf["default_category"].as(); - default_cat = -TD::Convert(sdefault_cat); - cerr << "Default category: " << sdefault_cat << endl; - } - ReadFile rf(conf["input"].as()); - istream& in = *rf.stream(); - - char buf[MAX_LINE_LENGTH]; - AnnotatedParallelSentence sentence; - vector phrases; - vector all_cats; - int max_base_phrase_size = conf["max_base_phrase_size"].as(); - bool write_phrase_contexts = conf.count("phrase_context") > 0; - const bool write_base_phrases = conf.count("base_phrase") > 0; - const bool write_base_phrase_spans = conf.count("base_phrase_spans") > 0; - const bool loose_phrases = conf.count("loose") > 0; - const bool silent = conf.count("silent") > 0; - const int max_syms = conf["max_syms"].as(); - const int max_vars = conf["max_vars"].as(); - const int ctx_size = conf["phrase_context_size"].as(); - const int num_categories = conf["topics"].as(); - const bool permit_adjacent_nonterminals = conf.count("permit_adjacent_nonterminals") > 0; - const bool require_aligned_terminal = conf.count("no_required_aligned_terminal") == 0; - const string ps = conf["phrase_language"].as(); - const bool phrase_s = ps == "source" || ps == "both"; - const bool phrase_t = ps == "target" || ps == "both"; - const string cs = conf["context_language"].as(); - const bool context_s = cs == "source" || cs == "both"; - const bool context_t = cs == "target" || cs == "both"; - const bool x_cdyer_pos = conf.count("x_cdyer_pos"); - int line = 0; - CountCombiner cc(conf["combiner_size"].as()); - HadoopStreamingRuleObserver o(&cc, - conf.count("bidir") > 0); - - assert(phrase_s || phrase_t); - assert(context_s || context_t); - - if(backoff) { - for (int i=0;i < num_categories;++i) - all_cats.push_back(TD::Convert("X"+boost::lexical_cast(i))); - } - - //SimpleRuleWriter o; - while(in) { - ++line; - in.getline(buf, MAX_LINE_LENGTH); - if (buf[0] == 0) continue; - //cerr << "line #" << line << " = " << buf << endl; - if (!silent) { - if (line % 200 == 0) cerr << '.'; - if (line % 8000 == 0) cerr << " [" << line << "]\n" << flush; - } - sentence.ParseInputLine(buf); - if (x_cdyer_pos) { - sentence.e = sentence.f; - sentence.AllocateForAlignment(); - for (int i = 0; i < sentence.e.size(); ++i) sentence.Align(i,i); - max_base_phrase_size = 1; - write_phrase_contexts = true; - } - phrases.clear(); - Extract::ExtractBasePhrases(max_base_phrase_size, sentence, &phrases); - if (loose_phrases) - Extract::LoosenPhraseBounds(sentence, max_base_phrase_size, &phrases); - if (phrases.empty()) { - cerr << "WARNING no phrases extracted line: " << line << endl; - continue; - } - if (write_phrase_contexts) { - WritePhraseContexts(sentence, phrases, ctx_size, phrase_s, phrase_t, context_s, context_t, &cc); - continue; - } - if (write_base_phrases) { - WriteBasePhrases(sentence, phrases); - continue; - } - if (write_base_phrase_spans) { - WriteBasePhraseSpans(sentence, phrases); - continue; - } - Extract::AnnotatePhrasesWithCategoryTypes(default_cat, sentence.span_types, &phrases); - Extract::ExtractConsistentRules(sentence, phrases, max_vars, max_syms, permit_adjacent_nonterminals, require_aligned_terminal, &o, &all_cats); - } - if (!silent) cerr << endl; - return 0; -} diff --git a/extools/extractor_monolingual.cc b/extools/extractor_monolingual.cc deleted file mode 100644 index 049ebc85..00000000 --- a/extools/extractor_monolingual.cc +++ /dev/null @@ -1,256 +0,0 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "tdict.h" -#include "fdict.h" -#include "wordid.h" -#include "filelib.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static const size_t MAX_LINE_LENGTH = 100000; -WordID kBOS, kEOS, kDIVIDER, kGAP; -int kCOUNT; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("input,i", po::value()->default_value("-"), "Input file") - ("phrases,p", po::value(), "File contatining phrases of interest") - ("phrase_context_size,S", po::value()->default_value(2), "Use this many words of context on left and write when writing base phrase contexts") - ("combiner_size,c", po::value()->default_value(30000), "Number of unique items to store in cache before writing rule counts. Set to 1 to disable cache. Set to 0 for no limit.") - ("prune", po::value()->default_value(0), "Prune items with count less than threshold; applies each time the cache is dumped.") - ("silent", "Write nothing to stderr except errors") - ("help,h", "Print this help message and exit"); - po::options_description clo("Command line options"); - po::options_description dcmdline_options; - dcmdline_options.add(opts); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - po::notify(*conf); - - if (conf->count("help") || conf->count("input") != 1 || conf->count("phrases") != 1) { - cerr << "\nUsage: extractor_monolingual [-options]\n"; - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct TrieNode -{ - TrieNode(int l) : finish(false), length(l) {}; - ~TrieNode() - { - for (unordered_map::iterator - it = next.begin(); it != next.end(); ++it) - delete it->second; - next.clear(); - } - - TrieNode *follow(int token) - { - unordered_map::iterator - found = next.find(token); - if (found != next.end()) - return found->second; - else - return 0; - } - - void insert(const vector &tokens) - { - insert(tokens.begin(), tokens.end()); - } - - void insert(vector::const_iterator begin, vector::const_iterator end) - { - if (begin == end) - finish = true; - else - { - int token = *begin; - unordered_map::iterator - nit = next.find(token); - if (nit == next.end()) - nit = next.insert(make_pair(token, new TrieNode(length+1))).first; - ++begin; - nit->second->insert(begin, end); - } - } - - bool finish; - int length; - unordered_map next; -}; - -struct CountCombiner { - CountCombiner(const size_t& csize, const size_t& prune) : combiner_size(csize), threshold(prune) { - if (csize == 0) { cerr << "Using unlimited combiner cache.\n"; } - } - ~CountCombiner() { - if (!cache.empty()) WriteAndClearCache(); - } - - void Count(const vector& key, - const vector& val, - const int count_type) - { - if (combiner_size != 1) { - cache[key][val] += count_type; - if (combiner_size > 1 && cache.size() > combiner_size) - WriteAndClearCache(); - } else { - cout << TD::GetString(key) << '\t' << TD::GetString(val) << " ||| C=" << count_type << "\n"; - } - } - - private: - void WriteAndClearCache() { - for (unordered_map, Vec2PhraseCount, boost::hash > >::iterator it = cache.begin(); - it != cache.end(); ++it) { - const Vec2PhraseCount& vals = it->second; - bool first = true; - for (Vec2PhraseCount::const_iterator vi = vals.begin(); vi != vals.end(); ++vi) - { - if (threshold > 1 && combiner_size != 1 && vi->second < threshold) - continue; - - if (!first) cout << " ||| "; - else - { - cout << TD::GetString(it->first) << '\t'; - first = false; - } - cout << TD::GetString(vi->first) << " ||| C=" << vi->second; - } - if (!first) - cout << '\n'; - } - cout << flush; - cache.clear(); - } - - const size_t combiner_size, threshold; - typedef unordered_map, int, boost::hash > > Vec2PhraseCount; - unordered_map, Vec2PhraseCount, boost::hash > > cache; -}; - -void WriteContext(const vector& sentence, int start, int end, int ctx_size, CountCombiner &combiner) -{ - vector phrase, context; - for (int i = start; i < end; ++i) - phrase.push_back(sentence[i]); - - for (int i = ctx_size; i > 0; --i) - context.push_back(sentence[start-i]); - context.push_back(kGAP); - for (int i = 0; i < ctx_size; ++i) - context.push_back(sentence[end+i]); - - combiner.Count(phrase, context, 1); -} - -inline bool IsWhitespace(char c) { - return c == ' ' || c == '\t'; -} - -inline void SkipWhitespace(const char* buf, int* ptr) { - while (buf[*ptr] && IsWhitespace(buf[*ptr])) { ++(*ptr); } -} - -vector ReadSentence(const char *buf, int padding) -{ - int ptr = 0; - SkipWhitespace(buf, &ptr); - int start = ptr; - vector sentence; - for (int i = 0; i < padding; ++i) - sentence.push_back(kBOS); - - while (char c = buf[ptr]) - { - if (!IsWhitespace(c)) - ++ptr; - else { - sentence.push_back(TD::Convert(string(buf, start, ptr-start))); - SkipWhitespace(buf, &ptr); - start = ptr; - } - } - for (int i = 0; i < padding; ++i) - sentence.push_back(kEOS); - - return sentence; -} - -int main(int argc, char** argv) -{ - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kBOS = TD::Convert(""); - kEOS = TD::Convert(""); - kDIVIDER = TD::Convert("|||"); - kGAP = TD::Convert(""); - kCOUNT = FD::Convert("C"); - - bool silent = conf.count("silent") > 0; - const int ctx_size = conf["phrase_context_size"].as(); - CountCombiner cc(conf["combiner_size"].as(), conf["prune"].as()); - - char buf[MAX_LINE_LENGTH]; - TrieNode phrase_trie(0); - ReadFile rpf(conf["phrases"].as()); - istream& pin = *rpf.stream(); - while (pin) { - pin.getline(buf, MAX_LINE_LENGTH); - phrase_trie.insert(ReadSentence(buf, 0)); - } - - ReadFile rif(conf["input"].as()); - istream &iin = *rif.stream(); - int line = 0; - while (iin) { - ++line; - iin.getline(buf, MAX_LINE_LENGTH); - //cout << "line: " << line << " '" << buf << "'" << endl; - if (buf[0] == 0) continue; - if (!silent) { - if (line % 200 == 0) cerr << '.'; - if (line % 8000 == 0) cerr << " [" << line << "]\n" << flush; - } - - vector sentence = ReadSentence(buf, ctx_size); - //cout << "sentence: " << TD::GetString(sentence) << endl; - vector tries; - for (int i = ctx_size; i < (int)sentence.size() - ctx_size; ++i) - { - //cout << "i: " << i << " token: " << TD::Convert(sentence[i]) << " tries: " << tries.size() << endl; - vector tries_prime; - tries.push_back(&phrase_trie); - for (vector::iterator tit = tries.begin(); tit != tries.end(); ++tit) - { - TrieNode* next = (*tit)->follow(sentence[i]); - if (next != 0) - { - //cout << "\tfollowed edge: " << next->finish << endl; - if (next->finish) - WriteContext(sentence, i + 1 - next->length, i + 1, ctx_size, cc); - tries_prime.push_back(next); - } - } - swap(tries, tries_prime); - } - //cout << "/sentence" << endl; - } - if (!silent) cerr << endl; - return 0; -} diff --git a/extools/featurize_grammar.cc b/extools/featurize_grammar.cc deleted file mode 100644 index 78175202..00000000 --- a/extools/featurize_grammar.cc +++ /dev/null @@ -1,716 +0,0 @@ -/* - * Featurize a grammar in striped format - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lex_trans_tbl.h" -#include "sparse_vector.h" -#include "sentence_pair.h" -#include "extract.h" -#include "fdict.h" -#include "tdict.h" -#include "filelib.h" -#include "striped_grammar.h" - -#include -#include -#include -#include -#include - -using namespace std; -using namespace std::tr1; -using boost::shared_ptr; -namespace po = boost::program_options; - -static string aligned_corpus; -static const size_t MAX_LINE_LENGTH = 64000000; - -// Data structures for indexing and counting rules -//typedef boost::tuple< WordID, vector, vector > RuleTuple; -struct RuleTuple { - RuleTuple(const WordID& lhs, const vector& s, const vector& t) - : m_lhs(lhs), m_source(s), m_target(t) { - hash_value(); - m_dirty = false; - } - - size_t hash_value() const { -// if (m_dirty) { - size_t hash = 0; - boost::hash_combine(hash, m_lhs); - boost::hash_combine(hash, m_source); - boost::hash_combine(hash, m_target); -// } -// m_dirty = false; - return hash; - } - - bool operator==(RuleTuple const& b) const - { return m_lhs == b.m_lhs && m_source == b.m_source && m_target == b.m_target; } - - WordID& lhs() { m_dirty=true; return m_lhs; } - vector& source() { m_dirty=true; return m_source; } - vector& target() { m_dirty=true; return m_target; } - const WordID& lhs() const { return m_lhs; } - const vector& source() const { return m_source; } - const vector& target() const { return m_target; } - -// mutable size_t m_hash; -private: - WordID m_lhs; - vector m_source, m_target; - mutable bool m_dirty; -}; -std::size_t hash_value(RuleTuple const& b) { return b.hash_value(); } -bool operator<(RuleTuple const& l, RuleTuple const& r) { - if (l.lhs() < r.lhs()) return true; - else if (l.lhs() == r.lhs()) { - if (l.source() < r.source()) return true; - else if (l.source() == r.source()) { - if (l.target() < r.target()) return true; - } - } - return false; -} - -ostream& operator<<(ostream& o, RuleTuple const& r) { - o << "(" << r.lhs() << "-->" << "<"; - for (vector::const_iterator it=r.source().begin(); it!=r.source().end(); ++it) - o << TD::Convert(*it) << " "; - o << "|||"; - for (vector::const_iterator it=r.target().begin(); it!=r.target().end(); ++it) - o << " " << TD::Convert(*it); - o << ">)"; - return o; -} - -template -struct FreqCount { - //typedef unordered_map > Counts; - typedef map Counts; - Counts counts; - - int inc(const Key& r, int c=1) { - pair itb - = counts.insert(make_pair(r,c)); - if (!itb.second) - itb.first->second += c; - return itb.first->second; - } - - int inc_if_exists(const Key& r, int c=1) { - typename Counts::iterator it = counts.find(r); - if (it != counts.end()) - it->second += c; - return it->second; - } - - int count(const Key& r) const { - typename Counts::const_iterator it = counts.find(r); - if (it == counts.end()) return 0; - return it->second; - } - - int operator()(const Key& r) const { return count(r); } -}; -typedef FreqCount RuleFreqCount; - -class FeatureExtractor; -class FERegistry; -struct FEFactoryBase { - virtual ~FEFactoryBase() {} - virtual boost::shared_ptr Create() const = 0; -}; - - -class FERegistry { - friend class FEFactoryBase; - public: - FERegistry() {} - boost::shared_ptr Create(const std::string& ffname) const { - map >::const_iterator it = reg_.find(ffname); - boost::shared_ptr res; - if (it == reg_.end()) { - cerr << "I don't know how to create feature " << ffname << endl; - } else { - res = it->second->Create(); - } - return res; - } - void DisplayList(ostream* out) const { - bool first = true; - for (map >::const_iterator it = reg_.begin(); - it != reg_.end(); ++it) { - if (first) {first=false;} else {*out << ' ';} - *out << it->first; - } - } - - void Register(const std::string& ffname, FEFactoryBase* factory) { - if (reg_.find(ffname) != reg_.end()) { - cerr << "Duplicate registration of FeatureExtractor with name " << ffname << "!\n"; - exit(1); - } - reg_[ffname].reset(factory); - } - - private: - std::map > reg_; -}; - -template -class FEFactory : public FEFactoryBase { - boost::shared_ptr Create() const { - return boost::shared_ptr(new FE); - } -}; - -void InitCommandLine(const FERegistry& r, int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - ostringstream feats; - feats << "[multiple] Features to extract ("; - r.DisplayList(&feats); - feats << ")"; - opts.add_options() - ("filtered_grammar,g", po::value(), "Grammar to add features to") - ("list_features,L", "List extractable features") - ("feature,f", po::value >()->composing(), feats.str().c_str()) - ("aligned_corpus,c", po::value(), "Aligned corpus (single line format)") - ("help,h", "Print this help message and exit"); - po::options_description clo("Command line options"); - po::options_description dcmdline_options; - dcmdline_options.add(opts); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - po::notify(*conf); - - if (conf->count("help") || conf->count("aligned_corpus")==0 || conf->count("feature") == 0) { - cerr << "\nUsage: featurize_grammar -g FILTERED-GRAMMAR.gz -c ALIGNED_CORPUS.fr-en-al -f Feat1 -f Feat2 ... < UNFILTERED-GRAMMAR\n"; - cerr << dcmdline_options << endl; - exit(1); - } -} - -static const bool DEBUG = false; - -void LexTranslationTable::createTTable(const char* buf){ - AnnotatedParallelSentence sent; - sent.ParseInputLine(buf); - - //iterate over the alignment to compute aligned words - - for(int i =0;i (sent.f[i], sent.e[j])]; - ++total_foreign[sent.f[i]]; - ++total_english[sent.e[j]]; - } - } - if (DEBUG) cerr << endl; - } - if (DEBUG) cerr << endl; - - const WordID NULL_ = TD::Convert("NULL"); - //handle unaligned words - align them to null - for (int j =0; j < sent.e_len; j++) { - if (sent.e_aligned[j]) continue; - ++word_translation[pair (NULL_, sent.e[j])]; - ++total_foreign[NULL_]; - ++total_english[sent.e[j]]; - } - - for (int i =0; i < sent.f_len; i++) { - if (sent.f_aligned[i]) continue; - ++word_translation[pair (sent.f[i], NULL_)]; - ++total_english[NULL_]; - ++total_foreign[sent.f[i]]; - } -} - -inline float safenlog(float v) { - if (v == 1.0f) return 0.0f; - float res = -log(v); - if (res > 100.0f) res = 100.0f; - return res; -} - -static bool IsZero(float f) { return (f > 0.999 && f < 1.001); } - -struct FeatureExtractor { - // create any keys necessary - virtual void ObserveFilteredRule(const WordID /* lhs */, - const vector& /* src */, - const vector& /* trg */) {} - - // compute statistics over keys, the same lhs-src-trg tuple may be seen - // more than once - virtual void ObserveUnfilteredRule(const WordID /* lhs */, - const vector& /* src */, - const vector& /* trg */, - const RuleStatistics& /* info */) {} - - // compute features, a unique lhs-src-trg tuple will be seen exactly once - virtual void ExtractFeatures(const WordID lhs, - const vector& src, - const vector& trg, - const RuleStatistics& info, - SparseVector* result) const = 0; - - virtual ~FeatureExtractor() {} -}; - -struct LogRuleCount : public FeatureExtractor { - LogRuleCount() : - fid_(FD::Convert("LogRuleCount")), - sfid_(FD::Convert("SingletonRule")), - kCFE(FD::Convert("CFE")) {} - virtual void ExtractFeatures(const WordID lhs, - const vector& src, - const vector& trg, - const RuleStatistics& info, - SparseVector* result) const { - (void) lhs; (void) src; (void) trg; - //result->set_value(fid_, log(info.counts.get(kCFE))); - result->set_value(fid_, log(info.counts.get(kCFE))); - if (IsZero(info.counts.get(kCFE))) - result->set_value(sfid_, 1); - } - const int fid_; - const int sfid_; - const int kCFE; -}; - -struct RulePenalty : public FeatureExtractor { - RulePenalty() : fid_(FD::Convert("RulePenalty")) {} - virtual void ExtractFeatures(const WordID /*lhs*/, - const vector& /*src*/, - const vector& /*trg*/, - const RuleStatistics& /*info*/, - SparseVector* result) const - { result->set_value(fid_, 1); } - - const int fid_; -}; - -// The negative log of the condition rule probs -// ignoring the identities of the non-terminals. -// i.e. the prob Hiero would assign. -// Also extracts Labelled features. -struct XFeatures: public FeatureExtractor { - XFeatures() : - fid_xfe(FD::Convert("XFE")), - fid_xef(FD::Convert("XEF")), - fid_labelledfe(FD::Convert("LabelledFE")), - fid_labelledef(FD::Convert("LabelledEF")), - fid_xesingleton(FD::Convert("XE_Singleton")), - fid_xfsingleton(FD::Convert("XF_Singleton")), - kCFE(FD::Convert("CFE")) {} - virtual void ObserveFilteredRule(const WordID /*lhs*/, - const vector& src, - const vector& trg) { - RuleTuple r(-1, src, trg); - map_rule(r); - rule_counts.inc(r, 0); - source_counts.inc(r.source(), 0); - target_counts.inc(r.target(), 0); - } - - virtual void ObserveUnfilteredRule(const WordID /*lhs*/, - const vector& src, - const vector& trg, - const RuleStatistics& info) { - RuleTuple r(-1, src, trg); - map_rule(r); - const int count = info.counts.get(kCFE); - assert(count > 0); - rule_counts.inc_if_exists(r, count); - source_counts.inc_if_exists(r.source(), count); - target_counts.inc_if_exists(r.target(), count); - } - - virtual void ExtractFeatures(const WordID /*lhs*/, - const vector& src, - const vector& trg, - const RuleStatistics& info, - SparseVector* result) const { - RuleTuple r(-1, src, trg); - map_rule(r); - double l_r_freq = log(rule_counts(r)); - - const int t_c = target_counts(r.target()); - assert(t_c > 0); - result->set_value(fid_xfe, log(t_c) - l_r_freq); - result->set_value(fid_labelledfe, log(t_c) - log(info.counts.get(kCFE))); -// if (t_c == 1) -// result->set_value(fid_xesingleton, 1.0); - - const int s_c = source_counts(r.source()); - assert(s_c > 0); - result->set_value(fid_xef, log(s_c) - l_r_freq); - result->set_value(fid_labelledef, log(s_c) - log(info.counts.get(kCFE))); -// if (s_c == 1) -// result->set_value(fid_xfsingleton, 1.0); - } - - void map_rule(RuleTuple& r) const { - vector indexes; int i=0; - for (vector::iterator it = r.target().begin(); it != r.target().end(); ++it) { - if (*it <= 0) - indexes.push_back(*it); - } - for (vector::iterator it = r.source().begin(); it != r.source().end(); ++it) { - if (*it <= 0) - *it = indexes.at(i++); - } - } - - const int fid_xfe, fid_xef; - const int fid_labelledfe, fid_labelledef; - const int fid_xesingleton, fid_xfsingleton; - const int kCFE; - RuleFreqCount rule_counts; - FreqCount< vector > source_counts, target_counts; -}; - - -struct LabelledRuleConditionals: public FeatureExtractor { - LabelledRuleConditionals() : - fid_fe(FD::Convert("LabelledFE")), - fid_ef(FD::Convert("LabelledEF")), - kCFE(FD::Convert("CFE")) {} - virtual void ObserveFilteredRule(const WordID lhs, - const vector& src, - const vector& trg) { - RuleTuple r(lhs, src, trg); - rule_counts.inc(r, 0); - source_counts.inc(r.source(), 0); - - target_counts.inc(r.target(), 0); - } - - virtual void ObserveUnfilteredRule(const WordID lhs, - const vector& src, - const vector& trg, - const RuleStatistics& info) { - RuleTuple r(lhs, src, trg); - rule_counts.inc_if_exists(r, info.counts.get(kCFE)); - source_counts.inc_if_exists(r.source(), info.counts.get(kCFE)); - - target_counts.inc_if_exists(r.target(), info.counts.get(kCFE)); - } - - virtual void ExtractFeatures(const WordID lhs, - const vector& src, - const vector& trg, - const RuleStatistics& /*info*/, - SparseVector* result) const { - RuleTuple r(lhs, src, trg); - double l_r_freq = log(rule_counts(r)); - result->set_value(fid_fe, log(target_counts(r.target())) - l_r_freq); - result->set_value(fid_ef, log(source_counts(r.source())) - l_r_freq); - } - - const int fid_fe, fid_ef; - const int kCFE; - RuleFreqCount rule_counts; - FreqCount< vector > source_counts, target_counts; -}; - -struct LHSProb: public FeatureExtractor { - LHSProb() : fid_(FD::Convert("LHSProb")), kCFE(FD::Convert("CFE")), total_count(0) {} - - virtual void ObserveUnfilteredRule(const WordID lhs, - const vector& /*src*/, - const vector& /*trg*/, - const RuleStatistics& info) { - int count = info.counts.get(kCFE); - total_count += count; - lhs_counts.inc(lhs, count); - } - - virtual void ExtractFeatures(const WordID lhs, - const vector& /*src*/, - const vector& /*trg*/, - const RuleStatistics& /*info*/, - SparseVector* result) const { - double lhs_log_prob = log(total_count) - log(lhs_counts(lhs)); - result->set_value(fid_, lhs_log_prob); - } - - const int fid_; - const int kCFE; - int total_count; - FreqCount lhs_counts; -}; - -// Proper rule generative probability: p( s,t | lhs) -struct GenerativeProb: public FeatureExtractor { - GenerativeProb() : - fid_(FD::Convert("GenerativeProb")), - kCFE(FD::Convert("CFE")) {} - - virtual void ObserveUnfilteredRule(const WordID lhs, - const vector& /*src*/, - const vector& /*trg*/, - const RuleStatistics& info) - { lhs_counts.inc(lhs, info.counts.get(kCFE)); } - - virtual void ExtractFeatures(const WordID lhs, - const vector& /*src*/, - const vector& /*trg*/, - const RuleStatistics& info, - SparseVector* result) const { - double log_prob = log(lhs_counts(lhs)) - log(info.counts.get(kCFE)); - result->set_value(fid_, log_prob); - } - - const int fid_; - const int kCFE; - FreqCount lhs_counts; -}; - -// remove terminals from the rules before estimating the conditional prob -struct LabellingShape: public FeatureExtractor { - LabellingShape() : fid_(FD::Convert("LabellingShape")), kCFE(FD::Convert("CFE")) {} - - virtual void ObserveFilteredRule(const WordID /*lhs*/, - const vector& src, - const vector& trg) { - RuleTuple r(-1, src, trg); - map_rule(r); - rule_counts.inc(r, 0); - source_counts.inc(r.source(), 0); - } - - virtual void ObserveUnfilteredRule(const WordID /*lhs*/, - const vector& src, - const vector& trg, - const RuleStatistics& info) { - RuleTuple r(-1, src, trg); - map_rule(r); - rule_counts.inc_if_exists(r, info.counts.get(kCFE)); - source_counts.inc_if_exists(r.source(), info.counts.get(kCFE)); - } - - virtual void ExtractFeatures(const WordID /*lhs*/, - const vector& src, - const vector& trg, - const RuleStatistics& /*info*/, - SparseVector* result) const { - RuleTuple r(-1, src, trg); - map_rule(r); - double l_r_freq = log(rule_counts(r)); - result->set_value(fid_, log(source_counts(r.source())) - l_r_freq); - } - - // Replace all terminals with generic -1 - void map_rule(RuleTuple& r) const { - for (vector::iterator it = r.target().begin(); it != r.target().end(); ++it) - if (*it <= 0) *it = -1; - for (vector::iterator it = r.source().begin(); it != r.source().end(); ++it) - if (*it <= 0) *it = -1; - } - - const int fid_, kCFE; - RuleFreqCount rule_counts; - FreqCount< vector > source_counts; -}; - - -// this extracts the lexical translation prob features -// in BOTH directions. -struct LexProbExtractor : public FeatureExtractor { - LexProbExtractor() : - e2f_(FD::Convert("LexE2F")), f2e_(FD::Convert("LexF2E")) { - ReadFile rf(aligned_corpus); - //create lexical translation table - cerr << "Computing lexical translation probabilities from " << aligned_corpus << "..." << endl; - char* buf = new char[MAX_LINE_LENGTH]; - istream& alignment = *rf.stream(); - while(alignment) { - alignment.getline(buf, MAX_LINE_LENGTH); - if (buf[0] == 0) continue; - table.createTTable(buf); - } - delete[] buf; - } - - virtual void ExtractFeatures(const WordID /*lhs*/, - const vector& src, - const vector& trg, - const RuleStatistics& info, - SparseVector* result) const { - map > foreign_aligned; - map > english_aligned; - - //Loop over all the alignment points to compute lexical translation probability - const vector< pair >& al = info.aligns; - vector< pair >::const_iterator ita; - for (ita = al.begin(); ita != al.end(); ++ita) { - if (DEBUG) { - cerr << "\nA:" << ita->first << "," << ita->second << "::"; - cerr << TD::Convert(src[ita->first]) << "-" << TD::Convert(trg[ita->second]); - } - - //Lookup this alignment probability in the table - int temp = table.word_translation[pair (src[ita->first],trg[ita->second])]; - float f2e=0, e2f=0; - if ( table.total_foreign[src[ita->first]] != 0) - f2e = (float) temp / table.total_foreign[src[ita->first]]; - if ( table.total_english[trg[ita->second]] !=0 ) - e2f = (float) temp / table.total_english[trg[ita->second]]; - if (DEBUG) printf (" %d %E %E\n", temp, f2e, e2f); - - //local counts to keep track of which things haven't been aligned, to later compute their null alignment - if (foreign_aligned.count(src[ita->first])) { - foreign_aligned[ src[ita->first] ].first++; - foreign_aligned[ src[ita->first] ].second += e2f; - } else { - foreign_aligned[ src[ita->first] ] = pair (1,e2f); - } - - if (english_aligned.count( trg[ ita->second] )) { - english_aligned[ trg[ ita->second] ].first++; - english_aligned[ trg[ ita->second] ].second += f2e; - } else { - english_aligned[ trg[ ita->second] ] = pair (1,f2e); - } - } - - float final_lex_f2e=1, final_lex_e2f=1; - static const WordID NULL_ = TD::Convert("NULL"); - - //compute lexical weight P(F|E) and include unaligned foreign words - for(int i=0;i temp_lex_prob = foreign_aligned[src[i]]; - final_lex_e2f *= temp_lex_prob.second / temp_lex_prob.first; - } - else //dealing with null alignment - { - int temp_count = table.word_translation[pair (src[i],NULL_)]; - float temp_e2f = (float) temp_count / table.total_english[NULL_]; - final_lex_e2f *= temp_e2f; - } - - } - - //compute P(E|F) unaligned english words - for(int j=0; j< trg.size(); j++) { - if (!table.total_english.count(trg[j])) continue; - - if (english_aligned.count(trg[j])) - { - pair temp_lex_prob = english_aligned[trg[j]]; - final_lex_f2e *= temp_lex_prob.second / temp_lex_prob.first; - } - else //dealing with null - { - int temp_count = table.word_translation[pair (NULL_,trg[j])]; - float temp_f2e = (float) temp_count / table.total_foreign[NULL_]; - final_lex_f2e *= temp_f2e; - } - } - result->set_value(e2f_, safenlog(final_lex_e2f)); - result->set_value(f2e_, safenlog(final_lex_f2e)); - } - const int e2f_, f2e_; - mutable LexTranslationTable table; -}; - -struct Featurizer { - Featurizer(const vector >& ex) : extractors(ex) { - } - void Callback1(WordID lhs, const vector& src, const ID2RuleStatistics& trgs) { - for (ID2RuleStatistics::const_iterator it = trgs.begin(); it != trgs.end(); ++it) { - for (int i = 0; i < extractors.size(); ++i) - extractors[i]->ObserveFilteredRule(lhs, src, it->first); - } - } - void Callback2(WordID lhs, const vector& src, const ID2RuleStatistics& trgs) { - for (ID2RuleStatistics::const_iterator it = trgs.begin(); it != trgs.end(); ++it) { - for (int i = 0; i < extractors.size(); ++i) - extractors[i]->ObserveUnfilteredRule(lhs, src, it->first, it->second); - } - } - void Callback3(WordID lhs, const vector& src, const ID2RuleStatistics& trgs) { - for (ID2RuleStatistics::const_iterator it = trgs.begin(); it != trgs.end(); ++it) { - SparseVector feats; - for (int i = 0; i < extractors.size(); ++i) - extractors[i]->ExtractFeatures(lhs, src, it->first, it->second, &feats); - cout << '[' << TD::Convert(-lhs) << "] ||| "; - WriteNamed(src, &cout); - cout << " ||| "; - WriteAnonymous(it->first, &cout); - cout << " ||| "; - print(cout,feats,"="); - cout << endl; - } - } - private: - vector > extractors; -}; - -void cb1(WordID lhs, const vector& src_rhs, const ID2RuleStatistics& rules, void* extra) { - static_cast(extra)->Callback1(lhs, src_rhs, rules); -} - -void cb2(WordID lhs, const vector& src_rhs, const ID2RuleStatistics& rules, void* extra) { - static_cast(extra)->Callback2(lhs, src_rhs, rules); -} - -void cb3(WordID lhs, const vector& src_rhs, const ID2RuleStatistics& rules, void* extra) { - static_cast(extra)->Callback3(lhs, src_rhs, rules); -} - -int main(int argc, char** argv){ - FERegistry reg; - reg.Register("LogRuleCount", new FEFactory); - reg.Register("LexProb", new FEFactory); - reg.Register("XFeatures", new FEFactory); - reg.Register("LabelledRuleConditionals", new FEFactory); - reg.Register("RulePenalty", new FEFactory); - reg.Register("LHSProb", new FEFactory); - reg.Register("LabellingShape", new FEFactory); - reg.Register("GenerativeProb", new FEFactory); - po::variables_map conf; - InitCommandLine(reg, argc, argv, &conf); - aligned_corpus = conf["aligned_corpus"].as(); // GLOBAL VAR - ReadFile fg1(conf["filtered_grammar"].as()); - - vector feats = conf["feature"].as >(); - vector > extractors(feats.size()); - for (int i = 0; i < feats.size(); ++i) - extractors[i] = reg.Create(feats[i]); - Featurizer fizer(extractors); - - cerr << "Reading filtered grammar to detect keys..." << endl; - StripedGrammarLexer::ReadStripedGrammar(fg1.stream(), cb1, &fizer); - - cerr << "Reading unfiltered grammar..." << endl; - StripedGrammarLexer::ReadStripedGrammar(&cin, cb2, &fizer); - - ReadFile fg2(conf["filtered_grammar"].as()); - cerr << "Reading filtered grammar and adding features..." << endl; - StripedGrammarLexer::ReadStripedGrammar(fg2.stream(), cb3, &fizer); - - return 0; -} - diff --git a/extools/filter_grammar.cc b/extools/filter_grammar.cc deleted file mode 100644 index cafcc923..00000000 --- a/extools/filter_grammar.cc +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Filter a grammar in striped format - */ -#include -#include -#include -#include -#include -#include - -#include "suffix_tree.h" -#include "sparse_vector.h" -#include "sentence_pair.h" -#include "extract.h" -#include "fdict.h" -#include "tdict.h" -#include "filelib.h" -#include "striped_grammar.h" - -#include -#include -#include -#include - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static const size_t MAX_LINE_LENGTH = 64000000; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("test_set,t", po::value(), "Filter for this test set") - ("top_e_given_f,n", po::value()->default_value(30), "Keep top N rules, according to p(e|f). 0 for all") - ("help,h", "Print this help message and exit"); - po::options_description clo("Command line options"); - po::options_description dcmdline_options; - dcmdline_options.add(opts); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - po::notify(*conf); - - if (conf->count("help") || conf->count("test_set")==0) { - cerr << "\nUsage: filter_grammar -t TEST-SET.fr [-options] < grammar\n"; - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct SourceFilter { - // return true to keep the rule, otherwise false - virtual bool Matches(const vector& key) const = 0; - virtual ~SourceFilter() {} -}; - -struct DumbSuffixTreeFilter : SourceFilter { - DumbSuffixTreeFilter(const string& corpus) { - cerr << "Build suffix tree from test set in " << corpus << endl; - assert(FileExists(corpus)); - ReadFile rfts(corpus); - istream& testSet = *rfts.stream(); - char* buf = new char[MAX_LINE_LENGTH]; - AnnotatedParallelSentence sent; - - /* process the data set to build suffix tree - */ - while(!testSet.eof()) { - testSet.getline(buf, MAX_LINE_LENGTH); - if (buf[0] == 0) continue; - - //hack to read in the test set using AnnotatedParallelSentence - strcat(buf," ||| fake ||| 0-0"); - sent.ParseInputLine(buf); - - //add each successive suffix to the tree - for(int i=0; i& src_rhs) const { - const Node* curnode = &root; - for(int i=0; i < src_rhs.size(); i++) { - if (src_rhs[i] <= 0) { - curnode = &root; - } else if (curnode) { - curnode = curnode->Extend(src_rhs[i]); - if (!curnode) return false; - } - } - return true; - } - Node root; -}; - -boost::shared_ptr filter; -multimap options; -int kCOUNT; -int max_options; - -void cb(WordID lhs, const vector& src_rhs, const ID2RuleStatistics& rules, void*) { - options.clear(); - if (!filter || filter->Matches(src_rhs)) { - for (ID2RuleStatistics::const_iterator it = rules.begin(); it != rules.end(); ++it) { - options.insert(make_pair(-it->second.counts.get(kCOUNT), it)); - } - int ocount = 0; - cout << '[' << TD::Convert(-lhs) << ']' << " ||| "; - WriteNamed(src_rhs, &cout); - cout << '\t'; - bool first = true; - for (multimap::iterator it = options.begin(); it != options.end(); ++it) { - if (first) { first = false; } else { cout << " ||| "; } - WriteAnonymous(it->second->first, &cout); - cout << " ||| " << it->second->second; - ++ocount; - if (ocount == max_options) break; - } - cout << endl; - } -} - -int main(int argc, char** argv){ - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - max_options = conf["top_e_given_f"].as();; - kCOUNT = FD::Convert("CFE"); - istream& unscored_grammar = cin; - cerr << "Loading test set " << conf["test_set"].as() << "...\n"; - filter.reset(new DumbSuffixTreeFilter(conf["test_set"].as())); - cerr << "Filtering...\n"; - StripedGrammarLexer::ReadStripedGrammar(&unscored_grammar, cb, NULL); -} - diff --git a/extools/lex_trans_tbl.h b/extools/lex_trans_tbl.h deleted file mode 100644 index 161b4a0d..00000000 --- a/extools/lex_trans_tbl.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - * lex_trans_tbl.h - * - * Created on: May 25, 2010 - * Author: Vlad - */ - -#ifndef LEX_TRANS_TBL_H_ -#define LEX_TRANS_TBL_H_ - -#include "wordid.h" -#include - -class LexTranslationTable -{ - public: - - std::map < std::pair,int > word_translation; - std::map total_foreign; - std::map total_english; - void createTTable(const char* buf); - -}; - -#endif /* LEX_TRANS_TBL_H_ */ diff --git a/extools/merge_lines.pl b/extools/merge_lines.pl deleted file mode 100755 index 8711e4ce..00000000 --- a/extools/merge_lines.pl +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -if (scalar @ARGV < 2) { - die "Usage: $0 file1.txt file2.txt ...\n\n Concatenate the nth line of each input file. All files\n must be the same length.\n\n"; -} - -my @fhs=(); -for my $file (@ARGV) { - my $fh; - open $fh, "<$file" or die "Can't read $file: $!\n"; - push @fhs, $fh; -} - -my $first = shift @fhs; - -while(my $x = <$first>) { - my $ind = 0; - chomp $x; - my @fields = ($x); - for my $fh (@fhs) { - $ind++; - $x = <$fh>; - die "ERROR: Mismatched number of lines: $ARGV[$ind]\n" unless $x; - chomp $x; - push @fields, $x; - } - print join ' ||| ', @fields; - print "\n"; -} -my $ind = 0; -for my $fh (@fhs) { - $ind++; - my $x=<$fh>; - die "ERROR: $ARGV[$ind] has extra lines!\n" if $x; -} - -exit 0; - -for my $fh (@fhs) { - close $fh; -} - diff --git a/extools/mr_stripe_rule_reduce.cc b/extools/mr_stripe_rule_reduce.cc deleted file mode 100644 index c9b2eb2a..00000000 --- a/extools/mr_stripe_rule_reduce.cc +++ /dev/null @@ -1,172 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "striped_grammar.h" -#include "tdict.h" -#include "sentence_pair.h" -#include "fdict.h" -#include "extract.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static const size_t MAX_LINE_LENGTH = 64000000; - -bool use_hadoop_counters = false; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("phrase_marginals,p", "Compute phrase marginals") - ("use_hadoop_counters,C", "Enable this if running inside Hadoop") - ("bidir,b", "Rules are tagged as being F->E or E->F, invert E rules in output") - ("help,h", "Print this help message and exit"); - po::options_description clo("Command line options"); - po::options_description dcmdline_options; - dcmdline_options.add(opts); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - po::notify(*conf); - - if (conf->count("help")) { - cerr << "\nUsage: mr_stripe_rule_reduce [-options]\n"; - cerr << dcmdline_options << endl; - exit(1); - } -} - -void PlusEquals(const ID2RuleStatistics& v, ID2RuleStatistics* self) { - for (ID2RuleStatistics::const_iterator it = v.begin(); it != v.end(); ++it) { - RuleStatistics& dest = (*self)[it->first]; - dest += it->second; - // TODO - do something smarter about alignments? - if (dest.aligns.empty() && !it->second.aligns.empty()) - dest.aligns = it->second.aligns; - } -} - -void WriteKeyValue(const vector& key, const ID2RuleStatistics& val) { - cout << TD::GetString(key) << '\t'; - bool needdiv = false; - for (ID2RuleStatistics::const_iterator it = val.begin(); it != val.end(); ++it) { - if (needdiv) cout << " ||| "; else needdiv = true; - cout << TD::GetString(it->first) << " ||| " << it->second; - } - cout << endl; - if (use_hadoop_counters) cerr << "reporter:counter:UserCounters,RuleCount," << val.size() << endl; -} - -void DoPhraseMarginals(const vector& key, const bool bidir, ID2RuleStatistics* val) { - static const WordID kF = TD::Convert("F"); - static const WordID kE = TD::Convert("E"); - static const int kCF = FD::Convert("CF"); - static const int kCE = FD::Convert("CE"); - static const int kCFE = FD::Convert("CFE"); - assert(key.size() > 0); - int cur_marginal_id = kCF; - if (bidir) { - if (key[0] != kF && key[0] != kE) { - cerr << "DoPhraseMarginals expects keys to have the from 'F|E [NT] word word word'\n"; - cerr << " but got: " << TD::GetString(key) << endl; - exit(1); - } - if (key[0] == kE) cur_marginal_id = kCE; - } - double tot = 0; - for (ID2RuleStatistics::iterator it = val->begin(); it != val->end(); ++it) - tot += it->second.counts.get(kCFE); - for (ID2RuleStatistics::iterator it = val->begin(); it != val->end(); ++it) { - it->second.counts.set_value(cur_marginal_id, tot); - - // prevent double counting of the joint - if (cur_marginal_id == kCE) it->second.counts.erase(kCFE); - } -} - -void WriteWithInversions(const vector& key, const ID2RuleStatistics& val) { - static const WordID kE = TD::Convert("E"); - static const WordID kDIV = TD::Convert("|||"); - vector new_key(key.size() - 1); - for (int i = 1; i < key.size(); ++i) - new_key[i - 1] = key[i]; - const bool do_invert = (key[0] == kE); - if (!do_invert) { - WriteKeyValue(new_key, val); - } else { - ID2RuleStatistics inv; - assert(new_key.size() > 2); - vector tk(new_key.size() - 2); - for (int i = 0; i < tk.size(); ++i) - tk[i] = new_key[2 + i]; - RuleStatistics& inv_stats = inv[tk]; - for (ID2RuleStatistics::const_iterator it = val.begin(); it != val.end(); ++it) { - inv_stats.counts = it->second.counts; - vector ekey(2 + it->first.size()); - ekey[0] = key[1]; - ekey[1] = kDIV; - for (int i = 0; i < it->first.size(); ++i) - ekey[2+i] = it->first[i]; - WriteKeyValue(ekey, inv); - } - } -} - -struct Reducer { - Reducer(bool phrase_marginals, bool bidir) : pm_(phrase_marginals), bidir_(bidir) {} - - void ProcessLine(const vector& key, const ID2RuleStatistics& rules) { - if (cur_key_ != key) { - if (cur_key_.size() > 0) Emit(); - acc_.clear(); - cur_key_ = key; - } - PlusEquals(rules, &acc_); - } - - ~Reducer() { - Emit(); - } - - void Emit() { - if (pm_) - DoPhraseMarginals(cur_key_, bidir_, &acc_); - if (bidir_) - WriteWithInversions(cur_key_, acc_); - else - WriteKeyValue(cur_key_, acc_); - } - - const bool pm_; - const bool bidir_; - vector cur_key_; - ID2RuleStatistics acc_; -}; - -void cb(const vector& key, const ID2RuleStatistics& contexts, void* red) { - static_cast(red)->ProcessLine(key, contexts); -} - - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - - char* buf = new char[MAX_LINE_LENGTH]; - vector key, cur_key; - int line = 0; - use_hadoop_counters = conf.count("use_hadoop_counters") > 0; - const bool phrase_marginals = conf.count("phrase_marginals") > 0; - const bool bidir = conf.count("bidir") > 0; - Reducer reducer(phrase_marginals, bidir); - StripedGrammarLexer::ReadContexts(&cin, cb, &reducer); - return 0; -} - diff --git a/extools/score_grammar.cc b/extools/score_grammar.cc deleted file mode 100644 index 0945e018..00000000 --- a/extools/score_grammar.cc +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Score a grammar in striped format - * ./score_grammar < filtered.grammar > scored.grammar - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sentence_pair.h" -#include "extract.h" -#include "fdict.h" -#include "tdict.h" -#include "lex_trans_tbl.h" -#include "filelib.h" - -#include -#include -#include - -using namespace std; -using namespace std::tr1; - - -static const size_t MAX_LINE_LENGTH = 64000000; - -typedef unordered_map, RuleStatistics, boost::hash > > ID2RuleStatistics; - - -namespace { - inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; } - inline bool IsBracket(char c){return c == '[' || c == ']';} - inline void SkipWhitespace(const char* buf, int* ptr) { - while (buf[*ptr] && IsWhitespace(buf[*ptr])) { ++(*ptr); } - } -} - -int ReadPhraseUntilDividerOrEnd(const char* buf, const int sstart, const int end, vector* p) { - static const WordID kDIV = TD::Convert("|||"); - int ptr = sstart; - while(ptr < end) { - while(ptr < end && IsWhitespace(buf[ptr])) { ++ptr; } - int start = ptr; - while(ptr < end && !IsWhitespace(buf[ptr])) { ++ptr; } - if (ptr == start) {cerr << "Warning! empty token.\n"; return ptr; } - const WordID w = TD::Convert(string(buf, start, ptr - start)); - - if((IsBracket(buf[start]) and IsBracket(buf[ptr-1])) or( w == kDIV)) - p->push_back(1 * w); - else { - if (w == kDIV) return ptr; - p->push_back(w); - } - } - return ptr; -} - - -void ParseLine(const char* buf, vector* cur_key, ID2RuleStatistics* counts) { - static const WordID kDIV = TD::Convert("|||"); - counts->clear(); - int ptr = 0; - while(buf[ptr] != 0 && buf[ptr] != '\t') { ++ptr; } - if (buf[ptr] != '\t') { - cerr << "Missing tab separator between key and value!\n INPUT=" << buf << endl; - exit(1); - } - cur_key->clear(); - // key is: "[X] ||| word word word" - int tmpp = ReadPhraseUntilDividerOrEnd(buf, 0, ptr, cur_key); - cur_key->push_back(kDIV); - ReadPhraseUntilDividerOrEnd(buf, tmpp, ptr, cur_key); - ++ptr; - int start = ptr; - int end = ptr; - int state = 0; // 0=reading label, 1=reading count - vector name; - while(buf[ptr] != 0) { - while(buf[ptr] != 0 && buf[ptr] != '|') { ++ptr; } - if (buf[ptr] == '|') { - ++ptr; - if (buf[ptr] == '|') { - ++ptr; - if (buf[ptr] == '|') { - ++ptr; - end = ptr - 3; - while (end > start && IsWhitespace(buf[end-1])) { --end; } - if (start == end) { - cerr << "Got empty token!\n LINE=" << buf << endl; - exit(1); - } - switch (state) { - case 0: ++state; name.clear(); ReadPhraseUntilDividerOrEnd(buf, start, end, &name); break; - case 1: --state; (*counts)[name].ParseRuleStatistics(buf, start, end); break; - default: cerr << "Can't happen\n"; abort(); - } - SkipWhitespace(buf, &ptr); - start = ptr; - } - } - } - } - end=ptr; - while (end > start && IsWhitespace(buf[end-1])) { --end; } - if (end > start) { - switch (state) { - case 0: ++state; name.clear(); ReadPhraseUntilDividerOrEnd(buf, start, end, &name); break; - case 1: --state; (*counts)[name].ParseRuleStatistics(buf, start, end); break; - default: cerr << "Can't happen\n"; abort(); - } - } -} - - - -void LexTranslationTable::createTTable(const char* buf){ - - bool DEBUG = false; - - AnnotatedParallelSentence sent; - - sent.ParseInputLine(buf); - - //iterate over the alignment to compute aligned words - - for(int i =0;i (sent.f[i], sent.e[j])]; - ++total_foreign[sent.f[i]]; - ++total_english[sent.e[j]]; - } - } - if (DEBUG) cerr << endl; - } - if (DEBUG) cerr << endl; - - static const WordID NULL_ = TD::Convert("NULL"); - //handle unaligned words - align them to null - for (int j =0; j < sent.e_len; j++) - { - if (sent.e_aligned[j]) continue; - ++word_translation[pair (NULL_, sent.e[j])]; - ++total_foreign[NULL_]; - ++total_english[sent.e[j]]; - } - - for (int i =0; i < sent.f_len; i++) - { - if (sent.f_aligned[i]) continue; - ++word_translation[pair (sent.f[i], NULL_)]; - ++total_english[NULL_]; - ++total_foreign[sent.f[i]]; - } - -} - - -inline float safenlog(float v) { - if (v == 1.0f) return 0.0f; - float res = -log(v); - if (res > 100.0f) res = 100.0f; - return res; -} - -int main(int argc, char** argv){ - bool DEBUG= false; - if (argc != 2) { - cerr << "Usage: " << argv[0] << " corpus.al < filtered.grammar\n"; - return 1; - } - ifstream alignment (argv[1]); - istream& unscored_grammar = cin; - ostream& scored_grammar = cout; - - //create lexical translation table - cerr << "Creating table..." << endl; - char* buf = new char[MAX_LINE_LENGTH]; - - LexTranslationTable table; - - while(!alignment.eof()) - { - alignment.getline(buf, MAX_LINE_LENGTH); - if (buf[0] == 0) continue; - - table.createTTable(buf); - } - - bool PRINT_TABLE=false; - if (PRINT_TABLE) - { - ofstream trans_table; - trans_table.open("lex_trans_table.out"); - for(map < pair,int >::iterator it = table.word_translation.begin(); it != table.word_translation.end(); ++it) - { - trans_table << TD::Convert(it->first.first) << "|||" << TD::Convert(it->first.second) << "==" << it->second << "//" << table.total_foreign[it->first.first] << "//" << table.total_english[it->first.second] << endl; - } - - trans_table.close(); - } - - - //score unscored grammar - cerr <<"Scoring grammar..." << endl; - - ID2RuleStatistics acc, cur_counts; - vector key, cur_key,temp_key; - vector< pair > al; - vector< pair >::iterator ita; - int line = 0; - - static const int kCF = FD::Convert("CF"); - static const int kCE = FD::Convert("CE"); - static const int kCFE = FD::Convert("CFE"); - - while(!unscored_grammar.eof()) - { - ++line; - unscored_grammar.getline(buf, MAX_LINE_LENGTH); - if (buf[0] == 0) continue; - ParseLine(buf, &cur_key, &cur_counts); - - //loop over all the Target side phrases that this source aligns to - for (ID2RuleStatistics::const_iterator it = cur_counts.begin(); it != cur_counts.end(); ++it) - { - - /*Compute phrase translation prob. - Print out scores in this format: - Phrase trnaslation prob P(F|E) - Phrase translation prob P(E|F) - Lexical weighting prob lex(F|E) - Lexical weighting prob lex(E|F) - */ - - float pEF_ = it->second.counts.value(kCFE) / it->second.counts.value(kCF); - float pFE_ = it->second.counts.value(kCFE) / it->second.counts.value(kCE); - - map > foreign_aligned; - map > english_aligned; - - //Loop over all the alignment points to compute lexical translation probability - al = it->second.aligns; - for(ita = al.begin(); ita != al.end(); ++ita) - { - - if (DEBUG) - { - cerr << "\nA:" << ita->first << "," << ita->second << "::"; - cerr << TD::Convert(cur_key[ita->first + 2]) << "-" << TD::Convert(it->first[ita->second]); - } - - - //Lookup this alignment probability in the table - int temp = table.word_translation[pair (cur_key[ita->first+2],it->first[ita->second])]; - float f2e=0, e2f=0; - if ( table.total_foreign[cur_key[ita->first+2]] != 0) - f2e = (float) temp / table.total_foreign[cur_key[ita->first+2]]; - if ( table.total_english[it->first[ita->second]] !=0 ) - e2f = (float) temp / table.total_english[it->first[ita->second]]; - if (DEBUG) printf (" %d %E %E\n", temp, f2e, e2f); - - - //local counts to keep track of which things haven't been aligned, to later compute their null alignment - if (foreign_aligned.count(cur_key[ita->first+2])) - { - foreign_aligned[ cur_key[ita->first+2] ].first++; - foreign_aligned[ cur_key[ita->first+2] ].second += e2f; - } - else - foreign_aligned [ cur_key[ita->first+2] ] = pair (1,e2f); - - - - if (english_aligned.count( it->first[ ita->second] )) - { - english_aligned[ it->first[ ita->second ]].first++; - english_aligned[ it->first[ ita->second] ].second += f2e; - } - else - english_aligned [ it->first[ ita->second] ] = pair (1,f2e); - - - - - } - - float final_lex_f2e=1, final_lex_e2f=1; - static const WordID NULL_ = TD::Convert("NULL"); - - //compute lexical weight P(F|E) and include unaligned foreign words - for(int i=0;i temp_lex_prob = foreign_aligned[cur_key[i]]; - final_lex_e2f *= temp_lex_prob.second / temp_lex_prob.first; - } - else //dealing with null alignment - { - int temp_count = table.word_translation[pair (cur_key[i],NULL_)]; - float temp_e2f = (float) temp_count / table.total_english[NULL_]; - final_lex_e2f *= temp_e2f; - } - - } - - //compute P(E|F) unaligned english words - for(int j=0; j< it->first.size(); j++) - { - if (!table.total_english.count(it->first[j])) continue; - - if (english_aligned.count(it->first[j])) - { - pair temp_lex_prob = english_aligned[it->first[j]]; - final_lex_f2e *= temp_lex_prob.second / temp_lex_prob.first; - } - else //dealing with null - { - int temp_count = table.word_translation[pair (NULL_,it->first[j])]; - float temp_f2e = (float) temp_count / table.total_foreign[NULL_]; - final_lex_f2e *= temp_f2e; - } - } - - - scored_grammar << TD::GetString(cur_key); - string lhs = TD::Convert(cur_key[0]); - scored_grammar << " " << TD::GetString(it->first) << " |||"; - if(lhs.find('_')!=string::npos) { - scored_grammar << " Bkoff=" << safenlog(3.0f); - } else { - scored_grammar << " FGivenE=" << safenlog(pFE_) << " EGivenF=" << safenlog(pEF_); - scored_grammar << " LexE2F=" << safenlog(final_lex_e2f) << " LexF2E=" << safenlog(final_lex_f2e); - } - scored_grammar << endl; - } - } -} - diff --git a/extools/sentence_pair.cc b/extools/sentence_pair.cc deleted file mode 100644 index 7d60715a..00000000 --- a/extools/sentence_pair.cc +++ /dev/null @@ -1,198 +0,0 @@ -#include "sentence_pair.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "tdict.h" -#include "wordid.h" -#include "array2d.h" - -using namespace std; -using namespace boost; - -namespace { - inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; } - - inline void SkipWhitespace(const char* buf, int* ptr) { - while (buf[*ptr] && IsWhitespace(buf[*ptr])) { ++(*ptr); } - } -} - -void AnnotatedParallelSentence::Reset() { - f.clear(); - e.clear(); - e_aligned.clear(); - f_aligned.clear(); - aligns_by_fword.clear(); - aligned.clear(); - span_types.clear(); -} - -void AnnotatedParallelSentence::AllocateForAlignment() { - f_len = f.size(); - e_len = e.size(); - aligned.resize(f_len, e_len, false); - f_aligned.resize(f_len, 0); - e_aligned.resize(e_len, 0); - aligns_by_fword.resize(f_len); -} - -// read an alignment point of the form X-Y where X and Y are strings -// of digits. if permit_col is true, the right edge will be determined -// by the presence of a colon -int AnnotatedParallelSentence::ReadAlignmentPoint(const char* buf, - const int start, - const int end, - const bool permit_col, - short* a, short* b, short* c, short* d) { - if (end - start < 3) { - cerr << "Alignment point badly formed 1: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - int ch = start; - *a = 0; - while(ch < end && buf[ch] != '-') { - if (buf[ch] < '0' || buf[ch] > '9') { - cerr << "Alignment point badly formed 2: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*a) *= 10; - (*a) += buf[ch] - '0'; - ++ch; - } - ++ch; - if (ch >= end) { - cerr << "Alignment point badly formed 3: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*b) = 0; - while((ch < end) && (c == 0 && (!permit_col || (permit_col && buf[ch] != ':')) || c != 0 && buf[ch] != '-')) { - if ((buf[ch] < '0') || (buf[ch] > '9')) { - cerr << "Alignment point badly formed 4: " << string(buf, start, end-start) << endl << buf << endl << buf[ch] << endl; - exit(1); - } - (*b) *= 10; - (*b) += buf[ch] - '0'; - ++ch; - } - if (c != 0) - { - ++ch; - if (ch >= end) { - cerr << "Alignment point badly formed 5: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*c) = 0; - while(ch < end && buf[ch] != '-') { - if (buf[ch] < '0' || buf[ch] > '9') { - cerr << "Alignment point badly formed 6: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*c) *= 10; - (*c) += buf[ch] - '0'; - ++ch; - } - ++ch; - if (ch >= end) { - cerr << "Alignment point badly formed 7: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*d) = 0; - while(ch < end && (!permit_col || (permit_col && buf[ch] != ':'))) { - if (buf[ch] < '0' || buf[ch] > '9') { - cerr << "Alignment point badly formed 8: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - (*d) *= 10; - (*d) += buf[ch] - '0'; - ++ch; - } - } - return ch; -} - -void AnnotatedParallelSentence::Align(const short a, const short b) { - aligned(a,b) = true; - ++f_aligned[a]; - ++e_aligned[b]; - aligns_by_fword[a].push_back(make_pair(a,b)); - // cerr << a << " " << b << endl; -} - -void AnnotatedParallelSentence::ParseAlignmentPoint(const char* buf, int start, int end) { - short a, b; - ReadAlignmentPoint(buf, start, end, false, &a, &b, 0, 0); - if (a >= f_len || b >= e_len) { - cerr << "(" << a << ',' << b << ") is out of bounds. INPUT=\n" << buf << endl; - exit(1); - } - Align(a,b); -} - -void AnnotatedParallelSentence::ParseSpanLabel(const char* buf, int start, int end) { - short a,b,c,d; - int ch = ReadAlignmentPoint(buf, start, end, true, &a, &b, &c, &d) + 1; - if (buf[ch-1] != ':' || ch >= end) { - cerr << "Span badly formed: " << string(buf, start, end-start) << endl << buf << endl; - exit(1); - } - if (a >= f_len || b > f_len) { - cerr << "(" << a << ',' << b << ") is out of bounds in labeled span. INPUT=\n" << buf << endl; - exit(1); - } - if (c >= e_len || d > e_len) { - cerr << "(" << c << ',' << d << ") is out of bounds in labeled span. INPUT=\n" << buf << endl; - exit(1); - } - // cerr << a << " " << b << " " << string(buf,c,end-c) << endl; - span_types[boost::make_tuple(a,b,c,d)].push_back(-TD::Convert(string(buf, ch, end-ch))); -} - -// INPUT FORMAT -// ein haus ||| a house ||| 0-0 1-1 ||| 0-0:DT 1-1:NN 0-1:NP -void AnnotatedParallelSentence::ParseInputLine(const char* buf) { - Reset(); - int ptr = 0; - SkipWhitespace(buf, &ptr); - int start = ptr; - int state = 0; // 0 = French, 1 = English, 2 = Alignment, 3 = Spans - while(char c = buf[ptr]) { - if (!IsWhitespace(c)) { ++ptr; continue; } else { - if (ptr - start == 3 && buf[start] == '|' && buf[start+1] == '|' && buf[start+2] == '|') { - ++state; - if (state == 4) { cerr << "Too many fields (ignoring):\n " << buf << endl; return; } - if (state == 2) { - // cerr << "FLEN=" << f->size() << " ELEN=" << e->size() << endl; - AllocateForAlignment(); - } - SkipWhitespace(buf, &ptr); - start = ptr; - continue; - } - switch (state) { - case 0: f.push_back(TD::Convert(string(buf, start, ptr-start))); break; - case 1: e.push_back(TD::Convert(string(buf, start, ptr-start))); break; - case 2: ParseAlignmentPoint(buf, start, ptr); break; - case 3: ParseSpanLabel(buf, start, ptr); break; - default: cerr << "Can't happen\n"; abort(); - } - SkipWhitespace(buf, &ptr); - start = ptr; - } - } - if (ptr > start) { - switch (state) { - case 0: f.push_back(TD::Convert(string(buf, start, ptr-start))); break; - case 1: e.push_back(TD::Convert(string(buf, start, ptr-start))); break; - case 2: ParseAlignmentPoint(buf, start, ptr); break; - case 3: ParseSpanLabel(buf, start, ptr); break; - default: cerr << "Can't happen\n"; abort(); - } - } -} - diff --git a/extools/sentence_pair.h b/extools/sentence_pair.h deleted file mode 100644 index a05275e7..00000000 --- a/extools/sentence_pair.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef _SENTENCE_PAIR_H_ -#define _SENTENCE_PAIR_H_ - -#include -#include -#include -#include -#include "wordid.h" -#include "array2d.h" - -// represents a parallel sentence with a word alignment and category -// annotations over subspans (currently in terms of f) -// you should read one using ParseInputLine and then use the public -// member variables to query things about it -struct AnnotatedParallelSentence { - // read annotated parallel sentence from string - void ParseInputLine(const char* buf); - - std::vector f, e; // words in f and e - - // word alignment information - std::vector e_aligned, f_aligned; // counts the number of times column/row x is aligned - Array2D aligned; - std::vector > > aligns_by_fword; - - // span type information - std::map< boost::tuple, std::vector > span_types; - // span_types(i,j,k,l) is the list of category span (i,j) in source and (k,l) in the target language. - - int f_len, e_len; - - void Align(const short a, const short b); - void AllocateForAlignment(); - - static int ReadAlignmentPoint(const char* buf, int start, int end, bool permit_col, short* a, short* b, short* c, short* d); - - private: - void Reset(); - void ParseAlignmentPoint(const char* buf, int start, int end); - void ParseSpanLabel(const char* buf, int start, int end); -}; - -#endif diff --git a/extools/sg_lexer.l b/extools/sg_lexer.l deleted file mode 100644 index c85cdea7..00000000 --- a/extools/sg_lexer.l +++ /dev/null @@ -1,294 +0,0 @@ -%{ -#include -#include -#include -#include -#include -#include "tdict.h" -#include "fdict.h" -#include "striped_grammar.h" - -int lex_line = 0; -int read_contexts = 0; -std::istream* sglex_stream = NULL; -StripedGrammarLexer::GrammarCallback grammar_callback = NULL; -StripedGrammarLexer::ContextCallback context_callback = NULL; -void* grammar_callback_extra = NULL; -void* context_callback_extra = NULL; - -#undef YY_INPUT -#define YY_INPUT(buf, result, max_size) (result = sglex_stream->read(buf, max_size).gcount()) - -#define YY_SKIP_YYWRAP 1 -int num_rules = 0; -int yywrap() { return 1; } -bool fl = true; -#define MAX_TOKEN_SIZE 255 -std::string sglex_tmp_token(MAX_TOKEN_SIZE, '\0'); - -#define MAX_RULE_SIZE 48 -WordID sglex_src_rhs[MAX_RULE_SIZE]; -WordID sglex_trg_rhs[MAX_RULE_SIZE]; -int sglex_src_rhs_size; -int sglex_trg_rhs_size; -WordID sglex_lhs; -int sglex_src_arity; -int sglex_trg_arity; - -#define MAX_FEATS 100 -int sglex_feat_ids[MAX_FEATS]; -double sglex_feat_vals[MAX_FEATS]; -int sglex_num_feats; - -#define MAX_ARITY 20 -int sglex_nt_sanity[MAX_ARITY]; -int sglex_src_nts[MAX_ARITY]; -float sglex_nt_size_means[MAX_ARITY]; -float sglex_nt_size_vars[MAX_ARITY]; - -std::vector cur_src_rhs; -std::vector cur_trg_rhs; -ID2RuleStatistics cur_options; -RuleStatistics* cur_stats = NULL; -int sglex_cur_fid = 0; - -static void sanity_check_trg_index(int index) { - if (index > sglex_src_arity) { - std::cerr << "Target index " << index << " exceeds source arity " << sglex_src_arity << std::endl; - abort(); - } - int& flag = sglex_nt_sanity[index - 1]; - if (flag) { - std::cerr << "Target index " << index << " used multiple times!" << std::endl; - abort(); - } - flag = 1; -} - -static void sglex_reset() { - sglex_src_arity = 0; - sglex_trg_arity = 0; - sglex_num_feats = 0; - sglex_src_rhs_size = 0; - sglex_trg_rhs_size = 0; -} - -%} - -REAL [\-+]?[0-9]+(\.[0-9]*([eE][-+]*[0-9]+)?)?|inf|[\-+]inf -NT [^\t \[\],]+ -ALIGN [0-9]+-[0-9]+ - -%x LHS_END SRC TRG FEATS FEATVAL ALIGNS -%% - -[ ] ; -[\t] { - if (read_contexts) { - cur_options.clear(); - BEGIN(TRG); - } else { - std::cerr << "Unexpected tab while reading striped grammar\n"; - exit(1); - } - } - -\[{NT}\] { - if (read_contexts) { - sglex_tmp_token.assign(yytext, yyleng); - sglex_src_rhs[sglex_src_rhs_size] = TD::Convert(sglex_tmp_token); - ++sglex_src_rhs_size; - } else { - sglex_tmp_token.assign(yytext + 1, yyleng - 2); - sglex_lhs = -TD::Convert(sglex_tmp_token); - // std::cerr << sglex_tmp_token << "\n"; - BEGIN(LHS_END); - } - } - -[^ \t]+ { - if (read_contexts) { - // std::cerr << "Context: " << yytext << std::endl; - sglex_tmp_token.assign(yytext, yyleng); - sglex_src_rhs[sglex_src_rhs_size] = TD::Convert(sglex_tmp_token); - ++sglex_src_rhs_size; - } else { - std::cerr << "Unexpected input: " << yytext << " when NT expected\n"; - exit(1); - } - } - -\[{NT}\] { - sglex_tmp_token.assign(yytext + 1, yyleng - 2); - sglex_src_nts[sglex_src_arity] = sglex_src_rhs[sglex_src_rhs_size] = -TD::Convert(sglex_tmp_token); - ++sglex_src_arity; - ++sglex_src_rhs_size; - } - -[ ] { ; } -\|\|\| { - sglex_reset(); - BEGIN(SRC); - } - -. { - std::cerr << "Line " << lex_line << ": unexpected input in LHS: " << yytext << std::endl; - exit(1); - } - - -\[{NT},[1-9][0-9]?\] { - int index = yytext[yyleng - 2] - '0'; - if (yytext[yyleng - 3] == ',') { - sglex_tmp_token.assign(yytext + 1, yyleng - 4); - } else { - sglex_tmp_token.assign(yytext + 1, yyleng - 5); - index += 10 * (yytext[yyleng - 3] - '0'); - } - if ((sglex_src_arity+1) != index) { - std::cerr << "Src indices must go in order: expected " << sglex_src_arity << " but got " << index << std::endl; - abort(); - } - sglex_src_nts[sglex_src_arity] = sglex_src_rhs[sglex_src_rhs_size] = -TD::Convert(sglex_tmp_token); - ++sglex_src_rhs_size; - ++sglex_src_arity; - } - -[^ \t]+ { - sglex_tmp_token.assign(yytext, yyleng); - sglex_src_rhs[sglex_src_rhs_size] = TD::Convert(sglex_tmp_token); - ++sglex_src_rhs_size; - } -[ ] { ; } -\t { - //std::cerr << "LHS=" << TD::Convert(-sglex_lhs) << " "; - //std::cerr << " src_size: " << sglex_src_rhs_size << std::endl; - //std::cerr << " src_arity: " << sglex_src_arity << std::endl; - cur_options.clear(); - memset(sglex_nt_sanity, 0, sglex_src_arity * sizeof(int)); - sglex_trg_rhs_size = 0; - BEGIN(TRG); - } - -\[[1-9][0-9]?\] { - if (read_contexts) { - sglex_tmp_token.assign(yytext, yyleng); - sglex_trg_rhs[sglex_trg_rhs_size] = TD::Convert(sglex_tmp_token); - ++sglex_trg_rhs_size; - } else { - int index = yytext[yyleng - 2] - '0'; - if (yyleng == 4) { - index += 10 * (yytext[yyleng - 3] - '0'); - } - ++sglex_trg_arity; - sanity_check_trg_index(index); - sglex_trg_rhs[sglex_trg_rhs_size] = 1 - index; - ++sglex_trg_rhs_size; - } -} - -\|\|\| { - //std::cerr << " trg_size: " << sglex_trg_rhs_size << std::endl; - //std::cerr << " trg_arity: " << sglex_trg_arity << std::endl; - assert(sglex_trg_rhs_size > 0); - cur_trg_rhs.resize(sglex_trg_rhs_size); - for (int i = 0; i < sglex_trg_rhs_size; ++i) - cur_trg_rhs[i] = sglex_trg_rhs[i]; - cur_stats = &cur_options[cur_trg_rhs]; - BEGIN(FEATS); - } - -[^ ]+ { - sglex_tmp_token.assign(yytext, yyleng); - sglex_trg_rhs[sglex_trg_rhs_size] = TD::Convert(sglex_tmp_token); - - ++sglex_trg_rhs_size; - } -[ ]+ { ; } - -\n { - assert(sglex_src_rhs_size > 0); - cur_src_rhs.resize(sglex_src_rhs_size); - for (int i = 0; i < sglex_src_rhs_size; ++i) - cur_src_rhs[i] = sglex_src_rhs[i]; - if (read_contexts) { - context_callback(cur_src_rhs, cur_options, context_callback_extra); - } else { - assert(sglex_lhs < 0); - grammar_callback(sglex_lhs, cur_src_rhs, cur_options, grammar_callback_extra); - } - cur_options.clear(); - sglex_reset(); - BEGIN(INITIAL); - } -[ ]+ { ; } -\|\|\| { - memset(sglex_nt_sanity, 0, sglex_src_arity * sizeof(int)); - sglex_trg_rhs_size = 0; - BEGIN(TRG); - } -[A-Z][A-Z_0-9]*= { - // std::cerr << "FV: " << yytext << std::endl; - sglex_tmp_token.assign(yytext, yyleng - 1); - sglex_cur_fid = FD::Convert(sglex_tmp_token); - static const int Afid = FD::Convert("A"); - if (sglex_cur_fid == Afid) { - BEGIN(ALIGNS); - } else { - BEGIN(FEATVAL); - } - } -{REAL} { - // std::cerr << "Feature val input: " << yytext << std::endl; - cur_stats->counts.add_value(sglex_cur_fid, strtod(yytext, NULL)); - BEGIN(FEATS); - } -. { - std::cerr << "Feature val unexpected input: " << yytext << std::endl; - exit(1); - } -. { - std::cerr << "Features unexpected input: " << yytext << std::endl; - exit(1); - } -{ALIGN}(,{ALIGN})* { - assert(cur_stats->aligns.empty()); - int i = 0; - while(i < yyleng) { - short a = 0; - short b = 0; - while (yytext[i] != '-') { a *= 10; a += yytext[i] - '0'; ++i; } - ++i; - while (yytext[i] != ',' && i < yyleng) { b *= 10; b += yytext[i] - '0'; ++i; } - ++i; - cur_stats->aligns.push_back(std::make_pair(a,b)); - } - BEGIN(FEATS); - } -. { - std::cerr << "Aligns unexpected input: " << yytext << std::endl; - exit(1); - } -%% - -#include "filelib.h" - -void StripedGrammarLexer::ReadStripedGrammar(std::istream* in, GrammarCallback func, void* extra) { - read_contexts = 0; - lex_line = 1; - sglex_stream = in; - grammar_callback_extra = extra; - grammar_callback = func; - yylex(); -} - -void StripedGrammarLexer::ReadContexts(std::istream* in, ContextCallback func, void* extra) { - read_contexts = 1; - lex_line = 1; - sglex_stream = in; - context_callback_extra = extra; - context_callback = func; - yylex(); -} - - diff --git a/extools/simple-extract-context.sh b/extools/simple-extract-context.sh deleted file mode 100755 index 17487b1c..00000000 --- a/extools/simple-extract-context.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -MYDIR=$(dirname $0) - -export LANG=C -date 1>&2 -$MYDIR/extractor -i $1 -c 500000 -L 12 -C | sort -t $'\t' -k 1 | $MYDIR/mr_stripe_rule_reduce -date 1>&2 - diff --git a/extools/simple-extract.sh b/extools/simple-extract.sh deleted file mode 100755 index ec5c5276..00000000 --- a/extools/simple-extract.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -export LANG=C -date -./extractor -i $1 -d X -c 500000 -L 12 -b | sort -t $'\t' -k 1 | gzip > ex.output.gz -date -# -p = compute phrase marginals -# -b = bidirectional rules (starting with F or E) were extracted -zcat ex.output.gz | ./mr_stripe_rule_reduce -p -b | sort -t $'\t' -k 1 | ./mr_stripe_rule_reduce | gzip > phrase-table.gz -date - diff --git a/extools/striped_grammar.cc b/extools/striped_grammar.cc deleted file mode 100644 index 785f4bbe..00000000 --- a/extools/striped_grammar.cc +++ /dev/null @@ -1,67 +0,0 @@ -#include "striped_grammar.h" - -#include - -#include "sentence_pair.h" - -using namespace std; - -namespace { - inline bool IsWhitespace(char c) { return c == ' ' || c == '\t'; } - - inline void SkipWhitespace(const char* buf, int* ptr) { - while (buf[*ptr] && IsWhitespace(buf[*ptr])) { ++(*ptr); } - } -} - -void RuleStatistics::ParseRuleStatistics(const char* buf, int start, int end) { - int ptr = start; - counts.clear(); - aligns.clear(); - while (ptr < end) { - SkipWhitespace(buf, &ptr); - int vstart = ptr; - while(ptr < end && buf[ptr] != '=') ++ptr; - assert(buf[ptr] == '='); - assert(ptr > vstart); - if (buf[vstart] == 'A' && buf[vstart+1] == '=') { - ++ptr; - while (ptr < end && !IsWhitespace(buf[ptr])) { - while(ptr < end && buf[ptr] == ',') { ++ptr; } - assert(ptr < end); - vstart = ptr; - while(ptr < end && buf[ptr] != ',' && !IsWhitespace(buf[ptr])) { ++ptr; } - if (ptr > vstart) { - short a, b; - AnnotatedParallelSentence::ReadAlignmentPoint(buf, vstart, ptr, false, &a, &b, 0, 0); - aligns.push_back(make_pair(a,b)); - } - } - } else { - int name = FD::Convert(string(buf,vstart,ptr-vstart)); - ++ptr; - vstart = ptr; - while(ptr < end && !IsWhitespace(buf[ptr])) { ++ptr; } - assert(ptr > vstart); - counts.set_value(name, strtod(buf + vstart, NULL)); - } - } -} - -ostream& operator<<(ostream& os, const RuleStatistics& s) { - bool needspace = false; - for (SparseVector::const_iterator it = s.counts.begin(); it != s.counts.end(); ++it) { - if (needspace) os << ' '; else needspace = true; - os << FD::Convert(it->first) << '=' << it->second; - } - if (s.aligns.size() > 0) { - os << " A="; - needspace = false; - for (int i = 0; i < s.aligns.size(); ++i) { - if (needspace) os << ','; else needspace = true; - os << s.aligns[i].first << '-' << s.aligns[i].second; - } - } - return os; -} - diff --git a/extools/striped_grammar.h b/extools/striped_grammar.h deleted file mode 100644 index bf3aec7d..00000000 --- a/extools/striped_grammar.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef _STRIPED_GRAMMAR_H_ -#define _STRIPED_GRAMMAR_H_ - -#include -#include -#include -#include -#include "sparse_vector.h" -#include "wordid.h" -#include "tdict.h" - -// represents statistics / information about a rule pair -struct RuleStatistics { - SparseVector counts; - std::vector > aligns; - RuleStatistics() {} - RuleStatistics(int name, float val, const std::vector >& al) : - aligns(al) { - counts.set_value(name, val); - } - void ParseRuleStatistics(const char* buf, int start, int end); - RuleStatistics& operator+=(const RuleStatistics& rhs) { - counts += rhs.counts; - return *this; - } -}; -std::ostream& operator<<(std::ostream& os, const RuleStatistics& s); - -inline void WriteNamed(const std::vector& v, std::ostream* os) { - bool first = true; - for (int i = 0; i < v.size(); ++i) { - if (first) { first = false; } else { (*os) << ' '; } - if (v[i] < 0) { (*os) << '[' << TD::Convert(-v[i]) << ']'; } - else (*os) << TD::Convert(v[i]); - } -} - -inline void WriteAnonymous(const std::vector& v, std::ostream* os) { - bool first = true; - for (int i = 0; i < v.size(); ++i) { - if (first) { first = false; } else { (*os) << ' '; } - if (v[i] <= 0) { (*os) << '[' << (1-v[i]) << ']'; } - else (*os) << TD::Convert(v[i]); - } -} - -typedef std::tr1::unordered_map, RuleStatistics, boost::hash > > ID2RuleStatistics; - -struct StripedGrammarLexer { - typedef void (*GrammarCallback)(WordID lhs, const std::vector& src_rhs, const ID2RuleStatistics& rules, void *extra); - static void ReadStripedGrammar(std::istream* in, GrammarCallback func, void* extra); - typedef void (*ContextCallback)(const std::vector& phrase, const ID2RuleStatistics& rules, void *extra); - static void ReadContexts(std::istream* in, ContextCallback func, void* extra); -}; - -#endif diff --git a/extools/suffix_tree.h b/extools/suffix_tree.h deleted file mode 100644 index f62f53f4..00000000 --- a/extools/suffix_tree.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * suffix_tree.h - * - * Created on: May 17, 2010 - * Author: Vlad - -NOTE (graehl): this seems to be a (forward) trie of the suffixes (of sentences). -so O(m*n^2) for m sentences of length n. - -For a real suffix tree (linear size/time), see: -http://en.wikipedia.org/wiki/Suffix_tree -http://www.cs.helsinki.fi/u/ukkonen/SuffixT1withFigs.pdf - - */ - -#ifndef SUFFIX_TREE_H_ -#define SUFFIX_TREE_H_ - -#include -#include -#include - -template -class Node { - public: - std::map edge_list_; - int InsertPath(const std::vector& p, int start, int end); - const Node* Extend(const T& e) const { - typename std::map::const_iterator it = edge_list_.find(e); - if (it == edge_list_.end()) return NULL; - return &it->second; - } -}; - -bool DEBUG = false; - -template -int Node::InsertPath(const std::vector& p, int start, int end){ - Node* currNode = this; - for(int i=start;i<= end; i++ ) { - currNode = &(currNode->edge_list_)[p[i]]; - } - return 1; -} - -#endif /* SUFFIX_TRIE_H_ */ diff --git a/extools/test_data/README b/extools/test_data/README deleted file mode 100644 index e368cffc..00000000 --- a/extools/test_data/README +++ /dev/null @@ -1,10 +0,0 @@ -The following was used to create the test data. The real inputs -were corpus.fr, corpus.en, and corpus.aligned. The generated files -were corpus.len_cats and fr-en.al.len. - - - ./make_len_cats.pl corpus.en > corpus.len_cats - - ../merge_lines.pl corpus.fr corpus.en corpus.aligned corpus.len_cats > fr-en.al.len - - diff --git a/extools/test_data/corpus.aligned b/extools/test_data/corpus.aligned deleted file mode 100644 index aa09e9ab..00000000 --- a/extools/test_data/corpus.aligned +++ /dev/null @@ -1,5 +0,0 @@ -0-0 1-2 2-1 -0-0 1-1 -0-0 0-1 1-0 1-1 2-0 2-1 3-2 4-3 -0-0 -0-0 1-1 diff --git a/extools/test_data/corpus.en b/extools/test_data/corpus.en deleted file mode 100644 index 2d4751bf..00000000 --- a/extools/test_data/corpus.en +++ /dev/null @@ -1,5 +0,0 @@ -the blue house -the hat -there is a hat -cap -the cat diff --git a/extools/test_data/corpus.fr b/extools/test_data/corpus.fr deleted file mode 100644 index 75b5e127..00000000 --- a/extools/test_data/corpus.fr +++ /dev/null @@ -1,5 +0,0 @@ -la maison bleue -le chapeau -il y a un chapeau -chapeau -le chat diff --git a/extools/test_data/corpus.len_cats b/extools/test_data/corpus.len_cats deleted file mode 100644 index 18d321de..00000000 --- a/extools/test_data/corpus.len_cats +++ /dev/null @@ -1,5 +0,0 @@ -0-1:SHORT 0-2:SHORT 0-3:MID 1-2:SHORT 1-3:SHORT 2-3:SHORT -0-1:SHORT 0-2:SHORT 1-2:SHORT -0-1:SHORT 0-2:SHORT 0-3:MID 0-4:MID 1-2:SHORT 1-3:SHORT 1-4:MID 2-3:SHORT 2-4:SHORT 3-4:SHORT -0-1:SHORT -0-1:SHORT 0-2:SHORT 1-2:SHORT diff --git a/extools/test_data/fr-en.al.len b/extools/test_data/fr-en.al.len deleted file mode 100644 index 7ee6b85d..00000000 --- a/extools/test_data/fr-en.al.len +++ /dev/null @@ -1,5 +0,0 @@ -la maison bleue ||| the blue house ||| 0-0 1-2 2-1 ||| 0-1:SHORT 0-2:SHORT 0-3:MID 1-2:SHORT 1-3:SHORT 2-3:SHORT -le chapeau ||| the hat ||| 0-0 1-1 ||| 0-1:SHORT 0-2:SHORT 1-2:SHORT -il y a un chapeau ||| there is a hat ||| 0-0 0-1 1-0 1-1 2-0 2-1 3-2 4-3 ||| 0-1:SHORT 0-2:SHORT 0-3:MID 0-4:MID 1-2:SHORT 1-3:SHORT 1-4:MID 2-3:SHORT 2-4:SHORT 3-4:SHORT -chapeau ||| cap ||| 0-0 ||| 0-1:SHORT -le chat ||| the cat ||| 0-0 1-1 ||| 0-1:SHORT 0-2:SHORT 1-2:SHORT diff --git a/extools/test_data/make_len_cats.pl b/extools/test_data/make_len_cats.pl deleted file mode 100755 index 25ef75fa..00000000 --- a/extools/test_data/make_len_cats.pl +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $max_len = 15; -my @cat_names = qw( NULL SHORT SHORT MID MID MID LONG LONG LONG LONG LONG VLONG VLONG VLONG VLONG VLONG ); - -while(<>) { - chomp; - my @words = split /\s+/; - my $len = scalar @words; - my @spans; - for (my $i =0; $i < $len; $i++) { - for (my $k = 1; $k <= $max_len; $k++) { - my $j = $i + $k; - next if ($j > $len); - my $cat = $cat_names[$k]; - die unless $cat; - push @spans, "$i-$j:$cat"; - } - } - print "@spans\n"; -} - -- cgit v1.2.3 From 3f5bd140a20aa039788c0973d98f709d5747494b Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Sun, 12 Aug 2012 23:36:44 -0400 Subject: redefine HG --- decoder/hg_intersect.cc | 6 +++++- decoder/hg_intersect.h | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'decoder') diff --git a/decoder/hg_intersect.cc b/decoder/hg_intersect.cc index e9a91061..ad5b701a 100644 --- a/decoder/hg_intersect.cc +++ b/decoder/hg_intersect.cc @@ -79,7 +79,9 @@ static bool FastLinearIntersect(const Lattice& target, Hypergraph* hg) { return (cov.size() == target.size()); } -bool HG::Intersect(const Lattice& target, Hypergraph* hg) { +namespace HG { + +bool Intersect(const Lattice& target, Hypergraph* hg) { // there are a number of faster algorithms available for restricted // classes of hypergraph and/or target. if (hg->IsLinearChain() && target.IsSentence()) @@ -160,3 +162,5 @@ bool HG::Intersect(const Lattice& target, Hypergraph* hg) { return true; } +} + diff --git a/decoder/hg_intersect.h b/decoder/hg_intersect.h index 826bdaae..29a5ea2a 100644 --- a/decoder/hg_intersect.h +++ b/decoder/hg_intersect.h @@ -1,13 +1,11 @@ #ifndef _HG_INTERSECT_H_ #define _HG_INTERSECT_H_ -#include - #include "lattice.h" class Hypergraph; -struct HG { - static bool Intersect(const Lattice& target, Hypergraph* hg); +namespace HG { + bool Intersect(const Lattice& target, Hypergraph* hg); }; #endif -- cgit v1.2.3 From 2ce5f342ec61c3628d142cf1e9074b8cb2b6d786 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 13 Aug 2012 00:05:50 -0400 Subject: comment out one test --- decoder/hg_test.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'decoder') diff --git a/decoder/hg_test.cc b/decoder/hg_test.cc index c47af850..37469748 100644 --- a/decoder/hg_test.cc +++ b/decoder/hg_test.cc @@ -140,12 +140,15 @@ BOOST_AUTO_TEST_CASE(PruneInsideOutside) { cerr << TD::GetString(trans) << "\n"; cerr << "cost: " << cost << "\n"; hg.PrintGraphviz(); +#if 0 hg.DensityPruneInsideOutside(0.5, false, 2.0); hg.BeamPruneInsideOutside(0.5, false, 0.5); cost = ViterbiESentence(hg, &trans); cerr << "Ncst: " << cost << endl; cerr << TD::GetString(trans) << "\n"; hg.PrintGraphviz(); +#endif + cerr << "FIX PLEASE\n"; } BOOST_AUTO_TEST_CASE(TestPruneEdges) { -- cgit v1.2.3 From ff7ad6621a2d003eb5281a4ae570635c6e693cf6 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 13 Aug 2012 09:59:00 -0400 Subject: maybe fix? --- decoder/hg_test.h | 1 + 1 file changed, 1 insertion(+) (limited to 'decoder') diff --git a/decoder/hg_test.h b/decoder/hg_test.h index c098961d..e96cb0b1 100644 --- a/decoder/hg_test.h +++ b/decoder/hg_test.h @@ -68,6 +68,7 @@ void AddNullEdge(Hypergraph* hg) { TRule x; x.arity_ = 0; hg->nodes_[0].in_edges_.push_back(hg->AddEdge(TRulePtr(new TRule(x)), Hypergraph::TailNodeVector())->id_); + hg->edges_.back().head_node_ = 0; } void HGSetup::CreateTinyLatticeHG(Hypergraph* hg) { -- cgit v1.2.3 From bc94f8c8c3fc3978a68859a67d33ccd243437016 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 16 Aug 2012 17:32:33 -0400 Subject: Update build system, add missing cc files --- Jamroot | 2 +- decoder/Jamfile | 3 ++ jam-files/sanity.jam | 116 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 96 insertions(+), 25 deletions(-) (limited to 'decoder') diff --git a/Jamroot b/Jamroot index 87a5a53d..ef426146 100644 --- a/Jamroot +++ b/Jamroot @@ -14,7 +14,7 @@ boost 104400 ; external-lib z ; with-google-hash = [ option.get "with-google-hash" ] ; -if [ test_header google/dense_hash_map ] || $(with-google-hash) { +if [ test_header sparsehash/dense_hash_map ] || $(with-google-hash) { requirements += HAVE_SPARSEHASH $(with-google-hash) ; } diff --git a/decoder/Jamfile b/decoder/Jamfile index 06c5bfda..da02d063 100644 --- a/decoder/Jamfile +++ b/decoder/Jamfile @@ -55,6 +55,9 @@ lib decoder : JSON_parser.c json_parse.cc grammar.cc + rescore_translator.cc + hg_remove_eps.cc + hg_union.cc $(glc) ..//utils ..//mteval diff --git a/jam-files/sanity.jam b/jam-files/sanity.jam index aefed9c0..809a186b 100644 --- a/jam-files/sanity.jam +++ b/jam-files/sanity.jam @@ -3,6 +3,8 @@ import option ; import os ; import path ; import project ; +import build-system ; +import version ; #Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php rule trim-nl ( str extras * ) { @@ -50,13 +52,25 @@ rule test_library ( name ) { constant CLEANING : $(cleaning) ; } +requirements = ; + +FORCE-STATIC = [ option.get "static" : : "yes" ] ; +if $(FORCE-STATIC) { + requirements += static ; +} + #Determine if a library can be compiled statically. rule auto-shared ( name : additional * ) { additional ?= "" ; if [ test_flags $(additional)" -static -l"$(name) ] { return ; } else { - return "shared" ; + if $(FORCE-STATIC) { + echo "Could not statically link against lib $(name). Your build will probably fail." ; + return ; + } else { + return "shared" ; + } } } @@ -86,28 +100,19 @@ else { boost-include = ; } } - -requirements = ; #Are we linking static binaries against shared boost? boost-auto-shared = [ auto-shared "boost_program_options" : $(L-boost-search) ] ; #Convenience rule for boost libraries. Defines library boost_$(name). -rule boost-lib ( name macro ) { +rule boost-lib ( name macro : deps * ) { #Link multi-threaded programs against the -mt version if available. Old #versions of boost do not have -mt tagged versions of all libraries. Sadly, #boost.jam does not handle this correctly. - if [ test_flags $(L-boost-search)" -lboost_"$(name)"-mt" ] { -# if [ test_flags $(L-boost-search)" -lboost_"$(name) ] { -# lib inner_boost_$(name) : : single $(boost-search) boost_$(name) ; -# lib inner_boost_$(name) : : multi $(boost-search) boost_$(name)-mt ; -# } else { - if ! multi in $(requirements) { - requirements += multi ; - } - lib inner_boost_$(name) : : multi $(boost-search) boost_$(name)-mt ; -# } + if [ test_flags $(L-boost-search)" -lboost_"$(name)"-mt-$(boost-lib-version)" ] { + lib inner_boost_$(name) : : single $(boost-search) boost_$(name)-$(boost-lib-version) : : $(deps) ; + lib inner_boost_$(name) : : multi $(boost-search) boost_$(name)-mt-$(boost-lib-version) : : $(deps) ; } else { - lib inner_boost_$(name) : : $(boost-search) boost_$(name) ; + lib inner_boost_$(name) : : $(boost-search) boost_$(name)-$(boost-lib-version) : : $(deps) ; } alias boost_$(name) : inner_boost_$(name) : $(boost-auto-shared) : : shared:BOOST_$(macro) $(boost-include) ; @@ -115,7 +120,7 @@ rule boost-lib ( name macro ) { #Argument is e.g. 103600 rule boost ( min-version ) { - local cmd = "bash -c \"g++ "$(I-boost-include)" -dM -x c++ -E /dev/null -include boost/version.hpp 2>/dev/null |grep '#define BOOST_VERSION '\"" ; + local cmd = "bash -c \"g++ "$(I-boost-include)" -dM -x c++ -E /dev/null -include boost/version.hpp 2>/dev/null |grep '#define BOOST_'\"" ; local boost-shell = [ SHELL "$(cmd)" : exit-status ] ; if $(boost-shell[2]) != 0 && $(CLEANING) = no { echo Failed to run "$(cmd)" ; @@ -125,13 +130,14 @@ rule boost ( min-version ) { if $(boost-version) < $(min-version) && $(CLEANING) = no { exit You have Boost $(boost-version). This package requires Boost at least $(min-version) (and preferably newer). : 1 ; } + boost-lib-version = [ MATCH "#define BOOST_LIB_VERSION \"([^\"]*)\"" : $(boost-shell[1]) ] ; #See tools/build/v2/contrib/boost.jam in a boost distribution for a table of macros to define. - boost-lib thread THREAD_DYN_DLL ; + boost-lib system SYSTEM_DYN_LINK ; + boost-lib thread THREAD_DYN_DLL : boost_system ; boost-lib program_options PROGRAM_OPTIONS_DYN_LINK ; boost-lib unit_test_framework TEST_DYN_LINK ; boost-lib iostreams IOSTREAMS_DYN_LINK ; - boost-lib serialization SERIALIZATION_DYN_LINK ; - boost-lib mpi MPI_DYN_LINK ; + boost-lib filesystem FILE_SYSTEM_DYN_LINK ; } #Link normally to a library, but sometimes static isn't installed so fall back to dynamic. @@ -157,10 +163,10 @@ rule external-lib ( name : search-path * ) { local ignored = @($(build-log):E=$(script)) ; } -{ - #Boost jam's static clang for Linux is buggy. - requirements += $(cxxflags) $(cflags) $(ldflags) LINUX,clang:shared ; +#Boost jam's static clang for Linux is buggy. +requirements += $(cxxflags) $(cflags) $(ldflags) LINUX,clang:shared ; +if ! [ option.get "without-libsegfault" : : "yes" ] && ! $(FORCE-STATIC) { #libSegFault prints a stack trace on segfault. Link against it if available. if [ test_flags "-lSegFault" ] { external-lib SegFault ; @@ -178,8 +184,9 @@ if [ option.get "git" : : "yes" ] { prefix = [ option.get "prefix" ] ; if $(prefix) { prefix = [ path.root $(prefix) [ path.pwd ] ] ; + prefix = $(prefix)$(GITTAG) ; } else { - prefix = $(TOP)/dist$(GITTAG) ; + prefix = $(TOP)$(GITTAG) ; } bindir = [ option.get "bindir" : $(prefix)/bin ] ; @@ -195,7 +202,68 @@ rule install-headers ( name : list * : source-root ? ) { } rule build-projects ( projects * ) { - for p in $(projects) { + for local p in $(projects) { build-project $(p) ; } } + +#Only one post build hook is allowed. Allow multiple. +post-hooks = ; +rule post-build ( ok ? ) { + for local r in $(post-hooks) { + $(r) $(ok) ; + } +} +IMPORT $(__name__) : post-build : : $(__name__).post-build ; +build-system.set-post-build-hook $(__name__).post-build ; +rule add-post-hook ( names * ) { + post-hooks += $(names) ; +} + + +#Backend for writing content to files after build completes. +post-files = ; +post-contents = ; +rule save-post-build ( ok ? ) { + if $(ok) { + while $(post-files) { + local ignored = @($(post-files[1]):E=$(post-contents[1])) ; + post-files = $(post-files[2-]) ; + post-contents = $(post-contents[2-]) ; + } + } +} +add-post-hook save-post-build ; + +#Queue content to be written to file when build completes successfully. +rule add-post-write ( name content ) { + post-files += $(name) ; + post-contents += $(content) ; +} + +#Compare contents of file with current. If they're different, force the targets to rebuild then overwrite the file. +rule always-if-changed ( file current : targets * ) { + local previous = inconsistent ; + if [ path.exists $(file) ] { + previous = [ _shell "cat $(file)" ] ; + } + if $(current) != $(previous) { + #Write inconsistent while the build is running + if [ path.exists $(file) ] { + local ignored = @($(file):E=inconsistent) ; + } + add-post-write $(file) $(current) ; + for local i in $(targets) { + always $(i) ; + } + } +} + +if [ option.get "sanity-test" : : "yes" ] { + local current_version = [ modules.peek : JAM_VERSION ] ; + if ( $(current_version[0]) < 2000 && [ version.check-jam-version 3 1 16 ] ) || [ version.check-jam-version 2011 0 0 ] { + EXIT "Sane" : 0 ; + } else { + EXIT "Bad" : 1 ; + } +} -- cgit v1.2.3 From 58da794845e192cdae3b5ba96cacf234d6a66ed0 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Wed, 12 Sep 2012 18:36:03 +0100 Subject: Partially written bridge to lazy --- decoder/lazy.cc | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ decoder/lazy.h | 8 ++++ 2 files changed, 130 insertions(+) create mode 100644 decoder/lazy.cc create mode 100644 decoder/lazy.h (limited to 'decoder') diff --git a/decoder/lazy.cc b/decoder/lazy.cc new file mode 100644 index 00000000..f5b61c75 --- /dev/null +++ b/decoder/lazy.cc @@ -0,0 +1,122 @@ +#include "hg.h" +#include "lazy.h" +#include "tdict.h" + +#include "lm/enumerate_vocab.hh" +#include "lm/model.hh" +#include "search/edge.hh" +#include "search/vertex.hh" +#include "util/exception.hh" + +#include + +namespace { + +struct MapVocab : public lm::EnumerateVocab { + public: + MapVocab() {} + + // Do not call after Lookup. + void Add(lm::WordIndex index, const StringPiece &str) { + const WordID cdec_id = TD::Convert(str.as_string()); + if (cdec_id >= out_->size()) out_.resize(cdec_id + 1); + out_[cdec_id] = index; + } + + // Assumes Add has been called and will never be called again. + lm::WordIndex FromCDec(WordID id) const { + return out_[out.size() > id ? id : 0]; + } + + private: + std::vector out_; +}; + +class LazyBase { + public: + LazyBase() {} + + virtual ~LazyBase() {} + + virtual void Search(const Hypergraph &hg) const = 0; + + static LazyBase *Load(const char *model_file); + + protected: + lm::ngram::Config GetConfig() const { + lm::ngram::Config ret; + ret.enumerate_vocab = &vocab_; + return ret; + } + + MapVocab vocab_; +}; + +template class Lazy : public LazyBase { + public: + explicit Lazy(const char *model_file) : m_(model_file, GetConfig()) {} + + void Search(const Hypergraph &hg) const; + + private: + void ConvertEdge(const Context &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const; + + const Model m_; +}; + +static LazyBase *LazyBase::Load(const char *model_file) { + lm::ngram::ModelType model_type; + if (!lm::ngram::RecognizeBinary(lm_name, model_type)) model_type = lm::ngram::PROBING; + switch (model_type) { + case lm::ngram::PROBING: + return new Lazy(model_file); + case lm::ngram::REST_PROBING: + return new Lazy(model_file); + default: + UTIL_THROW(util::Exception, "Sorry this lm type isn't supported yet."); + } +} + +template void Lazy::Search(const Hypergraph &hg) const { + boost::scoped_array out_vertices(new search::Vertex[hg.nodes_.size()]); + boost::scoped_array out_edges(new search::Edge[hg.edges_.size()]); + for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { + search::Vertex *out_vertex = out_vertices[i]; + const Hypergraph::EdgesVector &down_edges = hg.nodes_[i].in_edges_; + for (unsigned int j = 0; j < edges.size(); ++j) { + unsigned int edge_index = down_edges[j]; + const Hypergraph::Edge &in_edge = hg.edges_[edge_index]; + search::Edge &out_edge = out_edges[edge_index]; + } + } +} + +// TODO: get weights into here somehow. +template void Lazy::ConvertEdge(const Context &context, bool final, search::Vertices *vertices, const Hypergraph::Edge &in, search::Edge &out) const { + const std::vector &e = in_edge.rule_->e(); + std::vector words; + unsigned int terminals = 0; + for (std::vector::const_iterator word = e.begin(); word != e.end(); ++word) { + if (*word <= 0) { + out.Add(vertices[edge.tail_nodes_[-*word]]); + words.push_back(lm::kMaxWordIndex); + } else { + ++terminals; + words.push_back(vocab_.FromCDec(*word)); + } + } + + if (final) { + words.push_back(m_.GetVocabulary().EndSentence()); + } + + float additive = edge.rule_->GetFeatureValues().dot(weight_vector); + + out.InitRule().Init(context, additive, words, final); +} + +} // namespace + +void PassToLazy(const Hypergraph &hg) { + +} diff --git a/decoder/lazy.h b/decoder/lazy.h new file mode 100644 index 00000000..aecd030d --- /dev/null +++ b/decoder/lazy.h @@ -0,0 +1,8 @@ +#ifndef _LAZY_H_ +#define _LAZY_H_ + +class Hypergraph; + +void PassToLazy(const Hypergraph &hg); + +#endif // _LAZY_H_ -- cgit v1.2.3 From 8505fdfdf0bc4ce9acec42e1980a2fdd4f254109 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 13 Sep 2012 11:15:32 +0100 Subject: It compiles. --- decoder/Jamfile | 2 ++ decoder/decoder.cc | 4 +++ decoder/lazy.cc | 78 +++++++++++++++++++++++++++++++++++++-------------- decoder/lazy.h | 5 +++- klm/search/config.hh | 6 ++-- klm/search/weights.cc | 2 ++ klm/search/weights.hh | 17 ++++++----- 7 files changed, 82 insertions(+), 32 deletions(-) (limited to 'decoder') diff --git a/decoder/Jamfile b/decoder/Jamfile index da02d063..d778dc7f 100644 --- a/decoder/Jamfile +++ b/decoder/Jamfile @@ -58,10 +58,12 @@ lib decoder : rescore_translator.cc hg_remove_eps.cc hg_union.cc + lazy.cc $(glc) ..//utils ..//mteval ../klm/lm//kenlm + ../klm/search//search ..//boost_program_options : . : : diff --git a/decoder/decoder.cc b/decoder/decoder.cc index a69a6d05..3a410cf2 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -38,6 +38,7 @@ #include "sampler.h" #include "forest_writer.h" // TODO this section should probably be handled by an Observer +#include "lazy.h" #include "hg_io.h" #include "aligner.h" @@ -832,6 +833,9 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { if (conf.count("show_target_graph")) HypergraphIO::WriteTarget(conf["show_target_graph"].as(), sent_id, forest); + if (conf.count("lazy_search")) + PassToLazy(forest, CurrentWeightVector()); + for (int pass = 0; pass < rescoring_passes.size(); ++pass) { const RescoringPass& rp = rescoring_passes[pass]; const vector& cur_weights = *rp.weight_vector; diff --git a/decoder/lazy.cc b/decoder/lazy.cc index f5b61c75..4776c1b8 100644 --- a/decoder/lazy.cc +++ b/decoder/lazy.cc @@ -1,15 +1,23 @@ #include "hg.h" #include "lazy.h" +#include "fdict.h" #include "tdict.h" #include "lm/enumerate_vocab.hh" #include "lm/model.hh" +#include "search/config.hh" +#include "search/context.hh" #include "search/edge.hh" #include "search/vertex.hh" +#include "search/vertex_generator.hh" #include "util/exception.hh" +#include #include +#include +#include + namespace { struct MapVocab : public lm::EnumerateVocab { @@ -19,13 +27,13 @@ struct MapVocab : public lm::EnumerateVocab { // Do not call after Lookup. void Add(lm::WordIndex index, const StringPiece &str) { const WordID cdec_id = TD::Convert(str.as_string()); - if (cdec_id >= out_->size()) out_.resize(cdec_id + 1); + if (cdec_id >= out_.size()) out_.resize(cdec_id + 1); out_[cdec_id] = index; } // Assumes Add has been called and will never be called again. lm::WordIndex FromCDec(WordID id) const { - return out_[out.size() > id ? id : 0]; + return out_[out_.size() > id ? id : 0]; } private: @@ -34,44 +42,50 @@ struct MapVocab : public lm::EnumerateVocab { class LazyBase { public: - LazyBase() {} + LazyBase(const std::vector &weights) : + cdec_weights_(weights), + config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) {} virtual ~LazyBase() {} virtual void Search(const Hypergraph &hg) const = 0; - static LazyBase *Load(const char *model_file); + static LazyBase *Load(const char *model_file, const std::vector &weights); protected: - lm::ngram::Config GetConfig() const { + lm::ngram::Config GetConfig() { lm::ngram::Config ret; ret.enumerate_vocab = &vocab_; return ret; } MapVocab vocab_; + + const std::vector &cdec_weights_; + + const search::Config config_; }; template class Lazy : public LazyBase { public: - explicit Lazy(const char *model_file) : m_(model_file, GetConfig()) {} + Lazy(const char *model_file, const std::vector &weights) : LazyBase(weights), m_(model_file, GetConfig()) {} void Search(const Hypergraph &hg) const; private: - void ConvertEdge(const Context &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const; + void ConvertEdge(const search::Context &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const; const Model m_; }; -static LazyBase *LazyBase::Load(const char *model_file) { +LazyBase *LazyBase::Load(const char *model_file, const std::vector &weights) { lm::ngram::ModelType model_type; - if (!lm::ngram::RecognizeBinary(lm_name, model_type)) model_type = lm::ngram::PROBING; + if (!lm::ngram::RecognizeBinary(model_file, model_type)) model_type = lm::ngram::PROBING; switch (model_type) { case lm::ngram::PROBING: - return new Lazy(model_file); + return new Lazy(model_file, weights); case lm::ngram::REST_PROBING: - return new Lazy(model_file); + return new Lazy(model_file, weights); default: UTIL_THROW(util::Exception, "Sorry this lm type isn't supported yet."); } @@ -80,25 +94,41 @@ static LazyBase *LazyBase::Load(const char *model_file) { template void Lazy::Search(const Hypergraph &hg) const { boost::scoped_array out_vertices(new search::Vertex[hg.nodes_.size()]); boost::scoped_array out_edges(new search::Edge[hg.edges_.size()]); + + search::Context context(config_, m_); + for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { - search::Vertex *out_vertex = out_vertices[i]; + search::Vertex &out_vertex = out_vertices[i]; const Hypergraph::EdgesVector &down_edges = hg.nodes_[i].in_edges_; - for (unsigned int j = 0; j < edges.size(); ++j) { + for (unsigned int j = 0; j < down_edges.size(); ++j) { unsigned int edge_index = down_edges[j]; - const Hypergraph::Edge &in_edge = hg.edges_[edge_index]; - search::Edge &out_edge = out_edges[edge_index]; + ConvertEdge(context, i == hg.nodes_.size() - 1, out_vertices.get(), hg.edges_[edge_index], out_edges[edge_index]); + out_vertex.Add(out_edges[edge_index]); } + out_vertex.FinishedAdding(); + search::VertexGenerator(context, out_vertex); + } + search::PartialVertex top = out_vertices[hg.nodes_.size() - 1].RootPartial(); + if (top.Empty()) { + std::cout << "NO PATH FOUND"; + } else { + search::PartialVertex continuation; + while (!top.Complete()) { + top.Split(continuation); + top = continuation; + } + std::cout << top.End().Bound() << std::endl; } } // TODO: get weights into here somehow. -template void Lazy::ConvertEdge(const Context &context, bool final, search::Vertices *vertices, const Hypergraph::Edge &in, search::Edge &out) const { - const std::vector &e = in_edge.rule_->e(); +template void Lazy::ConvertEdge(const search::Context &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const { + const std::vector &e = in.rule_->e(); std::vector words; unsigned int terminals = 0; for (std::vector::const_iterator word = e.begin(); word != e.end(); ++word) { if (*word <= 0) { - out.Add(vertices[edge.tail_nodes_[-*word]]); + out.Add(vertices[in.tail_nodes_[-*word]]); words.push_back(lm::kMaxWordIndex); } else { ++terminals; @@ -110,13 +140,19 @@ template void Lazy::ConvertEdge(const Context &conte words.push_back(m_.GetVocabulary().EndSentence()); } - float additive = edge.rule_->GetFeatureValues().dot(weight_vector); + float additive = in.rule_->GetFeatureValues().dot(cdec_weights_); + additive -= terminals * context.GetWeights().WordPenalty() * static_cast(terminals) / M_LN10; out.InitRule().Init(context, additive, words, final); } -} // namespace +boost::scoped_ptr AwfulGlobalLazy; -void PassToLazy(const Hypergraph &hg) { +} // namespace +void PassToLazy(const Hypergraph &hg, const std::vector &weights) { + if (!AwfulGlobalLazy.get()) { + AwfulGlobalLazy.reset(LazyBase::Load("lm", weights)); + } + AwfulGlobalLazy->Search(hg); } diff --git a/decoder/lazy.h b/decoder/lazy.h index aecd030d..3e71a3b0 100644 --- a/decoder/lazy.h +++ b/decoder/lazy.h @@ -1,8 +1,11 @@ #ifndef _LAZY_H_ #define _LAZY_H_ +#include "weights.h" +#include + class Hypergraph; -void PassToLazy(const Hypergraph &hg); +void PassToLazy(const Hypergraph &hg, const std::vector &weights); #endif // _LAZY_H_ diff --git a/klm/search/config.hh b/klm/search/config.hh index e21e4b7c..ef8e2354 100644 --- a/klm/search/config.hh +++ b/klm/search/config.hh @@ -8,15 +8,15 @@ namespace search { class Config { public: - Config(StringPiece weight_str, unsigned int pop_limit) : - weights_(weight_str), pop_limit_(pop_limit) {} + Config(const Weights &weights, unsigned int pop_limit) : + weights_(weights), pop_limit_(pop_limit) {} const Weights &GetWeights() const { return weights_; } unsigned int PopLimit() const { return pop_limit_; } private: - search::Weights weights_; + Weights weights_; unsigned int pop_limit_; }; diff --git a/klm/search/weights.cc b/klm/search/weights.cc index 82ff3f12..d65471ad 100644 --- a/klm/search/weights.cc +++ b/klm/search/weights.cc @@ -49,6 +49,8 @@ Weights::Weights(StringPiece text) { word_penalty_ = Steal("WordPenalty"); } +Weights::Weights(Score lm, Score oov, Score word_penalty) : lm_(lm), oov_(oov), word_penalty_(word_penalty) {} + search::Score Weights::DotNoLM(StringPiece text) const { DotProduct dot; Parse(text, map_, dot); diff --git a/klm/search/weights.hh b/klm/search/weights.hh index 4a4388c7..df1c419f 100644 --- a/klm/search/weights.hh +++ b/klm/search/weights.hh @@ -23,25 +23,28 @@ class Weights { // Parses weights, sets lm_weight_, removes it from map_. explicit Weights(StringPiece text); - search::Score DotNoLM(StringPiece text) const; + // Just the three scores we care about adding. + Weights(Score lm, Score oov, Score word_penalty); - search::Score LM() const { return lm_; } + Score DotNoLM(StringPiece text) const; - search::Score OOV() const { return oov_; } + Score LM() const { return lm_; } - search::Score WordPenalty() const { return word_penalty_; } + Score OOV() const { return oov_; } + + Score WordPenalty() const { return word_penalty_; } // Mostly for testing. - const boost::unordered_map &GetMap() const { return map_; } + const boost::unordered_map &GetMap() const { return map_; } private: float Steal(const std::string &str); - typedef boost::unordered_map Map; + typedef boost::unordered_map Map; Map map_; - search::Score lm_, oov_, word_penalty_; + Score lm_, oov_, word_penalty_; }; } // namespace search -- cgit v1.2.3 From a950a83a807518e465706c3712d6f80afff460b9 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 13 Sep 2012 04:28:30 -0700 Subject: Allow lm file name, print weights --- decoder/decoder.cc | 3 ++- decoder/lazy.cc | 10 +++++++--- decoder/lazy.h | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) (limited to 'decoder') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 3a410cf2..525c6ba6 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -416,6 +416,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream ("show_conditional_prob", "Output the conditional log prob to STDOUT instead of a translation") ("show_cfg_search_space", "Show the search space as a CFG") ("show_target_graph", po::value(), "Directory to write the target hypergraphs to") + ("lazy_search", po::value(), "Run lazy search with this language model file") ("coarse_to_fine_beam_prune", po::value(), "Prune paths from coarse parse forest before fine parse, keeping paths within exp(alpha>=0)") ("ctf_beam_widen", po::value()->default_value(2.0), "Expand coarse pass beam by this factor if no fine parse is found") ("ctf_num_widenings", po::value()->default_value(2), "Widen coarse beam this many times before backing off to full parse") @@ -834,7 +835,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { HypergraphIO::WriteTarget(conf["show_target_graph"].as(), sent_id, forest); if (conf.count("lazy_search")) - PassToLazy(forest, CurrentWeightVector()); + PassToLazy(conf["lazy_search"].as().c_str(), CurrentWeightVector(), forest); for (int pass = 0; pass < rescoring_passes.size(); ++pass) { const RescoringPass& rp = rescoring_passes[pass]; diff --git a/decoder/lazy.cc b/decoder/lazy.cc index 4776c1b8..58a9e08a 100644 --- a/decoder/lazy.cc +++ b/decoder/lazy.cc @@ -44,7 +44,9 @@ class LazyBase { public: LazyBase(const std::vector &weights) : cdec_weights_(weights), - config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) {} + config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) { + std::cerr << "Weights KLanguageModel " << config_.GetWeights().LM() << " KLanguageModel_OOV " << config_.GetWeights().OOV() << " WordPenalty " << config_.GetWeights().WordPenalty() << std::endl; + } virtual ~LazyBase() {} @@ -95,6 +97,7 @@ template void Lazy::Search(const Hypergraph &hg) const { boost::scoped_array out_vertices(new search::Vertex[hg.nodes_.size()]); boost::scoped_array out_edges(new search::Edge[hg.edges_.size()]); + search::Context context(config_, m_); for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { @@ -141,6 +144,7 @@ template void Lazy::ConvertEdge(const search::ContextGetFeatureValues().dot(cdec_weights_); + UTIL_THROW_IF(isnan(additive), util::Exception, "Bad dot product"); additive -= terminals * context.GetWeights().WordPenalty() * static_cast(terminals) / M_LN10; out.InitRule().Init(context, additive, words, final); @@ -150,9 +154,9 @@ boost::scoped_ptr AwfulGlobalLazy; } // namespace -void PassToLazy(const Hypergraph &hg, const std::vector &weights) { +void PassToLazy(const char *model_file, const std::vector &weights, const Hypergraph &hg) { if (!AwfulGlobalLazy.get()) { - AwfulGlobalLazy.reset(LazyBase::Load("lm", weights)); + AwfulGlobalLazy.reset(LazyBase::Load(model_file, weights)); } AwfulGlobalLazy->Search(hg); } diff --git a/decoder/lazy.h b/decoder/lazy.h index 3e71a3b0..d1f030d1 100644 --- a/decoder/lazy.h +++ b/decoder/lazy.h @@ -6,6 +6,6 @@ class Hypergraph; -void PassToLazy(const Hypergraph &hg, const std::vector &weights); +void PassToLazy(const char *model_file, const std::vector &weights, const Hypergraph &hg); #endif // _LAZY_H_ -- cgit v1.2.3 From df001c7fb050bf5e424af4a0948ae7e23486fc06 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 13 Sep 2012 14:15:03 +0100 Subject: Fix word penalty --- decoder/lazy.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'decoder') diff --git a/decoder/lazy.cc b/decoder/lazy.cc index 58a9e08a..9d69dac6 100644 --- a/decoder/lazy.cc +++ b/decoder/lazy.cc @@ -93,11 +93,21 @@ LazyBase *LazyBase::Load(const char *model_file, const std::vector &we } } +void PrintFinal(const Hypergraph &hg, const search::Edge *edge_base, const search::Final &final) { + const std::vector &words = hg.edges_[&final.From() - edge_base].rule_->e(); + boost::array::const_iterator child(final.Children().begin()); + for (std::vector::const_iterator i = words.begin(); i != words.end(); ++i) { + if (*i > 0) { + std::cout << TD::Convert(*i) << ' '; + } else { + PrintFinal(hg, edge_base, **child++); + } + } +} + template void Lazy::Search(const Hypergraph &hg) const { boost::scoped_array out_vertices(new search::Vertex[hg.nodes_.size()]); boost::scoped_array out_edges(new search::Edge[hg.edges_.size()]); - - search::Context context(config_, m_); for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { @@ -120,7 +130,8 @@ template void Lazy::Search(const Hypergraph &hg) const { top.Split(continuation); top = continuation; } - std::cout << top.End().Bound() << std::endl; + PrintFinal(hg, out_edges.get(), top.End()); + std::cout << "||| " << top.End().Bound() << std::endl; } } @@ -145,7 +156,7 @@ template void Lazy::ConvertEdge(const search::ContextGetFeatureValues().dot(cdec_weights_); UTIL_THROW_IF(isnan(additive), util::Exception, "Bad dot product"); - additive -= terminals * context.GetWeights().WordPenalty() * static_cast(terminals) / M_LN10; + additive -= static_cast(terminals) * context.GetWeights().WordPenalty() / M_LN10; out.InitRule().Init(context, additive, words, final); } -- cgit v1.2.3 From c32c03c1e0bb1e0407c90032cd3bf41f8bd61251 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Thu, 13 Sep 2012 07:53:16 -0700 Subject: Steal cubepruning_pop_limit command line argument --- decoder/decoder.cc | 2 +- decoder/lazy.cc | 19 ++++++++++--------- decoder/lazy.h | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) (limited to 'decoder') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 525c6ba6..83077a68 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -835,7 +835,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { HypergraphIO::WriteTarget(conf["show_target_graph"].as(), sent_id, forest); if (conf.count("lazy_search")) - PassToLazy(conf["lazy_search"].as().c_str(), CurrentWeightVector(), forest); + PassToLazy(conf["lazy_search"].as().c_str(), CurrentWeightVector(), pop_limit, forest); for (int pass = 0; pass < rescoring_passes.size(); ++pass) { const RescoringPass& rp = rescoring_passes[pass]; diff --git a/decoder/lazy.cc b/decoder/lazy.cc index 9d69dac6..0f12a1ff 100644 --- a/decoder/lazy.cc +++ b/decoder/lazy.cc @@ -44,13 +44,13 @@ class LazyBase { public: LazyBase(const std::vector &weights) : cdec_weights_(weights), - config_(search::Weights(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]), 1000) { - std::cerr << "Weights KLanguageModel " << config_.GetWeights().LM() << " KLanguageModel_OOV " << config_.GetWeights().OOV() << " WordPenalty " << config_.GetWeights().WordPenalty() << std::endl; + weights_(weights[FD::Convert("KLanguageModel")], weights[FD::Convert("KLanguageModel_OOV")], weights[FD::Convert("WordPenalty")]) { + std::cerr << "Weights KLanguageModel " << weights_.LM() << " KLanguageModel_OOV " << weights_.OOV() << " WordPenalty " << weights_.WordPenalty() << std::endl; } virtual ~LazyBase() {} - virtual void Search(const Hypergraph &hg) const = 0; + virtual void Search(unsigned int pop_limit, const Hypergraph &hg) const = 0; static LazyBase *Load(const char *model_file, const std::vector &weights); @@ -65,14 +65,14 @@ class LazyBase { const std::vector &cdec_weights_; - const search::Config config_; + const search::Weights weights_; }; template class Lazy : public LazyBase { public: Lazy(const char *model_file, const std::vector &weights) : LazyBase(weights), m_(model_file, GetConfig()) {} - void Search(const Hypergraph &hg) const; + void Search(unsigned int pop_limit, const Hypergraph &hg) const; private: void ConvertEdge(const search::Context &context, bool final, search::Vertex *vertices, const Hypergraph::Edge &in, search::Edge &out) const; @@ -105,10 +105,11 @@ void PrintFinal(const Hypergraph &hg, const search::Edge *edge_base, const searc } } -template void Lazy::Search(const Hypergraph &hg) const { +template void Lazy::Search(unsigned int pop_limit, const Hypergraph &hg) const { boost::scoped_array out_vertices(new search::Vertex[hg.nodes_.size()]); boost::scoped_array out_edges(new search::Edge[hg.edges_.size()]); - search::Context context(config_, m_); + search::Config config(weights_, pop_limit); + search::Context context(config, m_); for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { search::Vertex &out_vertex = out_vertices[i]; @@ -165,9 +166,9 @@ boost::scoped_ptr AwfulGlobalLazy; } // namespace -void PassToLazy(const char *model_file, const std::vector &weights, const Hypergraph &hg) { +void PassToLazy(const char *model_file, const std::vector &weights, unsigned int pop_limit, const Hypergraph &hg) { if (!AwfulGlobalLazy.get()) { AwfulGlobalLazy.reset(LazyBase::Load(model_file, weights)); } - AwfulGlobalLazy->Search(hg); + AwfulGlobalLazy->Search(pop_limit, hg); } diff --git a/decoder/lazy.h b/decoder/lazy.h index d1f030d1..94895b19 100644 --- a/decoder/lazy.h +++ b/decoder/lazy.h @@ -6,6 +6,6 @@ class Hypergraph; -void PassToLazy(const char *model_file, const std::vector &weights, const Hypergraph &hg); +void PassToLazy(const char *model_file, const std::vector &weights, unsigned int pop_limit, const Hypergraph &hg); #endif // _LAZY_H_ -- cgit v1.2.3 From 903fb23675b894f6d3cb2786c134f03313c5d474 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Fri, 14 Sep 2012 06:26:17 -0700 Subject: Skip top node, just integrate and with the top edges. --- decoder/lazy.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'decoder') diff --git a/decoder/lazy.cc b/decoder/lazy.cc index 0f12a1ff..c4138d7b 100644 --- a/decoder/lazy.cc +++ b/decoder/lazy.cc @@ -111,18 +111,18 @@ template void Lazy::Search(unsigned int pop_limit, const Hy search::Config config(weights_, pop_limit); search::Context context(config, m_); - for (unsigned int i = 0; i < hg.nodes_.size(); ++i) { + for (unsigned int i = 0; i < hg.nodes_.size() - 1; ++i) { search::Vertex &out_vertex = out_vertices[i]; const Hypergraph::EdgesVector &down_edges = hg.nodes_[i].in_edges_; for (unsigned int j = 0; j < down_edges.size(); ++j) { unsigned int edge_index = down_edges[j]; - ConvertEdge(context, i == hg.nodes_.size() - 1, out_vertices.get(), hg.edges_[edge_index], out_edges[edge_index]); + ConvertEdge(context, i == hg.nodes_.size() - 2, out_vertices.get(), hg.edges_[edge_index], out_edges[edge_index]); out_vertex.Add(out_edges[edge_index]); } out_vertex.FinishedAdding(); search::VertexGenerator(context, out_vertex); } - search::PartialVertex top = out_vertices[hg.nodes_.size() - 1].RootPartial(); + search::PartialVertex top = out_vertices[hg.nodes_.size() - 2].RootPartial(); if (top.Empty()) { std::cout << "NO PATH FOUND"; } else { @@ -168,6 +168,7 @@ boost::scoped_ptr AwfulGlobalLazy; void PassToLazy(const char *model_file, const std::vector &weights, unsigned int pop_limit, const Hypergraph &hg) { if (!AwfulGlobalLazy.get()) { + std::cerr << "Pop limit " << pop_limit << std::endl; AwfulGlobalLazy.reset(LazyBase::Load(model_file, weights)); } AwfulGlobalLazy->Search(pop_limit, hg); -- cgit v1.2.3 From e1d05e2d73abe9574cb938764976583bba936578 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 24 Sep 2012 22:18:32 -0400 Subject: add timing stats for Kenneth --- decoder/cdec.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'decoder') diff --git a/decoder/cdec.cc b/decoder/cdec.cc index c671af57..4819105c 100644 --- a/decoder/cdec.cc +++ b/decoder/cdec.cc @@ -4,6 +4,7 @@ #include "decoder.h" #include "ff_register.h" #include "verbose.h" +#include "timing_stats.h" using namespace std; @@ -27,6 +28,7 @@ int main(int argc, char** argv) { if (buf.empty()) continue; decoder.Decode(buf); } + Timer::Summarize(); #ifdef CP_TIME cerr << "Time required for Cube Pruning execution: " << CpTime::Get() -- cgit v1.2.3 From 3a96fa30bcb01b6693a927832ab282e589d9c226 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Mon, 1 Oct 2012 09:42:43 -0700 Subject: Add memory size logging --- decoder/cdec.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'decoder') diff --git a/decoder/cdec.cc b/decoder/cdec.cc index c671af57..25d3b6af 100644 --- a/decoder/cdec.cc +++ b/decoder/cdec.cc @@ -4,6 +4,7 @@ #include "decoder.h" #include "ff_register.h" #include "verbose.h" +#include "util/usage.hh" using namespace std; @@ -38,6 +39,7 @@ int main(int argc, char** argv) { cout << FD::Convert(i) << endl; } } + util::PrintUsage(std::cerr); return 0; } -- cgit v1.2.3 From 925087356b853e2099c1b60d8b757d7aa02121a9 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 2 Oct 2012 00:19:43 -0400 Subject: cdec cleanup, remove bayesian stuff, parsing stuff --- Jamroot | 45 - Makefile.am | 6 +- bjam | 23 - configure.ac | 5 - decoder/Jamfile | 81 - decoder/decoder.h | 2 +- decoder/hg.h | 4 +- dpmert/Jamfile | 32 - gi/clda/src/Makefile.am | 6 - gi/clda/src/ccrp.h | 291 -- gi/clda/src/clda.cc | 148 - gi/clda/src/crp.h | 50 - gi/clda/src/slice_sampler.h | 191 - gi/clda/src/timer.h | 20 - gi/evaluation/conditional_entropy.py | 61 - gi/evaluation/confusion_matrix.py | 123 - gi/evaluation/entropy.py | 38 - gi/evaluation/extract_ccg_labels.py | 129 - gi/evaluation/tree.py | 485 --- gi/markov_al/Makefile.am | 6 - gi/markov_al/README | 2 - gi/markov_al/ml.cc | 470 --- gi/morf-segmentation/filter_docs.pl | 24 - gi/morf-segmentation/invalid_vocab.patterns | 6 - gi/morf-segmentation/linestripper.py | 40 - gi/morf-segmentation/morf-pipeline.pl | 486 --- gi/morf-segmentation/morfsegment.py | 50 - gi/morf-segmentation/morftrain.sh | 110 - gi/morf-segmentation/vocabextractor.sh | 40 - gi/pf/Makefile.am | 44 - gi/pf/README | 2 - gi/pf/align-lexonly-pyp.cc | 243 -- gi/pf/align-tl.cc | 339 -- gi/pf/backward.cc | 89 - gi/pf/backward.h | 33 - gi/pf/base_distributions.cc | 241 -- gi/pf/base_distributions.h | 238 -- gi/pf/bayes_lattice_score.cc | 309 -- gi/pf/brat.cc | 543 --- gi/pf/cbgi.cc | 330 -- gi/pf/cfg_wfst_composer.cc | 731 ---- gi/pf/cfg_wfst_composer.h | 46 - gi/pf/conditional_pseg.h | 275 -- gi/pf/condnaive.cc | 298 -- gi/pf/corpus.cc | 62 - gi/pf/corpus.h | 19 - gi/pf/dpnaive.cc | 301 -- gi/pf/guess-translits.pl | 72 - gi/pf/hpyp_tm.cc | 133 - gi/pf/hpyp_tm.h | 38 - gi/pf/itg.cc | 275 -- gi/pf/learn_cfg.cc | 428 --- gi/pf/make-freq-bins.pl | 26 - gi/pf/mh_test.cc | 148 - gi/pf/monotonic_pseg.h | 89 - gi/pf/ngram_base.cc | 69 - gi/pf/ngram_base.h | 25 - gi/pf/nuisance_test.cc | 161 - gi/pf/os_phrase.h | 15 - gi/pf/pf.h | 84 - gi/pf/pf_test.cc | 148 - gi/pf/pfbrat.cc | 543 --- gi/pf/pfdist.cc | 598 --- gi/pf/pfdist.new.cc | 620 --- gi/pf/pfnaive.cc | 284 -- gi/pf/poisson_uniform_word_model.h | 50 - gi/pf/pyp_lm.cc | 273 -- gi/pf/pyp_tm.cc | 128 - gi/pf/pyp_tm.h | 36 - gi/pf/pyp_word_model.h | 61 - gi/pf/quasi_model2.h | 177 - gi/pf/reachability.cc | 74 - gi/pf/reachability.h | 34 - gi/pf/tied_resampler.h | 122 - gi/pf/tpf.cc | 99 - gi/pf/transliterations.cc | 334 -- gi/pf/transliterations.h | 24 - gi/pf/unigrams.cc | 80 - gi/pf/unigrams.h | 69 - gi/pipeline/OLD.clsp.config | 9 - gi/pipeline/OLD.evaluation-pipeline.pl | 277 -- gi/pipeline/backoff-pipe.pl | 215 -- gi/pipeline/blacklight.config | 9 - gi/pipeline/clsp.config | 10 - gi/pipeline/evaluation-pipeline.pl | 364 -- gi/pipeline/local-gi-pipeline.pl | 465 --- gi/pipeline/lticluster.config | 9 - gi/pipeline/scripts/filter-by-f.pl | 56 - gi/pipeline/scripts/patch-corpus.pl | 65 - gi/pipeline/scripts/refilter.pl | 40 - gi/pipeline/scripts/rekey.pl | 8 - gi/pipeline/scripts/remove-tags-from-contexts.pl | 53 - gi/pipeline/scripts/remove-tags-from-corpus.pl | 44 - gi/pipeline/scripts/sort-by-key.sh | 5 - gi/pipeline/scripts/xfeats.pl | 39 - gi/pipeline/valhalla.config | 9 - gi/posterior-regularisation/Corpus.java | 167 - gi/posterior-regularisation/Lexicon.java | 32 - .../PhraseContextModel.java | 466 --- gi/posterior-regularisation/README | 3 - gi/posterior-regularisation/alphabet.hh | 61 - gi/posterior-regularisation/canned.concordance | 4 - gi/posterior-regularisation/em.cc | 830 ---- gi/posterior-regularisation/invert.hh | 45 - gi/posterior-regularisation/linesearch.py | 58 - gi/posterior-regularisation/log_add.hh | 30 - gi/posterior-regularisation/prjava.jar | 1 - gi/posterior-regularisation/prjava/Makefile | 8 - gi/posterior-regularisation/prjava/build.xml | 38 - .../prjava/lib/commons-math-2.1.jar | Bin 832410 -> 0 bytes .../prjava/lib/jopt-simple-3.2.jar | Bin 53244 -> 0 bytes .../prjava/lib/trove-2.0.2.jar | Bin 737844 -> 0 bytes gi/posterior-regularisation/prjava/src/arr/F.java | 99 - .../prjava/src/data/Corpus.java | 233 -- .../prjava/src/hmm/HMM.java | 579 --- .../prjava/src/hmm/HMMObjective.java | 351 -- .../prjava/src/hmm/POS.java | 120 - .../prjava/src/io/FileUtil.java | 48 - .../prjava/src/io/SerializedObjects.java | 83 - .../examples/GeneralizedRosenbrock.java | 110 - .../prjava/src/optimization/examples/x2y2.java | 128 - .../optimization/examples/x2y2WithConstraints.java | 127 - .../AbstractGradientBaseMethod.java | 120 - .../gradientBasedMethods/ConjugateGradient.java | 92 - .../gradientBasedMethods/DebugHelpers.java | 65 - .../gradientBasedMethods/GradientDescent.java | 19 - .../optimization/gradientBasedMethods/LBFGS.java | 234 -- .../gradientBasedMethods/Objective.java | 87 - .../gradientBasedMethods/Optimizer.java | 19 - .../ProjectedAbstractGradientBaseMethod.java | 11 - .../ProjectedGradientDescent.java | 154 - .../gradientBasedMethods/ProjectedObjective.java | 29 - .../gradientBasedMethods/ProjectedOptimizer.java | 10 - .../gradientBasedMethods/stats/OptimizerStats.java | 86 - .../stats/ProjectedOptimizerStats.java | 70 - .../linesearch/ArmijoLineSearchMinimization.java | 102 - ...joLineSearchMinimizationAlongProjectionArc.java | 141 - .../DifferentiableLineSearchObjective.java | 185 - .../linesearch/GenericPickFirstStep.java | 20 - .../linesearch/InterpolationPickFirstStep.java | 25 - .../optimization/linesearch/LineSearchMethod.java | 14 - .../NonNewtonInterpolationPickFirstStep.java | 33 - ...ProjectedDifferentiableLineSearchObjective.java | 137 - .../linesearch/WolfRuleLineSearch.java | 300 -- .../optimization/linesearch/WolfeConditions.java | 45 - .../optimization/projections/BoundsProjection.java | 104 - .../src/optimization/projections/Projection.java | 72 - .../projections/SimplexProjection.java | 127 - .../stopCriteria/CompositeStopingCriteria.java | 33 - .../optimization/stopCriteria/GradientL2Norm.java | 30 - .../stopCriteria/NormalizedGradientL2Norm.java | 48 - .../NormalizedProjectedGradientL2Norm.java | 60 - .../stopCriteria/NormalizedValueDifference.java | 54 - .../stopCriteria/ProjectedGradientL2Norm.java | 51 - .../optimization/stopCriteria/StopingCriteria.java | 8 - .../optimization/stopCriteria/ValueDifference.java | 41 - .../src/optimization/util/Interpolation.java | 37 - .../prjava/src/optimization/util/Logger.java | 7 - .../prjava/src/optimization/util/MathUtils.java | 339 -- .../prjava/src/optimization/util/MatrixOutput.java | 28 - .../prjava/src/optimization/util/StaticTools.java | 180 - .../prjava/src/phrase/Agree.java | 204 - .../prjava/src/phrase/Agree2Sides.java | 197 - .../prjava/src/phrase/C2F.java | 216 -- .../prjava/src/phrase/Corpus.java | 288 -- .../prjava/src/phrase/Lexicon.java | 34 - .../prjava/src/phrase/PhraseCluster.java | 540 --- .../prjava/src/phrase/PhraseContextObjective.java | 436 --- .../prjava/src/phrase/PhraseCorpus.java | 193 - .../prjava/src/phrase/PhraseObjective.java | 224 -- .../prjava/src/phrase/Trainer.java | 257 -- .../prjava/src/phrase/VB.java | 419 -- .../prjava/src/test/CorpusTest.java | 60 - .../prjava/src/test/HMMModelStats.java | 105 - .../prjava/src/test/IntDoublePair.java | 23 - .../prjava/src/test/X2y2WithConstraints.java | 131 - .../prjava/src/util/Array.java | 41 - .../prjava/src/util/ArrayMath.java | 186 - .../prjava/src/util/DifferentiableObjective.java | 14 - .../prjava/src/util/DigammaFunction.java | 21 - .../prjava/src/util/FileSystem.java | 21 - .../prjava/src/util/InputOutput.java | 67 - .../prjava/src/util/LogSummer.java | 86 - .../prjava/src/util/MathUtil.java | 148 - .../prjava/src/util/Matrix.java | 16 - .../prjava/src/util/MemoryTracker.java | 47 - .../prjava/src/util/Pair.java | 31 - .../prjava/src/util/Printing.java | 158 - .../prjava/src/util/Sorters.java | 39 - .../prjava/train-PR-cluster.sh | 4 - gi/posterior-regularisation/projected_gradient.cc | 87 - gi/posterior-regularisation/simplex_pg.py | 55 - gi/posterior-regularisation/split-languages.py | 23 - gi/posterior-regularisation/train_pr_agree.py | 400 -- gi/posterior-regularisation/train_pr_global.py | 296 -- gi/posterior-regularisation/train_pr_parallel.py | 333 -- gi/pyp-topics/scripts/contexts2documents.py | 37 - gi/pyp-topics/scripts/extract_contexts.py | 144 - gi/pyp-topics/scripts/extract_contexts_test.py | 72 - gi/pyp-topics/scripts/extract_leaves.py | 49 - gi/pyp-topics/scripts/map-documents.py | 20 - gi/pyp-topics/scripts/map-terms.py | 20 - gi/pyp-topics/scripts/run.sh | 13 - gi/pyp-topics/scripts/score-mkcls.py | 61 - gi/pyp-topics/scripts/score-topics.py | 64 - gi/pyp-topics/scripts/spans2labels.py | 137 - gi/pyp-topics/scripts/tokens2classes.py | 27 - gi/pyp-topics/scripts/topics.py | 20 - gi/pyp-topics/src/Makefile.am | 16 - gi/pyp-topics/src/Makefile.mpi | 26 - gi/pyp-topics/src/clock_gettime_stub.c | 141 - gi/pyp-topics/src/contexts_corpus.cc | 164 - gi/pyp-topics/src/contexts_corpus.hh | 90 - gi/pyp-topics/src/contexts_lexer.h | 22 - gi/pyp-topics/src/contexts_lexer.l | 113 - gi/pyp-topics/src/corpus.cc | 104 - gi/pyp-topics/src/corpus.hh | 133 - gi/pyp-topics/src/gammadist.c | 247 -- gi/pyp-topics/src/gammadist.h | 72 - gi/pyp-topics/src/gzstream.cc | 165 - gi/pyp-topics/src/gzstream.hh | 121 - gi/pyp-topics/src/log_add.h | 30 - gi/pyp-topics/src/macros.Linux | 18 - gi/pyp-topics/src/makefile.darwin | 15 - gi/pyp-topics/src/makefile.depend | 4042 -------------------- gi/pyp-topics/src/mpi-corpus.hh | 69 - gi/pyp-topics/src/mpi-pyp-topics.cc | 466 --- gi/pyp-topics/src/mpi-pyp-topics.hh | 106 - gi/pyp-topics/src/mpi-pyp.hh | 447 --- gi/pyp-topics/src/mpi-train-contexts.cc | 201 - gi/pyp-topics/src/mt19937ar.c | 194 - gi/pyp-topics/src/mt19937ar.h | 44 - gi/pyp-topics/src/pyp-topics.cc | 499 --- gi/pyp-topics/src/pyp-topics.hh | 98 - gi/pyp-topics/src/pyp.hh | 566 --- gi/pyp-topics/src/slice-sampler.h | 192 - gi/pyp-topics/src/timing.h | 37 - gi/pyp-topics/src/train-contexts.cc | 174 - gi/pyp-topics/src/train.cc | 135 - gi/pyp-topics/src/utility.h | 962 ----- gi/pyp-topics/src/workers.hh | 275 -- gi/scripts/buck2utf8.pl | 87 - jam-files/LICENSE_1_0.txt | 23 - jam-files/boost-build/boost-build.jam | 8 - jam-files/boost-build/bootstrap.jam | 18 - jam-files/boost-build/build-system.jam | 1008 ----- jam-files/boost-build/build/__init__.py | 0 jam-files/boost-build/build/ac.jam | 198 - jam-files/boost-build/build/alias.jam | 73 - jam-files/boost-build/build/alias.py | 63 - jam-files/boost-build/build/build-request.jam | 322 -- jam-files/boost-build/build/build_request.py | 216 -- jam-files/boost-build/build/configure.jam | 237 -- jam-files/boost-build/build/configure.py | 164 - jam-files/boost-build/build/engine.py | 172 - jam-files/boost-build/build/errors.py | 127 - jam-files/boost-build/build/feature.jam | 1335 ------- jam-files/boost-build/build/feature.py | 905 ----- jam-files/boost-build/build/generators.jam | 1408 ------- jam-files/boost-build/build/generators.py | 1089 ------ jam-files/boost-build/build/modifiers.jam | 232 -- jam-files/boost-build/build/project.ann.py | 996 ----- jam-files/boost-build/build/project.jam | 1110 ------ jam-files/boost-build/build/project.py | 1120 ------ jam-files/boost-build/build/property-set.jam | 481 --- jam-files/boost-build/build/property.jam | 788 ---- jam-files/boost-build/build/property.py | 593 --- jam-files/boost-build/build/property_set.py | 449 --- jam-files/boost-build/build/readme.txt | 13 - jam-files/boost-build/build/scanner.jam | 153 - jam-files/boost-build/build/scanner.py | 158 - jam-files/boost-build/build/targets.jam | 1659 -------- jam-files/boost-build/build/targets.py | 1401 ------- jam-files/boost-build/build/toolset.jam | 502 --- jam-files/boost-build/build/toolset.py | 398 -- jam-files/boost-build/build/type.jam | 425 -- jam-files/boost-build/build/type.py | 313 -- jam-files/boost-build/build/version.jam | 161 - jam-files/boost-build/build/virtual-target.jam | 1317 ------- jam-files/boost-build/build/virtual_target.py | 1118 ------ jam-files/boost-build/kernel/boost-build.jam | 5 - jam-files/boost-build/kernel/bootstrap.jam | 263 -- jam-files/boost-build/kernel/bootstrap.py | 25 - jam-files/boost-build/kernel/class.jam | 420 -- jam-files/boost-build/kernel/errors.jam | 274 -- jam-files/boost-build/kernel/modules.jam | 354 -- jam-files/boost-build/options/help.jam | 212 - jam-files/boost-build/site-config.jam | 4 - jam-files/boost-build/tools/__init__.py | 0 jam-files/boost-build/tools/acc.jam | 118 - jam-files/boost-build/tools/auto-index.jam | 212 - jam-files/boost-build/tools/bison.jam | 32 - jam-files/boost-build/tools/boostbook-config.jam | 13 - jam-files/boost-build/tools/boostbook.jam | 727 ---- jam-files/boost-build/tools/borland.jam | 220 -- jam-files/boost-build/tools/builtin.jam | 960 ----- jam-files/boost-build/tools/builtin.py | 718 ---- jam-files/boost-build/tools/cast.jam | 91 - jam-files/boost-build/tools/cast.py | 69 - jam-files/boost-build/tools/clang-darwin.jam | 170 - jam-files/boost-build/tools/clang-linux.jam | 196 - jam-files/boost-build/tools/clang.jam | 27 - jam-files/boost-build/tools/common.jam | 994 ----- jam-files/boost-build/tools/common.py | 840 ---- jam-files/boost-build/tools/como-linux.jam | 103 - jam-files/boost-build/tools/como-win.jam | 117 - jam-files/boost-build/tools/como.jam | 29 - jam-files/boost-build/tools/convert.jam | 62 - jam-files/boost-build/tools/cw-config.jam | 34 - jam-files/boost-build/tools/cw.jam | 246 -- jam-files/boost-build/tools/darwin.jam | 568 --- jam-files/boost-build/tools/darwin.py | 57 - jam-files/boost-build/tools/dmc.jam | 134 - jam-files/boost-build/tools/docutils.jam | 84 - jam-files/boost-build/tools/doxproc.py | 859 ----- jam-files/boost-build/tools/doxygen-config.jam | 11 - jam-files/boost-build/tools/doxygen.jam | 776 ---- .../tools/doxygen/windows-paths-check.doxyfile | 3 - .../tools/doxygen/windows-paths-check.hpp | 0 jam-files/boost-build/tools/fop.jam | 69 - jam-files/boost-build/tools/fortran.jam | 55 - jam-files/boost-build/tools/gcc.jam | 1185 ------ jam-files/boost-build/tools/gcc.py | 796 ---- jam-files/boost-build/tools/generate.jam | 108 - jam-files/boost-build/tools/gettext.jam | 230 -- jam-files/boost-build/tools/gfortran.jam | 39 - jam-files/boost-build/tools/hp_cxx.jam | 181 - jam-files/boost-build/tools/hpfortran.jam | 35 - jam-files/boost-build/tools/ifort.jam | 44 - jam-files/boost-build/tools/intel-darwin.jam | 220 -- jam-files/boost-build/tools/intel-linux.jam | 250 -- jam-files/boost-build/tools/intel-win.jam | 184 - jam-files/boost-build/tools/intel.jam | 34 - jam-files/boost-build/tools/lex.jam | 33 - jam-files/boost-build/tools/make.jam | 72 - jam-files/boost-build/tools/make.py | 59 - jam-files/boost-build/tools/mc.jam | 44 - jam-files/boost-build/tools/message.jam | 55 - jam-files/boost-build/tools/message.py | 46 - jam-files/boost-build/tools/midl.jam | 142 - jam-files/boost-build/tools/mipspro.jam | 145 - jam-files/boost-build/tools/mpi.jam | 583 --- jam-files/boost-build/tools/msvc-config.jam | 12 - jam-files/boost-build/tools/msvc.jam | 1392 ------- jam-files/boost-build/tools/notfile.jam | 74 - jam-files/boost-build/tools/notfile.py | 51 - jam-files/boost-build/tools/package.jam | 165 - jam-files/boost-build/tools/package.py | 168 - jam-files/boost-build/tools/pathscale.jam | 168 - jam-files/boost-build/tools/pch.jam | 95 - jam-files/boost-build/tools/pch.py | 83 - jam-files/boost-build/tools/pgi.jam | 147 - jam-files/boost-build/tools/python-config.jam | 27 - jam-files/boost-build/tools/python.jam | 1267 ------ jam-files/boost-build/tools/qcc.jam | 236 -- jam-files/boost-build/tools/qt.jam | 17 - jam-files/boost-build/tools/qt3.jam | 209 - jam-files/boost-build/tools/qt4.jam | 724 ---- jam-files/boost-build/tools/quickbook-config.jam | 44 - jam-files/boost-build/tools/quickbook.jam | 361 -- jam-files/boost-build/tools/rc.jam | 156 - jam-files/boost-build/tools/rc.py | 189 - jam-files/boost-build/tools/stage.jam | 524 --- jam-files/boost-build/tools/stage.py | 350 -- jam-files/boost-build/tools/stlport.jam | 303 -- jam-files/boost-build/tools/sun.jam | 142 - jam-files/boost-build/tools/symlink.jam | 140 - jam-files/boost-build/tools/symlink.py | 112 - jam-files/boost-build/tools/testing-aux.jam | 210 - jam-files/boost-build/tools/testing.jam | 581 --- jam-files/boost-build/tools/testing.py | 342 -- jam-files/boost-build/tools/types/__init__.py | 18 - jam-files/boost-build/tools/types/asm.jam | 4 - jam-files/boost-build/tools/types/asm.py | 13 - jam-files/boost-build/tools/types/cpp.jam | 86 - jam-files/boost-build/tools/types/cpp.py | 10 - jam-files/boost-build/tools/types/exe.jam | 9 - jam-files/boost-build/tools/types/exe.py | 11 - jam-files/boost-build/tools/types/html.jam | 4 - jam-files/boost-build/tools/types/html.py | 10 - jam-files/boost-build/tools/types/lib.jam | 74 - jam-files/boost-build/tools/types/lib.py | 77 - jam-files/boost-build/tools/types/obj.jam | 9 - jam-files/boost-build/tools/types/obj.py | 11 - jam-files/boost-build/tools/types/objc.jam | 26 - jam-files/boost-build/tools/types/preprocessed.jam | 9 - jam-files/boost-build/tools/types/qt.jam | 10 - jam-files/boost-build/tools/types/register.jam | 39 - jam-files/boost-build/tools/types/rsp.jam | 4 - jam-files/boost-build/tools/types/rsp.py | 10 - jam-files/boost-build/tools/unix.jam | 224 -- jam-files/boost-build/tools/unix.py | 150 - jam-files/boost-build/tools/vacpp.jam | 150 - jam-files/boost-build/tools/whale.jam | 116 - jam-files/boost-build/tools/xlf.jam | 39 - jam-files/boost-build/tools/xsltproc-config.jam | 37 - jam-files/boost-build/tools/xsltproc.jam | 194 - jam-files/boost-build/tools/xsltproc/included.xsl | 11 - jam-files/boost-build/tools/xsltproc/test.xml | 2 - jam-files/boost-build/tools/xsltproc/test.xsl | 12 - jam-files/boost-build/tools/zlib.jam | 92 - jam-files/boost-build/user-config.jam | 92 - jam-files/boost-build/util/__init__.py | 136 - jam-files/boost-build/util/assert.jam | 336 -- jam-files/boost-build/util/container.jam | 339 -- jam-files/boost-build/util/doc.jam | 997 ----- jam-files/boost-build/util/indirect.jam | 115 - jam-files/boost-build/util/indirect.py | 15 - jam-files/boost-build/util/logger.py | 46 - jam-files/boost-build/util/numbers.jam | 218 -- jam-files/boost-build/util/option.jam | 109 - jam-files/boost-build/util/option.py | 35 - jam-files/boost-build/util/order.jam | 169 - jam-files/boost-build/util/order.py | 121 - jam-files/boost-build/util/os.jam | 171 - jam-files/boost-build/util/os_j.py | 19 - jam-files/boost-build/util/path.jam | 934 ----- jam-files/boost-build/util/path.py | 904 ----- jam-files/boost-build/util/print.jam | 488 --- jam-files/boost-build/util/regex.jam | 193 - jam-files/boost-build/util/regex.py | 25 - jam-files/boost-build/util/sequence.jam | 335 -- jam-files/boost-build/util/sequence.py | 50 - jam-files/boost-build/util/set.jam | 93 - jam-files/boost-build/util/set.py | 42 - jam-files/boost-build/util/string.jam | 189 - jam-files/boost-build/util/utility.jam | 235 -- jam-files/boost-build/util/utility.py | 155 - jam-files/engine/Jambase | 2473 ------------ jam-files/engine/boost-jam.spec | 64 - jam-files/engine/boost-no-inspect | 1 - jam-files/engine/build.bat | 532 --- jam-files/engine/build.jam | 1070 ------ jam-files/engine/build.sh | 303 -- jam-files/engine/build_vms.com | 105 - jam-files/engine/builtins.c | 2310 ----------- jam-files/engine/builtins.h | 69 - jam-files/engine/bump_version.py | 80 - jam-files/engine/class.c | 141 - jam-files/engine/class.h | 13 - jam-files/engine/command.c | 100 - jam-files/engine/command.h | 61 - jam-files/engine/compile.c | 1424 ------- jam-files/engine/compile.h | 82 - jam-files/engine/debian/changelog | 72 - jam-files/engine/debian/control | 16 - jam-files/engine/debian/copyright | 25 - jam-files/engine/debian/jam.man.sgml | 236 -- jam-files/engine/debian/rules | 73 - jam-files/engine/debug.c | 132 - jam-files/engine/debug.h | 54 - jam-files/engine/execcmd.h | 45 - jam-files/engine/execmac.c | 69 - jam-files/engine/execnt.c | 1296 ------- jam-files/engine/execunix.c | 569 --- jam-files/engine/execvms.c | 161 - jam-files/engine/expand.c | 733 ---- jam-files/engine/expand.h | 14 - jam-files/engine/filemac.c | 175 - jam-files/engine/filent.c | 387 -- jam-files/engine/fileos2.c | 138 - jam-files/engine/filesys.c | 83 - jam-files/engine/filesys.h | 60 - jam-files/engine/fileunix.c | 501 --- jam-files/engine/filevms.c | 327 -- jam-files/engine/frames.c | 22 - jam-files/engine/frames.h | 37 - jam-files/engine/glob.c | 152 - jam-files/engine/hash.c | 459 --- jam-files/engine/hash.h | 25 - jam-files/engine/hcache.c | 434 --- jam-files/engine/hcache.h | 18 - jam-files/engine/hdrmacro.c | 137 - jam-files/engine/hdrmacro.h | 14 - jam-files/engine/headers.c | 203 - jam-files/engine/headers.h | 16 - jam-files/engine/jam.c | 632 --- jam-files/engine/jam.h | 579 --- jam-files/engine/jambase.c | 1691 -------- jam-files/engine/jambase.h | 15 - jam-files/engine/jamgram.c | 1830 --------- jam-files/engine/jamgram.h | 140 - jam-files/engine/jamgram.y | 371 -- jam-files/engine/jamgram.yy | 329 -- jam-files/engine/jamgramtab.h | 44 - jam-files/engine/lists.c | 339 -- jam-files/engine/lists.h | 108 - jam-files/engine/make.c | 814 ---- jam-files/engine/make.h | 41 - jam-files/engine/make1.c | 1145 ------ jam-files/engine/md5.c | 381 -- jam-files/engine/md5.h | 91 - jam-files/engine/mem.c | 75 - jam-files/engine/mem.h | 134 - jam-files/engine/mkjambase.c | 123 - jam-files/engine/modules.c | 168 - jam-files/engine/modules.h | 37 - jam-files/engine/modules/order.c | 144 - jam-files/engine/modules/path.c | 32 - jam-files/engine/modules/property-set.c | 110 - jam-files/engine/modules/readme.txt | 3 - jam-files/engine/modules/regex.c | 96 - jam-files/engine/modules/sequence.c | 42 - jam-files/engine/modules/set.c | 41 - jam-files/engine/native.c | 36 - jam-files/engine/native.h | 34 - jam-files/engine/newstr.c | 174 - jam-files/engine/newstr.h | 14 - jam-files/engine/option.c | 94 - jam-files/engine/option.h | 23 - jam-files/engine/output.c | 125 - jam-files/engine/output.h | 29 - jam-files/engine/parse.c | 132 - jam-files/engine/parse.h | 59 - jam-files/engine/patchlevel.h | 17 - jam-files/engine/pathmac.c | 252 -- jam-files/engine/pathsys.h | 91 - jam-files/engine/pathunix.c | 457 --- jam-files/engine/pathvms.c | 406 -- jam-files/engine/pwd.c | 66 - jam-files/engine/pwd.h | 10 - jam-files/engine/regexp.c | 1328 ------- jam-files/engine/regexp.h | 32 - jam-files/engine/rules.c | 810 ---- jam-files/engine/rules.h | 280 -- jam-files/engine/scan.c | 418 -- jam-files/engine/scan.h | 56 - jam-files/engine/search.c | 223 -- jam-files/engine/search.h | 11 - jam-files/engine/strings.c | 201 - jam-files/engine/strings.h | 34 - jam-files/engine/subst.c | 94 - jam-files/engine/timestamp.c | 226 -- jam-files/engine/timestamp.h | 12 - jam-files/engine/variable.c | 631 --- jam-files/engine/variable.h | 35 - jam-files/engine/w32_getreg.c | 207 - jam-files/engine/yyacc.c | 268 -- jam-files/sanity.jam | 277 -- klm/lm/Jamfile | 14 - klm/util/Jamfile | 10 - mira/Jamfile | 1 - mteval/Jamfile | 8 - mteval/ns_docscorer.cc | 4 +- mteval/ns_docscorer.h | 2 +- phrasinator/Jamfile | 4 - phrasinator/Makefile.am | 14 - phrasinator/README | 16 - phrasinator/gibbs_train_plm.cc | 309 -- phrasinator/gibbs_train_plm.notables.cc | 335 -- phrasinator/train-phrasinator.pl | 89 - rst_parser/Makefile.am | 20 - rst_parser/arc_factored.cc | 151 - rst_parser/arc_factored.h | 124 - rst_parser/arc_factored_marginals.cc | 58 - rst_parser/arc_ff.cc | 183 - rst_parser/arc_ff.h | 28 - rst_parser/dep_training.cc | 76 - rst_parser/dep_training.h | 19 - rst_parser/global_ff.cc | 44 - rst_parser/global_ff.h | 18 - rst_parser/mst_train.cc | 228 -- rst_parser/picojson.h | 979 ----- rst_parser/random_tree.cc | 36 - rst_parser/rst.cc | 82 - rst_parser/rst.h | 21 - rst_parser/rst_parse.cc | 111 - rst_parser/rst_train.cc | 144 - training/Jamfile | 25 - training/liblbfgs/Jamfile | 5 - utils/Jamfile | 32 - utils/Makefile.am | 7 +- utils/ccrp.h | 270 -- utils/ccrp_nt.h | 164 - utils/ccrp_onetable.h | 253 -- utils/crp_table_manager.h | 114 - utils/crp_test.cc | 91 - utils/fast_sparse_vector.h | 2 +- utils/gamma_poisson.h | 33 - utils/mfcr.h | 370 -- utils/mfcr_test.cc | 72 - utils/sampler.h | 2 +- utils/slice_sampler.h | 191 - utils/small_vector.h | 2 +- utils/stringlib.h | 2 +- utils/unigram_pyp_lm.cc | 214 -- 586 files changed, 13 insertions(+), 125976 deletions(-) delete mode 100644 Jamroot delete mode 100755 bjam delete mode 100644 decoder/Jamfile delete mode 100644 dpmert/Jamfile delete mode 100644 gi/clda/src/Makefile.am delete mode 100644 gi/clda/src/ccrp.h delete mode 100644 gi/clda/src/clda.cc delete mode 100644 gi/clda/src/crp.h delete mode 100644 gi/clda/src/slice_sampler.h delete mode 100644 gi/clda/src/timer.h delete mode 100644 gi/evaluation/conditional_entropy.py delete mode 100644 gi/evaluation/confusion_matrix.py delete mode 100644 gi/evaluation/entropy.py delete mode 100644 gi/evaluation/extract_ccg_labels.py delete mode 100644 gi/evaluation/tree.py delete mode 100644 gi/markov_al/Makefile.am delete mode 100644 gi/markov_al/README delete mode 100644 gi/markov_al/ml.cc delete mode 100755 gi/morf-segmentation/filter_docs.pl delete mode 100644 gi/morf-segmentation/invalid_vocab.patterns delete mode 100755 gi/morf-segmentation/linestripper.py delete mode 100755 gi/morf-segmentation/morf-pipeline.pl delete mode 100755 gi/morf-segmentation/morfsegment.py delete mode 100755 gi/morf-segmentation/morftrain.sh delete mode 100755 gi/morf-segmentation/vocabextractor.sh delete mode 100644 gi/pf/Makefile.am delete mode 100644 gi/pf/README delete mode 100644 gi/pf/align-lexonly-pyp.cc delete mode 100644 gi/pf/align-tl.cc delete mode 100644 gi/pf/backward.cc delete mode 100644 gi/pf/backward.h delete mode 100644 gi/pf/base_distributions.cc delete mode 100644 gi/pf/base_distributions.h delete mode 100644 gi/pf/bayes_lattice_score.cc delete mode 100644 gi/pf/brat.cc delete mode 100644 gi/pf/cbgi.cc delete mode 100644 gi/pf/cfg_wfst_composer.cc delete mode 100644 gi/pf/cfg_wfst_composer.h delete mode 100644 gi/pf/conditional_pseg.h delete mode 100644 gi/pf/condnaive.cc delete mode 100644 gi/pf/corpus.cc delete mode 100644 gi/pf/corpus.h delete mode 100644 gi/pf/dpnaive.cc delete mode 100755 gi/pf/guess-translits.pl delete mode 100644 gi/pf/hpyp_tm.cc delete mode 100644 gi/pf/hpyp_tm.h delete mode 100644 gi/pf/itg.cc delete mode 100644 gi/pf/learn_cfg.cc delete mode 100755 gi/pf/make-freq-bins.pl delete mode 100644 gi/pf/mh_test.cc delete mode 100644 gi/pf/monotonic_pseg.h delete mode 100644 gi/pf/ngram_base.cc delete mode 100644 gi/pf/ngram_base.h delete mode 100644 gi/pf/nuisance_test.cc delete mode 100644 gi/pf/os_phrase.h delete mode 100644 gi/pf/pf.h delete mode 100644 gi/pf/pf_test.cc delete mode 100644 gi/pf/pfbrat.cc delete mode 100644 gi/pf/pfdist.cc delete mode 100644 gi/pf/pfdist.new.cc delete mode 100644 gi/pf/pfnaive.cc delete mode 100644 gi/pf/poisson_uniform_word_model.h delete mode 100644 gi/pf/pyp_lm.cc delete mode 100644 gi/pf/pyp_tm.cc delete mode 100644 gi/pf/pyp_tm.h delete mode 100644 gi/pf/pyp_word_model.h delete mode 100644 gi/pf/quasi_model2.h delete mode 100644 gi/pf/reachability.cc delete mode 100644 gi/pf/reachability.h delete mode 100644 gi/pf/tied_resampler.h delete mode 100644 gi/pf/tpf.cc delete mode 100644 gi/pf/transliterations.cc delete mode 100644 gi/pf/transliterations.h delete mode 100644 gi/pf/unigrams.cc delete mode 100644 gi/pf/unigrams.h delete mode 100644 gi/pipeline/OLD.clsp.config delete mode 100755 gi/pipeline/OLD.evaluation-pipeline.pl delete mode 100644 gi/pipeline/backoff-pipe.pl delete mode 100644 gi/pipeline/blacklight.config delete mode 100644 gi/pipeline/clsp.config delete mode 100755 gi/pipeline/evaluation-pipeline.pl delete mode 100755 gi/pipeline/local-gi-pipeline.pl delete mode 100644 gi/pipeline/lticluster.config delete mode 100755 gi/pipeline/scripts/filter-by-f.pl delete mode 100755 gi/pipeline/scripts/patch-corpus.pl delete mode 100755 gi/pipeline/scripts/refilter.pl delete mode 100755 gi/pipeline/scripts/rekey.pl delete mode 100755 gi/pipeline/scripts/remove-tags-from-contexts.pl delete mode 100755 gi/pipeline/scripts/remove-tags-from-corpus.pl delete mode 100755 gi/pipeline/scripts/sort-by-key.sh delete mode 100755 gi/pipeline/scripts/xfeats.pl delete mode 100644 gi/pipeline/valhalla.config delete mode 100644 gi/posterior-regularisation/Corpus.java delete mode 100644 gi/posterior-regularisation/Lexicon.java delete mode 100644 gi/posterior-regularisation/PhraseContextModel.java delete mode 100644 gi/posterior-regularisation/README delete mode 100644 gi/posterior-regularisation/alphabet.hh delete mode 100644 gi/posterior-regularisation/canned.concordance delete mode 100644 gi/posterior-regularisation/em.cc delete mode 100644 gi/posterior-regularisation/invert.hh delete mode 100644 gi/posterior-regularisation/linesearch.py delete mode 100644 gi/posterior-regularisation/log_add.hh delete mode 120000 gi/posterior-regularisation/prjava.jar delete mode 100755 gi/posterior-regularisation/prjava/Makefile delete mode 100644 gi/posterior-regularisation/prjava/build.xml delete mode 100644 gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar delete mode 100644 gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar delete mode 100644 gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar delete mode 100644 gi/posterior-regularisation/prjava/src/arr/F.java delete mode 100644 gi/posterior-regularisation/prjava/src/data/Corpus.java delete mode 100644 gi/posterior-regularisation/prjava/src/hmm/HMM.java delete mode 100644 gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/hmm/POS.java delete mode 100644 gi/posterior-regularisation/prjava/src/io/FileUtil.java delete mode 100644 gi/posterior-regularisation/prjava/src/io/SerializedObjects.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/examples/GeneralizedRosenbrock.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/examples/x2y2.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/examples/x2y2WithConstraints.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/AbstractGradientBaseMethod.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ConjugateGradient.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/DebugHelpers.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/GradientDescent.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/LBFGS.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Objective.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Optimizer.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedAbstractGradientBaseMethod.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedGradientDescent.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedOptimizer.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/OptimizerStats.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/ProjectedOptimizerStats.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimization.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimizationAlongProjectionArc.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/DifferentiableLineSearchObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/GenericPickFirstStep.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/InterpolationPickFirstStep.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/LineSearchMethod.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/NonNewtonInterpolationPickFirstStep.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/ProjectedDifferentiableLineSearchObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfRuleLineSearch.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfeConditions.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/projections/BoundsProjection.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/projections/Projection.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/projections/SimplexProjection.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/CompositeStopingCriteria.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/GradientL2Norm.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedGradientL2Norm.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedProjectedGradientL2Norm.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedValueDifference.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ProjectedGradientL2Norm.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/StopingCriteria.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ValueDifference.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/util/Interpolation.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/util/Logger.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/util/MathUtils.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/util/MatrixOutput.java delete mode 100644 gi/posterior-regularisation/prjava/src/optimization/util/StaticTools.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/Agree.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/Agree2Sides.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/C2F.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/Corpus.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/Lexicon.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/PhraseContextObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/Trainer.java delete mode 100644 gi/posterior-regularisation/prjava/src/phrase/VB.java delete mode 100644 gi/posterior-regularisation/prjava/src/test/CorpusTest.java delete mode 100644 gi/posterior-regularisation/prjava/src/test/HMMModelStats.java delete mode 100644 gi/posterior-regularisation/prjava/src/test/IntDoublePair.java delete mode 100644 gi/posterior-regularisation/prjava/src/test/X2y2WithConstraints.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/Array.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/ArrayMath.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/DifferentiableObjective.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/DigammaFunction.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/FileSystem.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/InputOutput.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/LogSummer.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/MathUtil.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/Matrix.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/MemoryTracker.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/Pair.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/Printing.java delete mode 100644 gi/posterior-regularisation/prjava/src/util/Sorters.java delete mode 100755 gi/posterior-regularisation/prjava/train-PR-cluster.sh delete mode 100644 gi/posterior-regularisation/projected_gradient.cc delete mode 100644 gi/posterior-regularisation/simplex_pg.py delete mode 100755 gi/posterior-regularisation/split-languages.py delete mode 100644 gi/posterior-regularisation/train_pr_agree.py delete mode 100644 gi/posterior-regularisation/train_pr_global.py delete mode 100644 gi/posterior-regularisation/train_pr_parallel.py delete mode 100755 gi/pyp-topics/scripts/contexts2documents.py delete mode 100755 gi/pyp-topics/scripts/extract_contexts.py delete mode 100755 gi/pyp-topics/scripts/extract_contexts_test.py delete mode 100755 gi/pyp-topics/scripts/extract_leaves.py delete mode 100755 gi/pyp-topics/scripts/map-documents.py delete mode 100755 gi/pyp-topics/scripts/map-terms.py delete mode 100644 gi/pyp-topics/scripts/run.sh delete mode 100755 gi/pyp-topics/scripts/score-mkcls.py delete mode 100755 gi/pyp-topics/scripts/score-topics.py delete mode 100755 gi/pyp-topics/scripts/spans2labels.py delete mode 100755 gi/pyp-topics/scripts/tokens2classes.py delete mode 100755 gi/pyp-topics/scripts/topics.py delete mode 100644 gi/pyp-topics/src/Makefile.am delete mode 100644 gi/pyp-topics/src/Makefile.mpi delete mode 100644 gi/pyp-topics/src/clock_gettime_stub.c delete mode 100644 gi/pyp-topics/src/contexts_corpus.cc delete mode 100644 gi/pyp-topics/src/contexts_corpus.hh delete mode 100644 gi/pyp-topics/src/contexts_lexer.h delete mode 100644 gi/pyp-topics/src/contexts_lexer.l delete mode 100644 gi/pyp-topics/src/corpus.cc delete mode 100644 gi/pyp-topics/src/corpus.hh delete mode 100644 gi/pyp-topics/src/gammadist.c delete mode 100644 gi/pyp-topics/src/gammadist.h delete mode 100644 gi/pyp-topics/src/gzstream.cc delete mode 100644 gi/pyp-topics/src/gzstream.hh delete mode 100644 gi/pyp-topics/src/log_add.h delete mode 100644 gi/pyp-topics/src/macros.Linux delete mode 100644 gi/pyp-topics/src/makefile.darwin delete mode 100644 gi/pyp-topics/src/makefile.depend delete mode 100644 gi/pyp-topics/src/mpi-corpus.hh delete mode 100644 gi/pyp-topics/src/mpi-pyp-topics.cc delete mode 100644 gi/pyp-topics/src/mpi-pyp-topics.hh delete mode 100644 gi/pyp-topics/src/mpi-pyp.hh delete mode 100644 gi/pyp-topics/src/mpi-train-contexts.cc delete mode 100644 gi/pyp-topics/src/mt19937ar.c delete mode 100644 gi/pyp-topics/src/mt19937ar.h delete mode 100644 gi/pyp-topics/src/pyp-topics.cc delete mode 100644 gi/pyp-topics/src/pyp-topics.hh delete mode 100644 gi/pyp-topics/src/pyp.hh delete mode 100644 gi/pyp-topics/src/slice-sampler.h delete mode 100644 gi/pyp-topics/src/timing.h delete mode 100644 gi/pyp-topics/src/train-contexts.cc delete mode 100644 gi/pyp-topics/src/train.cc delete mode 100644 gi/pyp-topics/src/utility.h delete mode 100644 gi/pyp-topics/src/workers.hh delete mode 100755 gi/scripts/buck2utf8.pl delete mode 100644 jam-files/LICENSE_1_0.txt delete mode 100644 jam-files/boost-build/boost-build.jam delete mode 100644 jam-files/boost-build/bootstrap.jam delete mode 100644 jam-files/boost-build/build-system.jam delete mode 100644 jam-files/boost-build/build/__init__.py delete mode 100644 jam-files/boost-build/build/ac.jam delete mode 100644 jam-files/boost-build/build/alias.jam delete mode 100644 jam-files/boost-build/build/alias.py delete mode 100644 jam-files/boost-build/build/build-request.jam delete mode 100644 jam-files/boost-build/build/build_request.py delete mode 100644 jam-files/boost-build/build/configure.jam delete mode 100644 jam-files/boost-build/build/configure.py delete mode 100644 jam-files/boost-build/build/engine.py delete mode 100644 jam-files/boost-build/build/errors.py delete mode 100644 jam-files/boost-build/build/feature.jam delete mode 100644 jam-files/boost-build/build/feature.py delete mode 100644 jam-files/boost-build/build/generators.jam delete mode 100644 jam-files/boost-build/build/generators.py delete mode 100644 jam-files/boost-build/build/modifiers.jam delete mode 100644 jam-files/boost-build/build/project.ann.py delete mode 100644 jam-files/boost-build/build/project.jam delete mode 100644 jam-files/boost-build/build/project.py delete mode 100644 jam-files/boost-build/build/property-set.jam delete mode 100644 jam-files/boost-build/build/property.jam delete mode 100644 jam-files/boost-build/build/property.py delete mode 100644 jam-files/boost-build/build/property_set.py delete mode 100644 jam-files/boost-build/build/readme.txt delete mode 100644 jam-files/boost-build/build/scanner.jam delete mode 100644 jam-files/boost-build/build/scanner.py delete mode 100644 jam-files/boost-build/build/targets.jam delete mode 100644 jam-files/boost-build/build/targets.py delete mode 100644 jam-files/boost-build/build/toolset.jam delete mode 100644 jam-files/boost-build/build/toolset.py delete mode 100644 jam-files/boost-build/build/type.jam delete mode 100644 jam-files/boost-build/build/type.py delete mode 100644 jam-files/boost-build/build/version.jam delete mode 100644 jam-files/boost-build/build/virtual-target.jam delete mode 100644 jam-files/boost-build/build/virtual_target.py delete mode 100644 jam-files/boost-build/kernel/boost-build.jam delete mode 100644 jam-files/boost-build/kernel/bootstrap.jam delete mode 100644 jam-files/boost-build/kernel/bootstrap.py delete mode 100644 jam-files/boost-build/kernel/class.jam delete mode 100644 jam-files/boost-build/kernel/errors.jam delete mode 100644 jam-files/boost-build/kernel/modules.jam delete mode 100644 jam-files/boost-build/options/help.jam delete mode 100644 jam-files/boost-build/site-config.jam delete mode 100644 jam-files/boost-build/tools/__init__.py delete mode 100644 jam-files/boost-build/tools/acc.jam delete mode 100644 jam-files/boost-build/tools/auto-index.jam delete mode 100644 jam-files/boost-build/tools/bison.jam delete mode 100644 jam-files/boost-build/tools/boostbook-config.jam delete mode 100644 jam-files/boost-build/tools/boostbook.jam delete mode 100644 jam-files/boost-build/tools/borland.jam delete mode 100644 jam-files/boost-build/tools/builtin.jam delete mode 100644 jam-files/boost-build/tools/builtin.py delete mode 100644 jam-files/boost-build/tools/cast.jam delete mode 100644 jam-files/boost-build/tools/cast.py delete mode 100644 jam-files/boost-build/tools/clang-darwin.jam delete mode 100644 jam-files/boost-build/tools/clang-linux.jam delete mode 100644 jam-files/boost-build/tools/clang.jam delete mode 100644 jam-files/boost-build/tools/common.jam delete mode 100644 jam-files/boost-build/tools/common.py delete mode 100644 jam-files/boost-build/tools/como-linux.jam delete mode 100644 jam-files/boost-build/tools/como-win.jam delete mode 100644 jam-files/boost-build/tools/como.jam delete mode 100644 jam-files/boost-build/tools/convert.jam delete mode 100644 jam-files/boost-build/tools/cw-config.jam delete mode 100644 jam-files/boost-build/tools/cw.jam delete mode 100644 jam-files/boost-build/tools/darwin.jam delete mode 100644 jam-files/boost-build/tools/darwin.py delete mode 100644 jam-files/boost-build/tools/dmc.jam delete mode 100644 jam-files/boost-build/tools/docutils.jam delete mode 100644 jam-files/boost-build/tools/doxproc.py delete mode 100644 jam-files/boost-build/tools/doxygen-config.jam delete mode 100644 jam-files/boost-build/tools/doxygen.jam delete mode 100644 jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile delete mode 100644 jam-files/boost-build/tools/doxygen/windows-paths-check.hpp delete mode 100644 jam-files/boost-build/tools/fop.jam delete mode 100644 jam-files/boost-build/tools/fortran.jam delete mode 100644 jam-files/boost-build/tools/gcc.jam delete mode 100644 jam-files/boost-build/tools/gcc.py delete mode 100644 jam-files/boost-build/tools/generate.jam delete mode 100644 jam-files/boost-build/tools/gettext.jam delete mode 100644 jam-files/boost-build/tools/gfortran.jam delete mode 100644 jam-files/boost-build/tools/hp_cxx.jam delete mode 100644 jam-files/boost-build/tools/hpfortran.jam delete mode 100644 jam-files/boost-build/tools/ifort.jam delete mode 100644 jam-files/boost-build/tools/intel-darwin.jam delete mode 100644 jam-files/boost-build/tools/intel-linux.jam delete mode 100644 jam-files/boost-build/tools/intel-win.jam delete mode 100644 jam-files/boost-build/tools/intel.jam delete mode 100644 jam-files/boost-build/tools/lex.jam delete mode 100644 jam-files/boost-build/tools/make.jam delete mode 100644 jam-files/boost-build/tools/make.py delete mode 100644 jam-files/boost-build/tools/mc.jam delete mode 100644 jam-files/boost-build/tools/message.jam delete mode 100644 jam-files/boost-build/tools/message.py delete mode 100644 jam-files/boost-build/tools/midl.jam delete mode 100644 jam-files/boost-build/tools/mipspro.jam delete mode 100644 jam-files/boost-build/tools/mpi.jam delete mode 100644 jam-files/boost-build/tools/msvc-config.jam delete mode 100644 jam-files/boost-build/tools/msvc.jam delete mode 100644 jam-files/boost-build/tools/notfile.jam delete mode 100644 jam-files/boost-build/tools/notfile.py delete mode 100644 jam-files/boost-build/tools/package.jam delete mode 100644 jam-files/boost-build/tools/package.py delete mode 100644 jam-files/boost-build/tools/pathscale.jam delete mode 100644 jam-files/boost-build/tools/pch.jam delete mode 100644 jam-files/boost-build/tools/pch.py delete mode 100644 jam-files/boost-build/tools/pgi.jam delete mode 100644 jam-files/boost-build/tools/python-config.jam delete mode 100644 jam-files/boost-build/tools/python.jam delete mode 100644 jam-files/boost-build/tools/qcc.jam delete mode 100644 jam-files/boost-build/tools/qt.jam delete mode 100644 jam-files/boost-build/tools/qt3.jam delete mode 100644 jam-files/boost-build/tools/qt4.jam delete mode 100644 jam-files/boost-build/tools/quickbook-config.jam delete mode 100644 jam-files/boost-build/tools/quickbook.jam delete mode 100644 jam-files/boost-build/tools/rc.jam delete mode 100644 jam-files/boost-build/tools/rc.py delete mode 100644 jam-files/boost-build/tools/stage.jam delete mode 100644 jam-files/boost-build/tools/stage.py delete mode 100644 jam-files/boost-build/tools/stlport.jam delete mode 100644 jam-files/boost-build/tools/sun.jam delete mode 100644 jam-files/boost-build/tools/symlink.jam delete mode 100644 jam-files/boost-build/tools/symlink.py delete mode 100644 jam-files/boost-build/tools/testing-aux.jam delete mode 100644 jam-files/boost-build/tools/testing.jam delete mode 100644 jam-files/boost-build/tools/testing.py delete mode 100644 jam-files/boost-build/tools/types/__init__.py delete mode 100644 jam-files/boost-build/tools/types/asm.jam delete mode 100644 jam-files/boost-build/tools/types/asm.py delete mode 100644 jam-files/boost-build/tools/types/cpp.jam delete mode 100644 jam-files/boost-build/tools/types/cpp.py delete mode 100644 jam-files/boost-build/tools/types/exe.jam delete mode 100644 jam-files/boost-build/tools/types/exe.py delete mode 100644 jam-files/boost-build/tools/types/html.jam delete mode 100644 jam-files/boost-build/tools/types/html.py delete mode 100644 jam-files/boost-build/tools/types/lib.jam delete mode 100644 jam-files/boost-build/tools/types/lib.py delete mode 100644 jam-files/boost-build/tools/types/obj.jam delete mode 100644 jam-files/boost-build/tools/types/obj.py delete mode 100644 jam-files/boost-build/tools/types/objc.jam delete mode 100644 jam-files/boost-build/tools/types/preprocessed.jam delete mode 100644 jam-files/boost-build/tools/types/qt.jam delete mode 100644 jam-files/boost-build/tools/types/register.jam delete mode 100644 jam-files/boost-build/tools/types/rsp.jam delete mode 100644 jam-files/boost-build/tools/types/rsp.py delete mode 100644 jam-files/boost-build/tools/unix.jam delete mode 100644 jam-files/boost-build/tools/unix.py delete mode 100644 jam-files/boost-build/tools/vacpp.jam delete mode 100644 jam-files/boost-build/tools/whale.jam delete mode 100644 jam-files/boost-build/tools/xlf.jam delete mode 100644 jam-files/boost-build/tools/xsltproc-config.jam delete mode 100644 jam-files/boost-build/tools/xsltproc.jam delete mode 100644 jam-files/boost-build/tools/xsltproc/included.xsl delete mode 100644 jam-files/boost-build/tools/xsltproc/test.xml delete mode 100644 jam-files/boost-build/tools/xsltproc/test.xsl delete mode 100644 jam-files/boost-build/tools/zlib.jam delete mode 100644 jam-files/boost-build/user-config.jam delete mode 100644 jam-files/boost-build/util/__init__.py delete mode 100644 jam-files/boost-build/util/assert.jam delete mode 100644 jam-files/boost-build/util/container.jam delete mode 100644 jam-files/boost-build/util/doc.jam delete mode 100644 jam-files/boost-build/util/indirect.jam delete mode 100644 jam-files/boost-build/util/indirect.py delete mode 100644 jam-files/boost-build/util/logger.py delete mode 100644 jam-files/boost-build/util/numbers.jam delete mode 100644 jam-files/boost-build/util/option.jam delete mode 100644 jam-files/boost-build/util/option.py delete mode 100644 jam-files/boost-build/util/order.jam delete mode 100644 jam-files/boost-build/util/order.py delete mode 100644 jam-files/boost-build/util/os.jam delete mode 100644 jam-files/boost-build/util/os_j.py delete mode 100644 jam-files/boost-build/util/path.jam delete mode 100644 jam-files/boost-build/util/path.py delete mode 100644 jam-files/boost-build/util/print.jam delete mode 100644 jam-files/boost-build/util/regex.jam delete mode 100644 jam-files/boost-build/util/regex.py delete mode 100644 jam-files/boost-build/util/sequence.jam delete mode 100644 jam-files/boost-build/util/sequence.py delete mode 100644 jam-files/boost-build/util/set.jam delete mode 100644 jam-files/boost-build/util/set.py delete mode 100644 jam-files/boost-build/util/string.jam delete mode 100644 jam-files/boost-build/util/utility.jam delete mode 100644 jam-files/boost-build/util/utility.py delete mode 100644 jam-files/engine/Jambase delete mode 100644 jam-files/engine/boost-jam.spec delete mode 100644 jam-files/engine/boost-no-inspect delete mode 100644 jam-files/engine/build.bat delete mode 100644 jam-files/engine/build.jam delete mode 100755 jam-files/engine/build.sh delete mode 100644 jam-files/engine/build_vms.com delete mode 100644 jam-files/engine/builtins.c delete mode 100644 jam-files/engine/builtins.h delete mode 100644 jam-files/engine/bump_version.py delete mode 100644 jam-files/engine/class.c delete mode 100644 jam-files/engine/class.h delete mode 100644 jam-files/engine/command.c delete mode 100644 jam-files/engine/command.h delete mode 100644 jam-files/engine/compile.c delete mode 100644 jam-files/engine/compile.h delete mode 100644 jam-files/engine/debian/changelog delete mode 100644 jam-files/engine/debian/control delete mode 100644 jam-files/engine/debian/copyright delete mode 100644 jam-files/engine/debian/jam.man.sgml delete mode 100755 jam-files/engine/debian/rules delete mode 100644 jam-files/engine/debug.c delete mode 100644 jam-files/engine/debug.h delete mode 100644 jam-files/engine/execcmd.h delete mode 100644 jam-files/engine/execmac.c delete mode 100644 jam-files/engine/execnt.c delete mode 100644 jam-files/engine/execunix.c delete mode 100644 jam-files/engine/execvms.c delete mode 100644 jam-files/engine/expand.c delete mode 100644 jam-files/engine/expand.h delete mode 100644 jam-files/engine/filemac.c delete mode 100644 jam-files/engine/filent.c delete mode 100644 jam-files/engine/fileos2.c delete mode 100644 jam-files/engine/filesys.c delete mode 100644 jam-files/engine/filesys.h delete mode 100644 jam-files/engine/fileunix.c delete mode 100644 jam-files/engine/filevms.c delete mode 100644 jam-files/engine/frames.c delete mode 100644 jam-files/engine/frames.h delete mode 100644 jam-files/engine/glob.c delete mode 100644 jam-files/engine/hash.c delete mode 100644 jam-files/engine/hash.h delete mode 100644 jam-files/engine/hcache.c delete mode 100644 jam-files/engine/hcache.h delete mode 100644 jam-files/engine/hdrmacro.c delete mode 100644 jam-files/engine/hdrmacro.h delete mode 100644 jam-files/engine/headers.c delete mode 100644 jam-files/engine/headers.h delete mode 100644 jam-files/engine/jam.c delete mode 100644 jam-files/engine/jam.h delete mode 100644 jam-files/engine/jambase.c delete mode 100644 jam-files/engine/jambase.h delete mode 100644 jam-files/engine/jamgram.c delete mode 100644 jam-files/engine/jamgram.h delete mode 100644 jam-files/engine/jamgram.y delete mode 100644 jam-files/engine/jamgram.yy delete mode 100644 jam-files/engine/jamgramtab.h delete mode 100644 jam-files/engine/lists.c delete mode 100644 jam-files/engine/lists.h delete mode 100644 jam-files/engine/make.c delete mode 100644 jam-files/engine/make.h delete mode 100644 jam-files/engine/make1.c delete mode 100644 jam-files/engine/md5.c delete mode 100644 jam-files/engine/md5.h delete mode 100644 jam-files/engine/mem.c delete mode 100644 jam-files/engine/mem.h delete mode 100644 jam-files/engine/mkjambase.c delete mode 100644 jam-files/engine/modules.c delete mode 100644 jam-files/engine/modules.h delete mode 100644 jam-files/engine/modules/order.c delete mode 100644 jam-files/engine/modules/path.c delete mode 100644 jam-files/engine/modules/property-set.c delete mode 100644 jam-files/engine/modules/readme.txt delete mode 100644 jam-files/engine/modules/regex.c delete mode 100644 jam-files/engine/modules/sequence.c delete mode 100644 jam-files/engine/modules/set.c delete mode 100644 jam-files/engine/native.c delete mode 100644 jam-files/engine/native.h delete mode 100644 jam-files/engine/newstr.c delete mode 100644 jam-files/engine/newstr.h delete mode 100644 jam-files/engine/option.c delete mode 100644 jam-files/engine/option.h delete mode 100644 jam-files/engine/output.c delete mode 100644 jam-files/engine/output.h delete mode 100644 jam-files/engine/parse.c delete mode 100644 jam-files/engine/parse.h delete mode 100644 jam-files/engine/patchlevel.h delete mode 100644 jam-files/engine/pathmac.c delete mode 100644 jam-files/engine/pathsys.h delete mode 100644 jam-files/engine/pathunix.c delete mode 100644 jam-files/engine/pathvms.c delete mode 100644 jam-files/engine/pwd.c delete mode 100644 jam-files/engine/pwd.h delete mode 100644 jam-files/engine/regexp.c delete mode 100644 jam-files/engine/regexp.h delete mode 100644 jam-files/engine/rules.c delete mode 100644 jam-files/engine/rules.h delete mode 100644 jam-files/engine/scan.c delete mode 100644 jam-files/engine/scan.h delete mode 100644 jam-files/engine/search.c delete mode 100644 jam-files/engine/search.h delete mode 100644 jam-files/engine/strings.c delete mode 100644 jam-files/engine/strings.h delete mode 100644 jam-files/engine/subst.c delete mode 100644 jam-files/engine/timestamp.c delete mode 100644 jam-files/engine/timestamp.h delete mode 100644 jam-files/engine/variable.c delete mode 100644 jam-files/engine/variable.h delete mode 100644 jam-files/engine/w32_getreg.c delete mode 100644 jam-files/engine/yyacc.c delete mode 100644 jam-files/sanity.jam delete mode 100644 klm/lm/Jamfile delete mode 100644 klm/util/Jamfile delete mode 100644 mira/Jamfile delete mode 100644 mteval/Jamfile delete mode 100644 phrasinator/Jamfile delete mode 100644 phrasinator/Makefile.am delete mode 100644 phrasinator/README delete mode 100644 phrasinator/gibbs_train_plm.cc delete mode 100644 phrasinator/gibbs_train_plm.notables.cc delete mode 100755 phrasinator/train-phrasinator.pl delete mode 100644 rst_parser/Makefile.am delete mode 100644 rst_parser/arc_factored.cc delete mode 100644 rst_parser/arc_factored.h delete mode 100644 rst_parser/arc_factored_marginals.cc delete mode 100644 rst_parser/arc_ff.cc delete mode 100644 rst_parser/arc_ff.h delete mode 100644 rst_parser/dep_training.cc delete mode 100644 rst_parser/dep_training.h delete mode 100644 rst_parser/global_ff.cc delete mode 100644 rst_parser/global_ff.h delete mode 100644 rst_parser/mst_train.cc delete mode 100644 rst_parser/picojson.h delete mode 100644 rst_parser/random_tree.cc delete mode 100644 rst_parser/rst.cc delete mode 100644 rst_parser/rst.h delete mode 100644 rst_parser/rst_parse.cc delete mode 100644 rst_parser/rst_train.cc delete mode 100644 training/Jamfile delete mode 100644 training/liblbfgs/Jamfile delete mode 100644 utils/Jamfile delete mode 100644 utils/ccrp.h delete mode 100644 utils/ccrp_nt.h delete mode 100644 utils/ccrp_onetable.h delete mode 100644 utils/crp_table_manager.h delete mode 100644 utils/crp_test.cc delete mode 100644 utils/gamma_poisson.h delete mode 100644 utils/mfcr.h delete mode 100644 utils/mfcr_test.cc delete mode 100644 utils/slice_sampler.h delete mode 100644 utils/unigram_pyp_lm.cc (limited to 'decoder') diff --git a/Jamroot b/Jamroot deleted file mode 100644 index ef426146..00000000 --- a/Jamroot +++ /dev/null @@ -1,45 +0,0 @@ -#cdec compilation with bjam -# -#--with-boost=/usr/include -#--with-google-hash=/usr/include so that $with-google-hash/google/dense_hash_map exists -# -#-a forces the build to run from scratch -#-jN parallelizes just like make -# -#Respects CXXFLAGS, CFLAGS, and LDFLAGS environment variables. - -path-constant TOP : . ; -include $(TOP)/jam-files/sanity.jam ; -boost 104400 ; -external-lib z ; - -with-google-hash = [ option.get "with-google-hash" ] ; -if [ test_header sparsehash/dense_hash_map ] || $(with-google-hash) { - requirements += HAVE_SPARSEHASH $(with-google-hash) ; -} - -if [ test_header cmph.h ] || $(with-cmph) { - requirements += HAVE_CMPH $(with-cmph) ; -} - -if [ test_header boost/serialization/map.hpp ] && [ test_library boost_serialization ] { - requirements += HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP ; -} - -project : requirements $(requirements) darwin:static ; -project : default-build on release ; - -install-bin-libs dpmert//programs utils//programs mteval//programs klm/lm//programs training//liblbfgs decoder//cdec phrasinator//programs mira//kbest_mira ; - -install perl-scripts : dpmert//dpmert.pl : $(bindir) ; - -build-projects mteval decoder dpmert klm/lm training/liblbfgs ; - -#Compile everything ending with _test.cc into a test and run it. -rule all_tests ( targets * : dependencies : properties * ) { - targets ?= [ glob *_test.cc ] ; - for t in $(targets) { - local base = [ MATCH "^(.*).cc$" : $(t) ] ; - unit-test $(base) : $(t) $(dependencies) ..//boost_unit_test_framework : $(properties) ; - } -} diff --git a/Makefile.am b/Makefile.am index 24aafd63..c0826532 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,7 +7,6 @@ SUBDIRS = \ klm/util \ klm/lm \ decoder \ - phrasinator \ training \ training/liblbfgs \ mira \ @@ -15,10 +14,7 @@ SUBDIRS = \ dpmert \ pro-train \ rampion \ - minrisk \ - gi/pf \ - gi/markov_al \ - rst_parser + minrisk #gi/pyp-topics/src gi/clda/src gi/posterior-regularisation/prjava diff --git a/bjam b/bjam deleted file mode 100755 index d1ac8a55..00000000 --- a/bjam +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e -if - bjam="$(which bjam 2>/dev/null)" && #exists - [ ${#bjam} != 0 ] && #paranoia about which printing nothing then returning true - ! grep UFIHGUFIHBDJKNCFZXAEVA "${bjam}" /dev/null && #bjam in path isn't this script - "${bjam}" --help >/dev/null 2>/dev/null && #bjam in path isn't broken (i.e. has boost-build) - "${bjam}" --version |grep "Boost.Build 201" >/dev/null 2>/dev/null #It's recent enough. -then - #Delegate to system bjam - exec "${bjam}" "$@" -fi - -top="$(dirname "$0")" -if [ ! -x "$top"/jam-files/bjam ]; then - pushd "$top/jam-files/engine" - ./build.sh - cp -f bin.*/bjam ../bjam - popd -fi - -export BOOST_BUILD_PATH="$top"/jam-files/boost-build -exec "$top"/jam-files/bjam "$@" diff --git a/configure.ac b/configure.ac index ea9e84fb..07ef9fe1 100644 --- a/configure.ac +++ b/configure.ac @@ -114,7 +114,6 @@ AC_CONFIG_FILES([Makefile]) AC_CONFIG_FILES([utils/Makefile]) AC_CONFIG_FILES([mteval/Makefile]) AC_CONFIG_FILES([decoder/Makefile]) -AC_CONFIG_FILES([phrasinator/Makefile]) AC_CONFIG_FILES([training/Makefile]) AC_CONFIG_FILES([training/liblbfgs/Makefile]) AC_CONFIG_FILES([dpmert/Makefile]) @@ -125,10 +124,6 @@ AC_CONFIG_FILES([klm/util/Makefile]) AC_CONFIG_FILES([klm/lm/Makefile]) AC_CONFIG_FILES([mira/Makefile]) AC_CONFIG_FILES([dtrain/Makefile]) -AC_CONFIG_FILES([gi/pyp-topics/src/Makefile]) -AC_CONFIG_FILES([gi/clda/src/Makefile]) -AC_CONFIG_FILES([gi/pf/Makefile]) -AC_CONFIG_FILES([gi/markov_al/Makefile]) AC_CONFIG_FILES([rst_parser/Makefile]) AC_CONFIG_FILES([python/setup.py]) diff --git a/decoder/Jamfile b/decoder/Jamfile deleted file mode 100644 index da02d063..00000000 --- a/decoder/Jamfile +++ /dev/null @@ -1,81 +0,0 @@ -import testing ; -import lex ; -import option ; - -if [ option.get "with-glc" ] { - glc = ff_glc.cc string_util.cc feature-factory.cc ; -} - -lib decoder : - forest_writer.cc - maxtrans_blunsom.cc - cdec_ff.cc - cfg.cc - dwarf.cc - ff_dwarf.cc - rule_lexer.ll - fst_translator.cc - csplit.cc - translator.cc - scfg_translator.cc - hg.cc - hg_io.cc - decoder.cc - hg_intersect.cc - hg_sampler.cc - factored_lexicon_helper.cc - viterbi.cc - lattice.cc - aligner.cc - apply_models.cc - earley_composer.cc - phrasetable_fst.cc - trule.cc - ff.cc - ff_rules.cc - ff_wordset.cc - ff_context.cc - ff_charset.cc - ff_lm.cc - ff_klm.cc - ff_ngrams.cc - ff_spans.cc - ff_ruleshape.cc - ff_wordalign.cc - ff_csplit.cc - ff_tagger.cc - ff_source_syntax.cc - ff_bleu.cc - ff_factory.cc - lexalign.cc - lextrans.cc - tagger.cc - bottom_up_parser.cc - phrasebased_translator.cc - JSON_parser.c - json_parse.cc - grammar.cc - rescore_translator.cc - hg_remove_eps.cc - hg_union.cc - $(glc) - ..//utils - ..//mteval - ../klm/lm//kenlm - ..//boost_program_options - : . - : : - ..//utils - ..//mteval - ../klm/lm//kenlm - ..//boost_program_options - . - ; - -exe cdec : cdec.cc decoder ..//utils ..//mteval ../klm/lm//kenlm ..//boost_program_options ; - -all_tests [ glob *_test.cc : cfg_test.cc ] : decoder : $(TOP)/decoder/test_data ; - -install legacy : cdec - : $(TOP)/cdec EXE on shared:$(TOP)/cdec shared:LIB ; - diff --git a/decoder/decoder.h b/decoder/decoder.h index bef2ff5e..79c7a602 100644 --- a/decoder/decoder.h +++ b/decoder/decoder.h @@ -24,7 +24,7 @@ private: #endif class SentenceMetadata; -struct Hypergraph; +class Hypergraph; struct DecoderImpl; struct DecoderObserver { diff --git a/decoder/hg.h b/decoder/hg.h index 591e98ce..6d67f2fa 100644 --- a/decoder/hg.h +++ b/decoder/hg.h @@ -503,9 +503,9 @@ public: template void visit_edges_topo(V &v) { - for (int i = 0; i < nodes_.size(); ++i) { + for (unsigned i = 0; i < nodes_.size(); ++i) { EdgesVector const& in=nodes_[i].in_edges_; - for (int j=0;j. - : : - ..//utils - ..//mteval - ../klm/lm//kenlm - ..//boost_program_options - . - ; - -all_tests [ glob *_test.cc ] : dpmert : $(TOP)/dpmert/test_data ; - -exe sentserver : sentserver.c : multi ; -exe sentclient : sentclient.c ; -exe mr_dpmert_generate_mapper_input : mr_dpmert_generate_mapper_input.cc dpmert ..//boost_program_options ; -exe mr_dpmert_map : mr_dpmert_map.cc dpmert ..//boost_program_options ; -exe mr_dpmert_reduce : mr_dpmert_reduce.cc dpmert ..//boost_program_options ; - -alias programs : sentserver sentclient mr_dpmert_generate_mapper_input mr_dpmert_map mr_dpmert_reduce ; diff --git a/gi/clda/src/Makefile.am b/gi/clda/src/Makefile.am deleted file mode 100644 index cdca1f97..00000000 --- a/gi/clda/src/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -bin_PROGRAMS = clda - -clda_SOURCES = clda.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -AM_LDFLAGS = $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/clda/src/ccrp.h b/gi/clda/src/ccrp.h deleted file mode 100644 index a7c2825c..00000000 --- a/gi/clda/src/ccrp.h +++ /dev/null @@ -1,291 +0,0 @@ -#ifndef _CCRP_H_ -#define _CCRP_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "sampler.h" -#include "slice_sampler.h" - -// Chinese restaurant process (Pitman-Yor parameters) with table tracking. - -template > -class CCRP { - public: - CCRP(double disc, double conc) : - num_tables_(), - num_customers_(), - discount_(disc), - concentration_(conc), - discount_prior_alpha_(std::numeric_limits::quiet_NaN()), - discount_prior_beta_(std::numeric_limits::quiet_NaN()), - concentration_prior_shape_(std::numeric_limits::quiet_NaN()), - concentration_prior_rate_(std::numeric_limits::quiet_NaN()) {} - - CCRP(double d_alpha, double d_beta, double c_shape, double c_rate, double d = 0.1, double c = 10.0) : - num_tables_(), - num_customers_(), - discount_(d), - concentration_(c), - discount_prior_alpha_(d_alpha), - discount_prior_beta_(d_beta), - concentration_prior_shape_(c_shape), - concentration_prior_rate_(c_rate) {} - - double discount() const { return discount_; } - double concentration() const { return concentration_; } - - bool has_discount_prior() const { - return !std::isnan(discount_prior_alpha_); - } - - bool has_concentration_prior() const { - return !std::isnan(concentration_prior_shape_); - } - - void clear() { - num_tables_ = 0; - num_customers_ = 0; - dish_locs_.clear(); - } - - unsigned num_tables(const Dish& dish) const { - const typename std::tr1::unordered_map::const_iterator it = dish_locs_.find(dish); - if (it == dish_locs_.end()) return 0; - return it->second.table_counts_.size(); - } - - unsigned num_customers() const { - return num_customers_; - } - - unsigned num_customers(const Dish& dish) const { - const typename std::tr1::unordered_map::const_iterator it = dish_locs_.find(dish); - if (it == dish_locs_.end()) return 0; - return it->total_dish_count_; - } - - // returns +1 or 0 indicating whether a new table was opened - int increment(const Dish& dish, const double& p0, MT19937* rng) { - DishLocations& loc = dish_locs_[dish]; - bool share_table = false; - if (loc.total_dish_count_) { - const double p_empty = (concentration_ + num_tables_ * discount_) * p0; - const double p_share = (loc.total_dish_count_ - loc.table_counts_.size() * discount_); - share_table = rng->SelectSample(p_empty, p_share); - } - if (share_table) { - double r = rng->next() * (loc.total_dish_count_ - loc.table_counts_.size() * discount_); - for (typename std::list::iterator ti = loc.table_counts_.begin(); - ti != loc.table_counts_.end(); ++ti) { - r -= (*ti - discount_); - if (r <= 0.0) { - ++(*ti); - break; - } - } - if (r > 0.0) { - std::cerr << "Serious error: r=" << r << std::endl; - Print(&std::cerr); - assert(r <= 0.0); - } - } else { - loc.table_counts_.push_back(1u); - ++num_tables_; - } - ++loc.total_dish_count_; - ++num_customers_; - return (share_table ? 0 : 1); - } - - // returns -1 or 0, indicating whether a table was closed - int decrement(const Dish& dish, MT19937* rng) { - DishLocations& loc = dish_locs_[dish]; - assert(loc.total_dish_count_); - if (loc.total_dish_count_ == 1) { - dish_locs_.erase(dish); - --num_tables_; - --num_customers_; - return -1; - } else { - int delta = 0; - // sample customer to remove UNIFORMLY. that is, do NOT use the discount - // here. if you do, it will introduce (unwanted) bias! - double r = rng->next() * loc.total_dish_count_; - --loc.total_dish_count_; - for (typename std::list::iterator ti = loc.table_counts_.begin(); - ti != loc.table_counts_.end(); ++ti) { - r -= *ti; - if (r <= 0.0) { - if ((--(*ti)) == 0) { - --num_tables_; - delta = -1; - loc.table_counts_.erase(ti); - } - break; - } - } - if (r > 0.0) { - std::cerr << "Serious error: r=" << r << std::endl; - Print(&std::cerr); - assert(r <= 0.0); - } - --num_customers_; - return delta; - } - } - - double prob(const Dish& dish, const double& p0) const { - const typename std::tr1::unordered_map::const_iterator it = dish_locs_.find(dish); - const double r = num_tables_ * discount_ + concentration_; - if (it == dish_locs_.end()) { - return r * p0 / (num_customers_ + concentration_); - } else { - return (it->second.total_dish_count_ - discount_ * it->second.table_counts_.size() + r * p0) / - (num_customers_ + concentration_); - } - } - - double log_crp_prob() const { - return log_crp_prob(discount_, concentration_); - } - - static double log_beta_density(const double& x, const double& alpha, const double& beta) { - assert(x > 0.0); - assert(x < 1.0); - assert(alpha > 0.0); - assert(beta > 0.0); - const double lp = (alpha-1)*log(x)+(beta-1)*log(1-x)+lgamma(alpha+beta)-lgamma(alpha)-lgamma(beta); - return lp; - } - - static double log_gamma_density(const double& x, const double& shape, const double& rate) { - assert(x >= 0.0); - assert(shape > 0.0); - assert(rate > 0.0); - const double lp = (shape-1)*log(x) - shape*log(rate) - x/rate - lgamma(shape); - return lp; - } - - // taken from http://en.wikipedia.org/wiki/Chinese_restaurant_process - // does not include P_0's - double log_crp_prob(const double& discount, const double& concentration) const { - double lp = 0.0; - if (has_discount_prior()) - lp = log_beta_density(discount, discount_prior_alpha_, discount_prior_beta_); - if (has_concentration_prior()) - lp += log_gamma_density(concentration, concentration_prior_shape_, concentration_prior_rate_); - assert(lp <= 0.0); - if (num_customers_) { - if (discount > 0.0) { - const double r = lgamma(1.0 - discount); - lp += lgamma(concentration) - lgamma(concentration + num_customers_) - + num_tables_ * log(discount) + lgamma(concentration / discount + num_tables_) - - lgamma(concentration / discount); - assert(std::isfinite(lp)); - for (typename std::tr1::unordered_map::const_iterator it = dish_locs_.begin(); - it != dish_locs_.end(); ++it) { - const DishLocations& cur = it->second; - for (std::list::const_iterator ti = cur.table_counts_.begin(); ti != cur.table_counts_.end(); ++ti) { - lp += lgamma(*ti - discount) - r; - } - } - } else { - assert(!"not implemented yet"); - } - } - assert(std::isfinite(lp)); - return lp; - } - - void resample_hyperparameters(MT19937* rng) { - assert(has_discount_prior() || has_concentration_prior()); - DiscountResampler dr(*this); - ConcentrationResampler cr(*this); - const int niterations = 10; - double gamma_upper = std::numeric_limits::infinity(); - for (int iter = 0; iter < 5; ++iter) { - if (has_concentration_prior()) { - concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, - gamma_upper, 0.0, niterations, 100*niterations); - } - if (has_discount_prior()) { - discount_ = slice_sampler1d(dr, discount_, *rng, std::numeric_limits::min(), - 1.0, 0.0, niterations, 100*niterations); - } - } - concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, - gamma_upper, 0.0, niterations, 100*niterations); - } - - struct DiscountResampler { - DiscountResampler(const CCRP& crp) : crp_(crp) {} - const CCRP& crp_; - double operator()(const double& proposed_discount) const { - return crp_.log_crp_prob(proposed_discount, crp_.concentration_); - } - }; - - struct ConcentrationResampler { - ConcentrationResampler(const CCRP& crp) : crp_(crp) {} - const CCRP& crp_; - double operator()(const double& proposed_concentration) const { - return crp_.log_crp_prob(crp_.discount_, proposed_concentration); - } - }; - - struct DishLocations { - DishLocations() : total_dish_count_() {} - unsigned total_dish_count_; // customers at all tables with this dish - std::list table_counts_; // list<> gives O(1) deletion and insertion, which we want - // .size() is the number of tables for this dish - }; - - void Print(std::ostream* out) const { - for (typename std::tr1::unordered_map::const_iterator it = dish_locs_.begin(); - it != dish_locs_.end(); ++it) { - (*out) << it->first << " (" << it->second.total_dish_count_ << " on " << it->second.table_counts_.size() << " tables): "; - for (typename std::list::const_iterator i = it->second.table_counts_.begin(); - i != it->second.table_counts_.end(); ++i) { - (*out) << " " << *i; - } - (*out) << std::endl; - } - } - - typedef typename std::tr1::unordered_map::const_iterator const_iterator; - const_iterator begin() const { - return dish_locs_.begin(); - } - const_iterator end() const { - return dish_locs_.end(); - } - - unsigned num_tables_; - unsigned num_customers_; - std::tr1::unordered_map dish_locs_; - - double discount_; - double concentration_; - - // optional beta prior on discount_ (NaN if no prior) - double discount_prior_alpha_; - double discount_prior_beta_; - - // optional gamma prior on concentration_ (NaN if no prior) - double concentration_prior_shape_; - double concentration_prior_rate_; -}; - -template -std::ostream& operator<<(std::ostream& o, const CCRP& c) { - c.Print(&o); - return o; -} - -#endif diff --git a/gi/clda/src/clda.cc b/gi/clda/src/clda.cc deleted file mode 100644 index f548997f..00000000 --- a/gi/clda/src/clda.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include -#include - -#include "timer.h" -#include "crp.h" -#include "ccrp.h" -#include "sampler.h" -#include "tdict.h" -const size_t MAX_DOC_LEN_CHARS = 10000000; - -using namespace std; - -void ShowTopWordsForTopic(const map& counts) { - multimap ms; - for (map::const_iterator it = counts.begin(); it != counts.end(); ++it) - ms.insert(make_pair(it->second, it->first)); - int cc = 0; - for (multimap::reverse_iterator it = ms.rbegin(); it != ms.rend(); ++it) { - cerr << it->first << ':' << TD::Convert(it->second) << " "; - ++cc; - if (cc==20) break; - } - cerr << endl; -} - -int main(int argc, char** argv) { - if (argc != 3) { - cerr << "Usage: " << argv[0] << " num-classes num-samples\n"; - return 1; - } - const int num_classes = atoi(argv[1]); - const int num_iterations = atoi(argv[2]); - const int burnin_size = num_iterations * 0.9; - if (num_classes < 2) { - cerr << "Must request more than 1 class\n"; - return 1; - } - if (num_iterations < 5) { - cerr << "Must request more than 5 iterations\n"; - return 1; - } - cerr << "CLASSES: " << num_classes << endl; - char* buf = new char[MAX_DOC_LEN_CHARS]; - vector > wji; // w[j][i] - observed word i of doc j - vector > zji; // z[j][i] - topic assignment for word i of doc j - cerr << "READING DOCUMENTS\n"; - while(cin) { - cin.getline(buf, MAX_DOC_LEN_CHARS); - if (buf[0] == 0) continue; - wji.push_back(vector()); - TD::ConvertSentence(buf, &wji.back()); - } - cerr << "READ " << wji.size() << " DOCUMENTS\n"; - MT19937 rng; - cerr << "INITIALIZING RANDOM TOPIC ASSIGNMENTS\n"; - zji.resize(wji.size()); - double disc = 0.1; - double beta = 10.0; - double alpha = 50.0; - const double uniform_topic = 1.0 / num_classes; - const double uniform_word = 1.0 / TD::NumWords(); - vector > dr(zji.size(), CCRP(1,1,1,1,disc, beta)); // dr[i] describes the probability of using a topic in document i - vector > wr(num_classes, CCRP(1,1,1,1,disc, alpha)); // wr[k] describes the probability of generating a word in topic k - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector& zj = zji[j]; - const vector& wj = wji[j]; - zj.resize(num_words); - for (int i = 0; i < num_words; ++i) { - int random_topic = rng.next() * num_classes; - if (random_topic == num_classes) { --random_topic; } - zj[i] = random_topic; - const int word = wj[i]; - dr[j].increment(random_topic, uniform_topic, &rng); - wr[random_topic].increment(word, uniform_word, &rng); - } - } - cerr << "SAMPLING\n"; - vector > t2w(num_classes); - Timer timer; - SampleSet ss; - ss.resize(num_classes); - double total_time = 0; - for (int iter = 0; iter < num_iterations; ++iter) { - cerr << '.'; - if (iter && iter % 10 == 0) { - total_time += timer.Elapsed(); - timer.Reset(); - double llh = 0; -#if 1 - for (int j = 0; j < dr.size(); ++j) - dr[j].resample_hyperparameters(&rng); - for (int j = 0; j < wr.size(); ++j) - wr[j].resample_hyperparameters(&rng); -#endif - - for (int j = 0; j < dr.size(); ++j) - llh += dr[j].log_crp_prob(); - for (int j = 0; j < wr.size(); ++j) - llh += wr[j].log_crp_prob(); - cerr << " [LLH=" << llh << " I=" << iter << "]\n"; - } - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector& zj = zji[j]; - const vector& wj = wji[j]; - for (int i = 0; i < num_words; ++i) { - const int word = wj[i]; - const int cur_topic = zj[i]; - dr[j].decrement(cur_topic, &rng); - wr[cur_topic].decrement(word, &rng); - - for (int k = 0; k < num_classes; ++k) { - ss[k]= dr[j].prob(k, uniform_topic) * wr[k].prob(word, uniform_word); - } - const int new_topic = rng.SelectSample(ss); - dr[j].increment(new_topic, uniform_topic, &rng); - wr[new_topic].increment(word, uniform_word, &rng); - zj[i] = new_topic; - if (iter > burnin_size) { - ++t2w[cur_topic][word]; - } - } - } - } - for (int i = 0; i < num_classes; ++i) { - cerr << "---------------------------------\n"; - cerr << " final PYP(" << wr[i].discount() << "," << wr[i].concentration() << ")\n"; - ShowTopWordsForTopic(t2w[i]); - } - cerr << "-------------\n"; -#if 0 - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector& zj = zji[j]; - const vector& wj = wji[j]; - zj.resize(num_words); - for (int i = 0; i < num_words; ++i) { - cerr << TD::Convert(wji[j][i]) << '(' << zj[i] << ") "; - } - cerr << endl; - } -#endif - return 0; -} - diff --git a/gi/clda/src/crp.h b/gi/clda/src/crp.h deleted file mode 100644 index 9d35857e..00000000 --- a/gi/clda/src/crp.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef _CRP_H_ -#define _CRP_H_ - -// shamelessly adapted from code by Phil Blunsom and Trevor Cohn - -#include -#include - -#include "prob.h" - -template > -class CRP { - public: - CRP(double alpha) : alpha_(alpha), palpha_(alpha), total_customers_() {} - void increment(const DishType& dish); - void decrement(const DishType& dish); - void erase(const DishType& dish) { - counts_.erase(dish); - } - inline int count(const DishType& dish) const { - const typename MapType::const_iterator i = counts_.find(dish); - if (i == counts_.end()) return 0; else return i->second; - } - inline prob_t prob(const DishType& dish, const prob_t& p0) const { - return (prob_t(count(dish)) + palpha_ * p0) / prob_t(total_customers_ + alpha_); - } - private: - typedef std::tr1::unordered_map MapType; - MapType counts_; - const double alpha_; - const prob_t palpha_; - int total_customers_; -}; - -template -void CRP::increment(const Dish& dish) { - ++counts_[dish]; - ++total_customers_; -} - -template -void CRP::decrement(const Dish& dish) { - typename MapType::iterator i = counts_.find(dish); - assert(i != counts_.end()); - if (--i->second == 0) - counts_.erase(i); - --total_customers_; -} - -#endif diff --git a/gi/clda/src/slice_sampler.h b/gi/clda/src/slice_sampler.h deleted file mode 100644 index aa48a169..00000000 --- a/gi/clda/src/slice_sampler.h +++ /dev/null @@ -1,191 +0,0 @@ -//! slice-sampler.h is an MCMC slice sampler -//! -//! Mark Johnson, 1st August 2008 - -#ifndef SLICE_SAMPLER_H -#define SLICE_SAMPLER_H - -#include -#include -#include -#include -#include - -//! slice_sampler_rfc_type{} returns the value of a user-specified -//! function if the argument is within range, or - infinity otherwise -// -template -struct slice_sampler_rfc_type { - F min_x, max_x; - const Fn& f; - U max_nfeval, nfeval; - slice_sampler_rfc_type(F min_x, F max_x, const Fn& f, U max_nfeval) - : min_x(min_x), max_x(max_x), f(f), max_nfeval(max_nfeval), nfeval(0) { } - - F operator() (F x) { - if (min_x < x && x < max_x) { - assert(++nfeval <= max_nfeval); - F fx = f(x); - assert(std::isfinite(fx)); - return fx; - } - return -std::numeric_limits::infinity(); - } -}; // slice_sampler_rfc_type{} - -//! slice_sampler1d() implements the univariate "range doubling" slice sampler -//! described in Neal (2003) "Slice Sampling", The Annals of Statistics 31(3), 705-767. -// -template -F slice_sampler1d(const LogF& logF0, //!< log of function to sample - F x, //!< starting point - Uniform01& u01, //!< uniform [0,1) random number generator - F min_x = -std::numeric_limits::infinity(), //!< minimum value of support - F max_x = std::numeric_limits::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - slice_sampler_rfc_type logF(min_x, max_x, logF0, max_nfeval); - - assert(std::isfinite(x)); - - if (w <= 0.0) { // set w to a default width - if (min_x > -std::numeric_limits::infinity() && max_x < std::numeric_limits::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, (F) 0.1); - } - assert(std::isfinite(w)); - - F logFx = logF(x); - for (U sample = 0; sample < nsamples; ++sample) { - F logY = logFx + log(u01()+1e-100); //! slice logFx at this value - assert(std::isfinite(logY)); - - F xl = x - w*u01(); //! lower bound on slice interval - F logFxl = logF(xl); - F xr = xl + w; //! upper bound on slice interval - F logFxr = logF(xr); - - while (logY < logFxl || logY < logFxr) // doubling procedure - if (u01() < 0.5) - logFxl = logF(xl -= xr - xl); - else - logFxr = logF(xr += xr - xl); - - F xl1 = xl; - F xr1 = xr; - while (true) { // shrinking procedure - F x1 = xl1 + u01()*(xr1 - xl1); - if (logY < logF(x1)) { - F xl2 = xl; // acceptance procedure - F xr2 = xr; - bool d = false; - while (xr2 - xl2 > 1.1*w) { - F xm = (xl2 + xr2)/2; - if ((x < xm && x1 >= xm) || (x >= xm && x1 < xm)) - d = true; - if (x1 < xm) - xr2 = xm; - else - xl2 = xm; - if (d && logY >= logF(xl2) && logY >= logF(xr2)) - goto unacceptable; - } - x = x1; - goto acceptable; - } - goto acceptable; - unacceptable: - if (x1 < x) // rest of shrinking procedure - xl1 = x1; - else - xr1 = x1; - } - acceptable: - w = (4*w + (xr1 - xl1))/5; // update width estimate - } - return x; -} - -/* -//! slice_sampler1d() implements a 1-d MCMC slice sampler. -//! It should be correct for unimodal distributions, but -//! not for multimodal ones. -// -template -F slice_sampler1d(const LogP& logP, //!< log of distribution to sample - F x, //!< initial sample - Uniform01& u01, //!< uniform random number generator - F min_x = -std::numeric_limits::infinity(), //!< minimum value of support - F max_x = std::numeric_limits::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - assert(std::isfinite(x)); - if (w <= 0.0) { - if (min_x > -std::numeric_limits::infinity() && max_x < std::numeric_limits::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, 0.1); - } - // TRACE4(x, min_x, max_x, w); - F logPx = logP(x); - assert(std::isfinite(logPx)); - U nfeval = 1; - for (U sample = 0; sample < nsamples; ++sample) { - F x0 = x; - F logU = logPx + log(u01()+1e-100); - assert(std::isfinite(logU)); - F r = u01(); - F xl = std::max(min_x, x - r*w); - F xr = std::min(max_x, x + (1-r)*w); - // TRACE3(x, logPx, logU); - while (xl > min_x && logP(xl) > logU) { - xl -= w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << std::endl; - assert(nfeval < max_nfeval); - } - xl = std::max(xl, min_x); - while (xr < max_x && logP(xr) > logU) { - xr += w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xr = " << xr << std::endl; - assert(nfeval < max_nfeval); - } - xr = std::min(xr, max_x); - while (true) { - r = u01(); - x = r*xl + (1-r)*xr; - assert(std::isfinite(x)); - logPx = logP(x); - // TRACE4(logPx, x, xl, xr); - assert(std::isfinite(logPx)); - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << ", xr = " << xr << ", x = " << x << std::endl; - assert(nfeval < max_nfeval); - if (logPx > logU) - break; - else if (x > x0) - xr = x; - else - xl = x; - } - // w = (4*w + (xr-xl))/5; // gradually adjust w - } - // TRACE2(logPx, x); - return x; -} // slice_sampler1d() -*/ - -#endif // SLICE_SAMPLER_H diff --git a/gi/clda/src/timer.h b/gi/clda/src/timer.h deleted file mode 100644 index 123d9a94..00000000 --- a/gi/clda/src/timer.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _TIMER_STATS_H_ -#define _TIMER_STATS_H_ - -#include - -struct Timer { - Timer() { Reset(); } - void Reset() { - start_t = clock(); - } - double Elapsed() const { - const clock_t end_t = clock(); - const double elapsed = (end_t - start_t) / 1000000.0; - return elapsed; - } - private: - std::clock_t start_t; -}; - -#endif diff --git a/gi/evaluation/conditional_entropy.py b/gi/evaluation/conditional_entropy.py deleted file mode 100644 index 356d3b1d..00000000 --- a/gi/evaluation/conditional_entropy.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] input-1 input-2' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:') -slash_threshold = None -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - else: - usage() -if len(args) != 2: - usage() - -ginfile = open(args[0]) -pinfile = open(args[1]) - -# evaluating: H(G | P) = sum_{g,p} p(g,p) log { p(p) / p(g,p) } -# = sum_{g,p} c(g,p)/N { log c(p) - log N - log c(g,p) + log N } -# = 1/N sum_{g,p} c(g,p) { log c(p) - log c(g,p) } -# where G = gold, P = predicted, N = number of events - -N = 0 -gold_frequencies = {} -predict_frequencies = {} -joint_frequencies = {} - -for gline, pline in itertools.izip(ginfile, pinfile): - gparts = gline.split('||| ')[1].split() - pparts = pline.split('||| ')[1].split() - assert len(gparts) == len(pparts) - - for gpart, ppart in zip(gparts, pparts): - gtag = gpart.split(':',1)[1] - ptag = ppart.split(':',1)[1] - - if slash_threshold == None or gtag.count('/') + gtag.count('\\') <= slash_threshold: - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 - - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 - - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 - - N += 1 - -hg2p = 0 -hp2g = 0 -for (gtag, ptag), cgp in joint_frequencies.items(): - hp2g += cgp * (math.log(predict_frequencies[ptag], 2) - math.log(cgp, 2)) - hg2p += cgp * (math.log(gold_frequencies[gtag], 2) - math.log(cgp, 2)) -hg2p /= N -hp2g /= N - -print 'H(P|G)', hg2p, 'H(G|P)', hp2g, 'VI', hg2p + hp2g diff --git a/gi/evaluation/confusion_matrix.py b/gi/evaluation/confusion_matrix.py deleted file mode 100644 index 2dd7aa47..00000000 --- a/gi/evaluation/confusion_matrix.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] [-p output] [-m] input-1 input-2' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:mp:') -slash_threshold = None -output_fname = None -show_matrix = False -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - elif opt == '-p': - output_fname = arg - elif opt == '-m': - show_matrix = True - else: - usage() -if len(args) != 2 or (not show_matrix and not output_fname): - usage() - -ginfile = open(args[0]) -pinfile = open(args[1]) - -if output_fname: - try: - import Image, ImageDraw - except ImportError: - print >>sys.stderr, "Error: Python Image Library not available. Did you forget to set your PYTHONPATH environment variable?" - sys.exit(1) - -N = 0 -gold_frequencies = {} -predict_frequencies = {} -joint_frequencies = {} - -for gline, pline in itertools.izip(ginfile, pinfile): - gparts = gline.split('||| ')[1].split() - pparts = pline.split('||| ')[1].split() - assert len(gparts) == len(pparts) - - for gpart, ppart in zip(gparts, pparts): - gtag = gpart.split(':',1)[1] - ptag = ppart.split(':',1)[1] - - if slash_threshold == None or gtag.count('/') + gtag.count('\\') <= slash_threshold: - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 - - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 - - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 - - N += 1 - -# find top tags -gtags = gold_frequencies.items() -gtags.sort(lambda x,y: x[1]-y[1]) -gtags.reverse() -#gtags = gtags[:50] - -preds = predict_frequencies.items() -preds.sort(lambda x,y: x[1]-y[1]) -preds.reverse() - -if show_matrix: - print '%7s %7s' % ('pred', 'cnt'), - for gtag, gcount in gtags: print '%7s' % gtag, - print - print '=' * 80 - - for ptag, pcount in preds: - print '%7s %7d' % (ptag, pcount), - for gtag, gcount in gtags: - print '%7d' % joint_frequencies.get((gtag, ptag), 0), - print - - print '%7s %7d' % ('total', N), - for gtag, gcount in gtags: print '%7d' % gcount, - print - -if output_fname: - offset=10 - - image = Image.new("RGB", (len(preds), len(gtags)), (255, 255, 255)) - #hsl(hue, saturation%, lightness%) - - # re-sort preds to get a better diagonal - ptags=[] - if True: - ptags = map(lambda (p,c): p, preds) - else: - remaining = set(predict_frequencies.keys()) - for y, (gtag, gcount) in enumerate(gtags): - best = (None, 0) - for ptag in remaining: - #pcount = predict_frequencies[ptag] - p = joint_frequencies.get((gtag, ptag), 0)# / float(pcount) - if p > best[1]: best = (ptag, p) - ptags.append(ptag) - remaining.remove(ptag) - if not remaining: break - - print 'Predicted tag ordering:', ' '.join(ptags) - print 'Gold tag ordering:', ' '.join(map(lambda (t,c): t, gtags)) - - draw = ImageDraw.Draw(image) - for x, ptag in enumerate(ptags): - pcount = predict_frequencies[ptag] - minval = math.log(offset) - maxval = math.log(pcount + offset) - for y, (gtag, gcount) in enumerate(gtags): - f = math.log(offset + joint_frequencies.get((gtag, ptag), 0)) - z = int(240. * (maxval - f) / float(maxval - minval)) - #print x, y, z, f, maxval - draw.point([(x,y)], fill='hsl(%d, 100%%, 50%%)' % z) - del draw - image.save(output_fname) diff --git a/gi/evaluation/entropy.py b/gi/evaluation/entropy.py deleted file mode 100644 index ec1ef502..00000000 --- a/gi/evaluation/entropy.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] input file' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:') -slash_threshold = None -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - else: - usage() -if len(args) != 1: - usage() - -infile = open(args[0]) -N = 0 -frequencies = {} - -for line in infile: - - for part in line.split('||| ')[1].split(): - tag = part.split(':',1)[1] - - if slash_threshold == None or tag.count('/') + tag.count('\\') <= slash_threshold: - frequencies.setdefault(tag, 0) - frequencies[tag] += 1 - N += 1 - -h = 0 -for tag, c in frequencies.items(): - h -= c * (math.log(c, 2) - math.log(N, 2)) -h /= N - -print 'entropy', h diff --git a/gi/evaluation/extract_ccg_labels.py b/gi/evaluation/extract_ccg_labels.py deleted file mode 100644 index e0034648..00000000 --- a/gi/evaluation/extract_ccg_labels.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -# -# Takes spans input along with treebank and spits out CG style categories for each span. -# spans = output from CDEC's extools/extractor with --base_phrase_spans option -# treebank = PTB format, one tree per line -# -# Output is in CDEC labelled-span format -# - -import sys, itertools, tree - -tinfile = open(sys.argv[1]) -einfile = open(sys.argv[2]) - -def number_leaves(node, next=0): - left, right = None, None - for child in node.children: - l, r = number_leaves(child, next) - next = max(next, r+1) - if left == None or l < left: - left = l - if right == None or r > right: - right = r - - #print node, left, right, next - if left == None or right == None: - assert not node.children - left = right = next - - node.left = left - node.right = right - - return left, right - -def ancestor(node, indices): - #print node, node.left, node.right, indices - # returns the deepest node covering all the indices - if min(indices) >= node.left and max(indices) <= node.right: - # try the children - for child in node.children: - x = ancestor(child, indices) - if x: return x - return node - else: - return None - -def frontier(node, indices): - #print 'frontier for node', node, 'indices', indices - if node.left > max(indices) or node.right < min(indices): - #print '\toutside' - return [node] - elif node.children: - #print '\tcovering at least part' - ns = [] - for child in node.children: - n = frontier(child, indices) - ns.extend(n) - return ns - else: - return [node] - -def project_heads(node): - #print 'project_heads', node - is_head = node.data.tag.endswith('-HEAD') - if node.children: - found = 0 - for child in node.children: - x = project_heads(child) - if x: - node.data.tag = x - found += 1 - assert found == 1 - elif is_head: - node.data.tag = node.data.tag[:-len('-HEAD')] - - if is_head: - return node.data.tag - else: - return None - -for tline, eline in itertools.izip(tinfile, einfile): - if tline.strip() != '(())': - if tline.startswith('( '): - tline = tline[2:-1].strip() - tr = tree.parse_PST(tline) - if tr != None: - number_leaves(tr) - #project_heads(tr) # assumes Bikel-style head annotation for the input trees - else: - tr = None - - parts = eline.strip().split(" ||| ") - zh, en = parts[:2] - spans = parts[-1] - print '|||', - for span in spans.split(): - sps = span.split(":") - i, j, x, y = map(int, sps[0].split("-")) - - if tr: - a = ancestor(tr, range(x,y)) - try: - fs = frontier(a, range(x,y)) - except: - print >>sys.stderr, "problem with line", tline.strip(), "--", eline.strip() - raise - - #print x, y - #print 'ancestor', a - #print 'frontier', fs - - cat = a.data.tag - for f in fs: - if f.right < x: - cat += '\\' + f.data.tag - else: - break - fs.reverse() - for f in fs: - if f.left >= y: - cat += '/' + f.data.tag - else: - break - else: - cat = 'FAIL' - - print '%d-%d:%s' % (x, y, cat), - print diff --git a/gi/evaluation/tree.py b/gi/evaluation/tree.py deleted file mode 100644 index 702d80b6..00000000 --- a/gi/evaluation/tree.py +++ /dev/null @@ -1,485 +0,0 @@ -import re, sys - -class Symbol: - def __init__(self, nonterm, term=None, var=None): - assert not (term != None and var != None) - self.tag = nonterm - self.token = term - self.variable = var - - def is_variable(self): - return self.variable != None - - def __eq__(self, other): - return self.tag == other.tag and self.token == other.token and self.variable == other.variable - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash((self.tag, self.token, self.variable)) - - def __repr__(self): - return str(self) - - def __cmp__(self, other): - return cmp((self.tag, self.token, self.variable), - (other.tag, other.token, other.variable)) - - def __str__(self): - parts = [] - if False: # DEPENDENCY - if self.token: - parts.append(str(self.token)) - elif self.variable != None: - parts.append('#%d' % self.variable) - if self.tag: - parts.append(str(self.tag)) - return '/'.join(parts) - else: - if self.tag: - parts.append(str(self.tag)) - if self.token: - parts.append(str(self.token)) - elif self.variable != None: - parts.append('#%d' % self.variable) - return ' '.join(parts) - -class TreeNode: - def __init__(self, data, children=None, order=-1): - self.data = data - self.children = [] - self.order = order - self.parent = None - if children: self.children = children - - def insert(self, child): - self.children.append(child) - child.parent = self - - def leaves(self): - ls = [] - for node in self.xtraversal(): - if not node.children: - ls.append(node.data) - return ls - - def leaf_nodes(self): - ls = [] - for node in self.xtraversal(): - if not node.children: - ls.append(node) - return ls - - def max_depth(self): - d = 1 - for child in self.children: - d = max(d, 1 + child.max_depth()) - if not self.children and self.data.token: - d = 2 - return d - - def max_width(self): - w = 0 - for child in self.children: - w += child.max_width() - return max(1, w) - - def num_internal_nodes(self): - if self.children: - n = 1 - for child in self.children: - n += child.num_internal_nodes() - return n - elif self.data.token: - return 1 - else: - return 0 - - def postorder_traversal(self, visit): - """ - Postorder traversal; no guarantee that terminals will be read in the - correct order for dep. trees. - """ - for child in self.children: - child.traversal(visit) - visit(self) - - def traversal(self, visit): - """ - Preorder for phrase structure trees, and inorder for dependency trees. - In both cases the terminals will be read off in the correct order. - """ - visited_self = False - if self.order <= 0: - visited_self = True - visit(self) - - for i, child in enumerate(self.children): - child.traversal(visit) - if i + 1 == self.order: - visited_self = True - visit(self) - - assert visited_self - - def xpostorder_traversal(self): - for child in self.children: - for node in child.xpostorder_traversal(): - yield node - yield self - - def xtraversal(self): - visited_self = False - if self.order <= 0: - visited_self = True - yield self - - for i, child in enumerate(self.children): - for d in child.xtraversal(): - yield d - - if i + 1 == self.order: - visited_self = True - yield self - - assert visited_self - - def xpostorder_traversal(self): - for i, child in enumerate(self.children): - for d in child.xpostorder_traversal(): - yield d - yield self - - def edges(self): - es = [] - self.traverse_edges(lambda h,c: es.append((h,c))) - return es - - def traverse_edges(self, visit): - for child in self.children: - visit(self.data, child.data) - child.traverse_edges(visit) - - def subtrees(self, include_self=False): - st = [] - if include_self: - stack = [self] - else: - stack = self.children[:] - - while stack: - node = stack.pop() - st.append(node) - stack.extend(node.children) - return st - - def find_parent(self, node): - try: - index = self.children.index(node) - return self, index - except ValueError: - for child in self.children: - if isinstance(child, TreeNode): - r = child.find_parent(node) - if r: return r - return None - - def is_ancestor_of(self, node): - if self == node: - return True - for child in self.children: - if child.is_ancestor_of(child): - return True - return False - - def find(self, node): - if self == node: - return self - for child in self.children: - if isinstance(child, TreeNode): - r = child.find(node) - if r: return r - else: - if child == node: - return r - return None - - def equals_ignorecase(self, other): - if not isinstance(other, TreeNode): - return False - if self.data != other.data: - return False - if len(self.children) != len(other.children): - return False - for mc, oc in zip(self.children, other.children): - if isinstance(mc, TreeNode): - if not mc.equals_ignorecase(oc): - return False - else: - if mc.lower() != oc.lower(): - return False - return True - - def node_number(self, numbering, next=0): - if self.order <= 0: - numbering[id(self)] = next - next += 1 - - for i, child in enumerate(self.children): - next = child.node_number(numbering, next) - if i + 1 == self.order: - numbering[id(self)] = next - next += 1 - - return next - - def display_conll(self, out): - numbering = {} - self.node_number(numbering) - next = 0 - self.children[0].traversal(lambda x: \ - out.write('%d\t%s\t%s\t%s\t%s\t_\t%d\tLAB\n' \ - % (numbering[id(x)], x.data.token, x.data.token, - x.data.tag, x.data.tag, numbering[id(x.parent)]))) - out.write('\n') - - def size(self): - sz = 1 - for child in self.children: - sz += child.size() - return sz - - def __eq__(self, other): - if isinstance(other, TreeNode) and self.data == other.data \ - and self.children == other.children: - return True - return False - - def __cmp__(self, other): - if not isinstance(other, TreeNode): return 1 - n = cmp(self.data, other.data) - if n != 0: return n - n = len(self.children) - len(other.children) - if n != 0: return n - for sc, oc in zip(self.children, other.children): - n = cmp(sc, oc) - if n != 0: return n - return 0 - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.data, tuple(self.children))) - - def __repr__(self): - return str(self) - - def __str__(self): - s = '(' - space = False - if self.order <= 0: - s += str(self.data) - space = True - for i, child in enumerate(self.children): - if space: s += ' ' - s += str(child) - space = True - if i+1 == self.order: - s += ' ' + str(self.data) - return s + ')' - -def read_PSTs(fname): - infile = open(fname) - trees = [] - for line in infile: - trees.append(parse_PST(line.strip())) - infile.close() - return trees - -def parse_PST_multiline(infile, hash_is_var=True): - buf = '' - num_open = 0 - while True: - line = infile.readline() - if not line: - return None - buf += ' ' + line.rstrip() - num_open += line.count('(') - line.count(')') - if num_open == 0: - break - - return parse_PST(buf, hash_is_var) - -def parse_PST(line, hash_is_var=True): - line = line.rstrip() - if not line or line.lower() == 'null': - return None - - # allow either (a/DT) or (DT a) - #parts_re = re.compile(r'(\(*)([^/)]*)(?:/([^)]*))?(\)*)$') - - # only allow (DT a) - parts_re = re.compile(r'(\(*)([^)]*)(\)*)$') - - root = TreeNode(Symbol('TOP')) - stack = [root] - for part in line.rstrip().split(): - m = parts_re.match(part) - #opening, tok_or_tag, tag, closing = m.groups() - opening, tok_or_tag, closing = m.groups() - tag = None - #print 'token', part, 'bits', m.groups() - for i in opening: - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if tag: - stack[-1].data.tag = tag - if hash_is_var and tok_or_tag.startswith('#'): - stack[-1].data.variable = int(tok_or_tag[1:]) - else: - stack[-1].data.token = tok_or_tag - else: - if stack[-1].data.tag == None: - stack[-1].data.tag = tok_or_tag - else: - if hash_is_var and tok_or_tag.startswith('#'): - try: - stack[-1].data.variable = int(tok_or_tag[1:]) - except ValueError: # it's really a token! - #print >>sys.stderr, 'Warning: # used for token:', tok_or_tag - stack[-1].data.token = tok_or_tag - else: - stack[-1].data.token = tok_or_tag - - for i in closing: - stack.pop() - - #assert str(root.children[0]) == line - return root.children[0] - -def read_DTs(fname): - infile = open(fname) - trees = [] - while True: - t = parse_DT(infile) - if t: trees.append(t) - else: break - infile.close() - return trees - -def read_bracketed_DTs(fname): - infile = open(fname) - trees = [] - for line in infile: - trees.append(parse_bracketed_DT(line)) - infile.close() - return trees - -def parse_DT(infile): - tokens = [Symbol('ROOT')] - children = {} - - for line in infile: - parts = line.rstrip().split() - #print parts - if not parts: break - index = len(tokens) - token = parts[1] - tag = parts[3] - parent = int(parts[6]) - if token.startswith('#'): - tokens.append(Symbol(tag, var=int(token[1:]))) - else: - tokens.append(Symbol(tag, token)) - children.setdefault(parent, set()).add(index) - - if len(tokens) == 1: return None - - root = TreeNode(Symbol('ROOT'), [], 0) - schedule = [] - for child in sorted(children[0]): - schedule.append((root, child)) - - while schedule: - parent, index = schedule[0] - del schedule[0] - - node = TreeNode(tokens[index]) - node.order = 0 - parent.insert(node) - - for child in sorted(children.get(index, [])): - schedule.append((node, child)) - if child < index: - node.order += 1 - - return root - -_bracket_split_re = re.compile(r'([(]*)([^)/]*)(?:/([^)]*))?([)]*)') - -def parse_bracketed_DT(line, insert_root=True): - line = line.rstrip() - if not line or line == 'NULL': return None - #print line - - root = TreeNode(Symbol('ROOT')) - stack = [root] - for part in line.rstrip().split(): - m = _bracket_split_re.match(part) - - for c in m.group(1): - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if m.group(3) != None: - if m.group(2).startswith('#'): - stack[-1].data.variable = int(m.group(2)[1:]) - else: - stack[-1].data.token = m.group(2) - stack[-1].data.tag = m.group(3) - else: - stack[-1].data.tag = m.group(2) - stack[-1].order = len(stack[-1].children) - # FIXME: also check for vars - - for c in m.group(4): - stack.pop() - - assert len(stack) == 1 - if not insert_root or root.children[0].data.tag == 'ROOT': - return root.children[0] - else: - return root - -_bracket_split_notag_re = re.compile(r'([(]*)([^)/]*)([)]*)') - -def parse_bracketed_untagged_DT(line): - line = line.rstrip() - if not line or line == 'NULL': return None - - root = TreeNode(Symbol('TOP')) - stack = [root] - for part in line.rstrip().split(): - m = _bracket_split_notag_re.match(part) - - for c in m.group(1): - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if stack[-1].data.token == None: - stack[-1].data.token = m.group(2) - stack[-1].order = len(stack[-1].children) - else: - child = TreeNode(Symbol(nonterm=None, term=m.group(2))) - stack[-1].insert(child) - - for c in m.group(3): - stack.pop() - - return root.children[0] diff --git a/gi/markov_al/Makefile.am b/gi/markov_al/Makefile.am deleted file mode 100644 index fe3e3349..00000000 --- a/gi/markov_al/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -bin_PROGRAMS = ml - -ml_SOURCES = ml.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -I$(top_srcdir)/decoder -AM_LDFLAGS = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/markov_al/README b/gi/markov_al/README deleted file mode 100644 index 9c10f7cd..00000000 --- a/gi/markov_al/README +++ /dev/null @@ -1,2 +0,0 @@ -Experimental translation models with Markovian dependencies. - diff --git a/gi/markov_al/ml.cc b/gi/markov_al/ml.cc deleted file mode 100644 index 1e71edd6..00000000 --- a/gi/markov_al/ml.cc +++ /dev/null @@ -1,470 +0,0 @@ -#include -#include - -#include -#include -#include -#include - -#include "tdict.h" -#include "filelib.h" -#include "sampler.h" -#include "ccrp_onetable.h" -#include "array2d.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -void PrintTopCustomers(const CCRP_OneTable& crp) { - for (CCRP_OneTable::const_iterator it = crp.begin(); it != crp.end(); ++it) { - cerr << " " << TD::Convert(it->first) << " = " << it->second << endl; - } -} - -void PrintAlignment(const vector& src, const vector& trg, const vector& a) { - cerr << TD::GetString(src) << endl << TD::GetString(trg) << endl; - Array2D al(src.size(), trg.size()); - for (int i = 0; i < a.size(); ++i) - if (a[i] != 255) al(a[i], i) = true; - cerr << al << endl; -} - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct Unigram; -struct Bigram { - Bigram() : trg(), cond() {} - Bigram(WordID prev, WordID cur, WordID t) : trg(t) { cond.first = prev; cond.second = cur; } - const pair& ConditioningPair() const { - return cond; - } - WordID& prev_src() { return cond.first; } - WordID& cur_src() { return cond.second; } - const WordID& prev_src() const { return cond.first; } - const WordID& cur_src() const { return cond.second; } - WordID trg; - private: - pair cond; -}; - -struct Unigram { - Unigram() : cur_src(), trg() {} - Unigram(WordID s, WordID t) : cur_src(s), trg(t) {} - WordID cur_src; - WordID trg; -}; - -ostream& operator<<(ostream& os, const Bigram& b) { - os << "( " << TD::Convert(b.trg) << " | " << TD::Convert(b.prev_src()) << " , " << TD::Convert(b.cur_src()) << " )"; - return os; -} - -ostream& operator<<(ostream& os, const Unigram& u) { - os << "( " << TD::Convert(u.trg) << " | " << TD::Convert(u.cur_src) << " )"; - return os; -} - -bool operator==(const Bigram& a, const Bigram& b) { - return a.trg == b.trg && a.cur_src() == b.cur_src() && a.prev_src() == b.prev_src(); -} - -bool operator==(const Unigram& a, const Unigram& b) { - return a.trg == b.trg && a.cur_src == b.cur_src; -} - -size_t hash_value(const Bigram& b) { - size_t h = boost::hash_value(b.prev_src()); - boost::hash_combine(h, boost::hash_value(b.cur_src())); - boost::hash_combine(h, boost::hash_value(b.trg)); - return h; -} - -size_t hash_value(const Unigram& u) { - size_t h = boost::hash_value(u.cur_src); - boost::hash_combine(h, boost::hash_value(u.trg)); - return h; -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UnigramModel { - UnigramModel(size_t src_voc_size, size_t trg_voc_size) : - unigrams(TD::NumWords() + 1, CCRP_OneTable(1,1,1,1)), - p0(1.0 / trg_voc_size) {} - - void increment(const Bigram& b) { - unigrams[b.cur_src()].increment(b.trg); - } - - void decrement(const Bigram& b) { - unigrams[b.cur_src()].decrement(b.trg); - } - - double prob(const Bigram& b) const { - const double q0 = unigrams[b.cur_src()].prob(b.trg, p0); - return q0; - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < unigrams.size(); ++i) { - const CCRP_OneTable& crp = unigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - return llh; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < unigrams.size(); ++i) - unigrams[i].resample_hyperparameters(rng); - } - - vector > unigrams; // unigrams[src].prob(trg, p0) = p(trg|src) - - const double p0; -}; - -struct BigramModel { - BigramModel(size_t src_voc_size, size_t trg_voc_size) : - unigrams(TD::NumWords() + 1, CCRP_OneTable(1,1,1,1)), - p0(1.0 / trg_voc_size) {} - - void increment(const Bigram& b) { - BigramMap::iterator it = bigrams.find(b.ConditioningPair()); - if (it == bigrams.end()) { - it = bigrams.insert(make_pair(b.ConditioningPair(), CCRP_OneTable(1,1,1,1))).first; - } - if (it->second.increment(b.trg)) - unigrams[b.cur_src()].increment(b.trg); - } - - void decrement(const Bigram& b) { - BigramMap::iterator it = bigrams.find(b.ConditioningPair()); - assert(it != bigrams.end()); - if (it->second.decrement(b.trg)) { - unigrams[b.cur_src()].decrement(b.trg); - if (it->second.num_customers() == 0) - bigrams.erase(it); - } - } - - double prob(const Bigram& b) const { - const double q0 = unigrams[b.cur_src()].prob(b.trg, p0); - const BigramMap::const_iterator it = bigrams.find(b.ConditioningPair()); - if (it == bigrams.end()) return q0; - return it->second.prob(b.trg, q0); - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < unigrams.size(); ++i) { - const CCRP_OneTable& crp = unigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - for (BigramMap::const_iterator it = bigrams.begin(); it != bigrams.end(); ++it) { - const CCRP_OneTable& crp = it->second; - const WordID cur_src = it->first.second; - llh += crp.log_crp_prob(); - for (CCRP_OneTable::const_iterator bit = crp.begin(); bit != crp.end(); ++bit) { - llh += log(unigrams[cur_src].prob(bit->second, p0)); - } - } - return llh; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < unigrams.size(); ++i) - unigrams[i].resample_hyperparameters(rng); - for (BigramMap::iterator it = bigrams.begin(); it != bigrams.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - typedef unordered_map, CCRP_OneTable, boost::hash > > BigramMap; - BigramMap bigrams; // bigrams[(src-1,src)].prob(trg, q0) = p(trg|src,src-1) - vector > unigrams; // unigrams[src].prob(trg, p0) = p(trg|src) - - const double p0; -}; - -struct BigramAlignmentModel { - BigramAlignmentModel(size_t src_voc_size, size_t trg_voc_size) : bigrams(TD::NumWords() + 1, CCRP_OneTable(1,1,1,1)), p0(1.0 / src_voc_size) {} - void increment(WordID prev, WordID next) { - bigrams[prev].increment(next); // hierarchy? - } - void decrement(WordID prev, WordID next) { - bigrams[prev].decrement(next); // hierarchy? - } - double prob(WordID prev, WordID next) { - return bigrams[prev].prob(next, p0); - } - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < bigrams.size(); ++i) { - const CCRP_OneTable& crp = bigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - return llh; - } - - vector > bigrams; // bigrams[prev].prob(next, p0) = p(next|prev) - const double p0; -}; - -struct Alignment { - vector a; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned samples = conf["samples"].as(); - - boost::shared_ptr prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - const size_t corpus_len = corpusf.size(); - const WordID kNULL = TD::Convert(""); - const WordID kBOS = TD::Convert(""); - const WordID kEOS = TD::Convert(""); - Bigram TT(kBOS, TD::Convert("我"), TD::Convert("i")); - Bigram TT2(kBOS, TD::Convert("è¦"), TD::Convert("i")); - - UnigramModel model(vocabf.size(), vocabe.size()); - vector alignments(corpus_len); - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector& src = corpusf[ci]; - const vector& trg = corpuse[ci]; - vector& alg = alignments[ci].a; - alg.resize(trg.size()); - int lenp1 = src.size() + 1; - WordID prev_src = kBOS; - for (int j = 0; j < trg.size(); ++j) { - int samp = lenp1 * rng.next(); - --samp; - if (samp < 0) samp = 255; - alg[j] = samp; - WordID cur_src = (samp == 255 ? kNULL : src[alg[j]]); - Bigram b(prev_src, cur_src, trg[j]); - model.increment(b); - prev_src = cur_src; - } - Bigram b(prev_src, kEOS, kEOS); - model.increment(b); - } - cerr << "Initial LLH: " << model.LogLikelihood() << endl; - - SampleSet ss; - for (unsigned si = 0; si < 50; ++si) { - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector& src = corpusf[ci]; - const vector& trg = corpuse[ci]; - vector& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (unsigned j = 0; j < trg.size(); ++j) { - unsigned char& a_j = alg[j]; - WordID cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - Bigram b(prev_src, cur_e_a_j, trg[j]); - //cerr << "DEC: " << b << "\t" << nextb << endl; - model.decrement(b); - ss.clear(); - for (unsigned i = 0; i <= src.size(); ++i) { - const WordID cur_src = (i ? src[i-1] : kNULL); - b.cur_src() = cur_src; - ss.add(model.prob(b)); - } - int sampled_a_j = rng.SelectSample(ss); - a_j = (sampled_a_j ? sampled_a_j - 1 : 255); - cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - b.cur_src() = cur_e_a_j; - //cerr << "INC: " << b << "\t" << nextb << endl; - model.increment(b); - prev_src = cur_e_a_j; - } - } - cerr << '.' << flush; - if (si % 10 == 9) { - cerr << "[LLH prev=" << model.LogLikelihood(); - //model.ResampleHyperparameters(&rng); - cerr << " new=" << model.LogLikelihood() << "]\n"; - //pair xx = make_pair(kBOS, TD::Convert("我")); - //PrintTopCustomers(model.bigrams.find(xx)->second); - cerr << "p(" << TT << ") = " << model.prob(TT) << endl; - cerr << "p(" << TT2 << ") = " << model.prob(TT2) << endl; - PrintAlignment(corpusf[0], corpuse[0], alignments[0].a); - } - } - { - // MODEL 2 - BigramModel model(vocabf.size(), vocabe.size()); - BigramAlignmentModel amodel(vocabf.size(), vocabe.size()); - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector& src = corpusf[ci]; - const vector& trg = corpuse[ci]; - vector& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (int j = 0; j < trg.size(); ++j) { - WordID cur_src = (alg[j] == 255 ? kNULL : src[alg[j]]); - Bigram b(prev_src, cur_src, trg[j]); - model.increment(b); - amodel.increment(prev_src, cur_src); - prev_src = cur_src; - } - amodel.increment(prev_src, kEOS); - Bigram b(prev_src, kEOS, kEOS); - model.increment(b); - } - cerr << "Initial LLH: " << model.LogLikelihood() << " " << amodel.LogLikelihood() << endl; - - SampleSet ss; - for (unsigned si = 0; si < samples; ++si) { - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector& src = corpusf[ci]; - const vector& trg = corpuse[ci]; - vector& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (unsigned j = 0; j < trg.size(); ++j) { - unsigned char& a_j = alg[j]; - WordID cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - Bigram b(prev_src, cur_e_a_j, trg[j]); - WordID next_src = kEOS; - WordID next_trg = kEOS; - if (j < (trg.size() - 1)) { - next_src = (alg[j+1] == 255 ? kNULL : src[alg[j + 1]]); - next_trg = trg[j + 1]; - } - Bigram nextb(cur_e_a_j, next_src, next_trg); - //cerr << "DEC: " << b << "\t" << nextb << endl; - model.decrement(b); - model.decrement(nextb); - amodel.decrement(prev_src, cur_e_a_j); - amodel.decrement(cur_e_a_j, next_src); - ss.clear(); - for (unsigned i = 0; i <= src.size(); ++i) { - const WordID cur_src = (i ? src[i-1] : kNULL); - b.cur_src() = cur_src; - ss.add(model.prob(b) * model.prob(nextb) * amodel.prob(prev_src, cur_src) * amodel.prob(cur_src, next_src)); - //cerr << log(ss[ss.size() - 1]) << "\t" << b << endl; - } - int sampled_a_j = rng.SelectSample(ss); - a_j = (sampled_a_j ? sampled_a_j - 1 : 255); - cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - b.cur_src() = cur_e_a_j; - nextb.prev_src() = cur_e_a_j; - //cerr << "INC: " << b << "\t" << nextb << endl; - //exit(1); - model.increment(b); - model.increment(nextb); - amodel.increment(prev_src, cur_e_a_j); - amodel.increment(cur_e_a_j, next_src); - prev_src = cur_e_a_j; - } - } - cerr << '.' << flush; - if (si % 10 == 9) { - cerr << "[LLH prev=" << (model.LogLikelihood() + amodel.LogLikelihood()); - //model.ResampleHyperparameters(&rng); - cerr << " new=" << model.LogLikelihood() << "]\n"; - pair xx = make_pair(kBOS, TD::Convert("我")); - cerr << "p(" << TT << ") = " << model.prob(TT) << endl; - cerr << "p(" << TT2 << ") = " << model.prob(TT2) << endl; - pair xx2 = make_pair(kBOS, TD::Convert("è¦")); - PrintTopCustomers(model.bigrams.find(xx)->second); - //PrintTopCustomers(amodel.bigrams[TD::Convert("")]); - //PrintTopCustomers(model.unigrams[TD::Convert("")]); - PrintAlignment(corpusf[0], corpuse[0], alignments[0].a); - } - } - } - return 0; -} - diff --git a/gi/morf-segmentation/filter_docs.pl b/gi/morf-segmentation/filter_docs.pl deleted file mode 100755 index a78575da..00000000 --- a/gi/morf-segmentation/filter_docs.pl +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/perl - -#Filters the phrase&cluster document set to retain only documents that correspond to words or morphs, i.e. not crossing word boundaries. - -#Usage: filter_docs.pl [mark] -# STDIN: data in the doc.txt format (i.e. phrase\t blahblah ), most likely from cdec extractor -# STDOUT: the matching subset, same format - -use utf8; -my $letter=qr/\p{L}\p{M}*/; # see http://www.regular-expressions.info/unicode.html - -my $morph=qr/$letter+/; - -my $m = "##"; # marker used to indicate morphemes -if ((scalar @ARGV) >= 1) { - $m = $ARGV[0]; - shift; -} -print STDERR "Using $m to filter for morphemes\n"; - -my $expr = qr/^($morph\Q$m\E)? ?(\Q$m\E$morph\Q$m\E)* ?(\Q$m\E$morph)?\t/; #\Q and \E bounded sections are escaped -while(<>) { - /$expr/ && print; -} diff --git a/gi/morf-segmentation/invalid_vocab.patterns b/gi/morf-segmentation/invalid_vocab.patterns deleted file mode 100644 index 473ce1b1..00000000 --- a/gi/morf-segmentation/invalid_vocab.patterns +++ /dev/null @@ -1,6 +0,0 @@ -[[:digit:]] -[] !"#$%&()*+,./:;<=>?@[\^_`{|}~] -^'$ --$ -^- -^$ diff --git a/gi/morf-segmentation/linestripper.py b/gi/morf-segmentation/linestripper.py deleted file mode 100755 index 04e9044a..00000000 --- a/gi/morf-segmentation/linestripper.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python - -import sys - -#linestripper file file maxlen [numlines] - -if len(sys.argv) < 3: - print "linestripper file1 file2 maxlen [numlines]" - print " outputs subset of file1 to stdout, ..of file2 to stderr" - sys.exit(1) - - -f1 = open(sys.argv[1],'r') -f2 = open(sys.argv[2],'r') - -maxlen=int(sys.argv[3]) -numlines = 0 - -if len(sys.argv) > 4: - numlines = int(sys.argv[4]) - -count=0 -for line1 in f1: - line2 = f2.readline() - - w1 = len(line1.strip().split()) - w2 = len(line2.strip().split()) - - if w1 <= maxlen and w2 <= maxlen: - count = count + 1 - sys.stdout.write(line1) - sys.stderr.write(line2) - - if numlines > 0 and count >= numlines: - break - -f1.close() -f2.close() - - diff --git a/gi/morf-segmentation/morf-pipeline.pl b/gi/morf-segmentation/morf-pipeline.pl deleted file mode 100755 index 46eb5b46..00000000 --- a/gi/morf-segmentation/morf-pipeline.pl +++ /dev/null @@ -1,486 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use File::Copy; - - -# Preprocessing pipeline to take care of word segmentation -# Learns a segmentation model for each/either side of the parallel corpus using all train/dev/test data -# Applies the segmentation where necessary. -# Learns word alignments on the preprocessed training data. -# Outputs script files used later to score output. - - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -use Getopt::Long "GetOptions"; - -my $GZIP = 'gzip'; -my $ZCAT = 'gunzip -c'; -my $SED = 'sed -e'; - -my $MORF_TRAIN = "$SCRIPT_DIR/morftrain.sh"; -my $MORF_SEGMENT = "$SCRIPT_DIR/morfsegment.py"; - -my $LINESTRIPPER = "$SCRIPT_DIR/linestripper.py"; -my $ALIGNER = "/export/ws10smt/software/berkeleyaligner/berkeleyaligner.jar"; -#java -d64 -Xmx10g -jar $ALIGNER ++word-align.conf >> aligner.log -assert_exec($MORF_TRAIN, $LINESTRIPPER, $MORF_SEGMENT, $ALIGNER); - -my $OUTPUT = './morfwork'; -my $PPL_SRC = 50; -my $PPL_TRG = 50; -my $MARKER = "#"; -my $MAX_WORDS = 40; -my $SENTENCES;# = 100000; -my $SPLIT_TYPE = ""; #possible values: s, t, st, or (empty string) -my $NAME_SHORTCUT; - -usage() unless &GetOptions('max_words=i' => \$MAX_WORDS, - 'output=s' => \$OUTPUT, - 'ppl_src=i' => \$PPL_SRC, - 'ppl_trg=i' => \$PPL_TRG, - 'sentences=i' => \$SENTENCES, - 'marker=s' => \$MARKER, - 'split=s' => \$SPLIT_TYPE, - 'get_name_only' => \$NAME_SHORTCUT, - ); - -usage() unless scalar @ARGV >= 2; - -my %CORPUS; # for (src,trg) it has (orig, name, filtered, final) - -$CORPUS{'src'}{'orig'} = $ARGV[0]; -open F, "<$CORPUS{'src'}{'orig'}" or die "Can't read $CORPUS{'src'}{'orig'}: $!"; close F; -$CORPUS{'src'}{'name'} = get_basename($CORPUS{'src'}{'orig'}); - -$CORPUS{'trg'}{'orig'} = $ARGV[1]; -open F, "<$CORPUS{'trg'}{'orig'}" or die "Can't read $CORPUS{'trg'}{'orig'}: $!"; close F; -$CORPUS{'trg'}{'name'} = get_basename($CORPUS{'trg'}{'orig'}); - -my %DEV; # for (src,trg) has (orig, final.split final.unsplit -if (@ARGV >= 4) { - $DEV{'src'}{'orig'} = $ARGV[2]; - open F, "<$DEV{'src'}{'orig'}" or die "Can't read $DEV{'src'}{'orig'}: $!"; close F; - $DEV{'src'}{'name'} = get_basename($DEV{'src'}{'orig'}); - $DEV{'trg'}{'orig'} = $ARGV[3]; - open F, "<$DEV{'trg'}{'orig'}" or die "Can't read $DEV{'trg'}{'orig'}: $!"; close F; - $DEV{'trg'}{'name'} = get_basename($DEV{'trg'}{'orig'}); -} - -my %TEST; # for (src,trg) has (orig, name) -if (@ARGV >= 6) { - $TEST{'src'}{'orig'} = $ARGV[4]; - open F, "<$TEST{'src'}{'orig'}" or die "Can't read $TEST{'src'}{'orig'}: $!"; close F; - $TEST{'src'}{'name'} = get_basename($TEST{'src'}{'orig'}); - $TEST{'trg'}{'orig'} = $ARGV[5]; - open F, "<$TEST{'trg'}{'orig'}" or die "Can't read $TEST{'trg'}{'orig'}: $!"; close F; - $TEST{'trg'}{'name'} = get_basename($TEST{'trg'}{'orig'}); -} - -my $SPLIT_SRC; #use these to check whether that part is being split -my $SPLIT_TRG; - -#OUTPUT WILL GO IN THESE -my $CORPUS_DIR = $OUTPUT . '/' . corpus_dir(); #subsampled corpus -my $MODEL_SRC_DIR = $OUTPUT . '/' . model_dir("src"); #splitting.. -my $MODEL_TRG_DIR = $OUTPUT . '/' . model_dir("trg"); # .. models -my $PROCESSED_DIR = $OUTPUT . '/' . processed_dir(); #segmented copora+alignments -my $ALIGNMENT_DIR = $PROCESSED_DIR . '/alignments'; - -$CORPUS{'src'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'src'}{'name'}"; -$CORPUS{'trg'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'trg'}{'name'}"; - -print STDERR "Output: $OUTPUT\n"; -print STDERR "Corpus: $CORPUS_DIR\n"; -print STDERR "Model-src: $MODEL_SRC_DIR\n"; -print STDERR "Model-trg: $MODEL_TRG_DIR\n"; -print STDERR "Finaldir: $PROCESSED_DIR\n"; - -safemkdir($OUTPUT) or die "Couldn't create output directory $OUTPUT: $!"; -safemkdir($CORPUS_DIR) or die "Couldn't create output directory $CORPUS_DIR: $!"; -filter_corpus(); - -safemkdir($PROCESSED_DIR); -safemkdir($ALIGNMENT_DIR); - -if ($SPLIT_SRC) { - safemkdir($MODEL_SRC_DIR) or die "Couldn't create output directory $MODEL_SRC_DIR: $!"; - learn_segmentation("src"); - apply_segmentation_side("src", $MODEL_SRC_DIR); -} - -#assume that unsplit hypotheses will be scored against an aritificially split target test set; thus obtain a target splitting model -#TODO: add a flag to override this behaviour -safemkdir($MODEL_TRG_DIR) or die "Couldn't create output directory $MODEL_TRG_DIR: $!"; -learn_segmentation("trg"); -$TEST{'trg'}{'finalunsplit'} = "$PROCESSED_DIR/$TEST{'trg'}{'name'}"; -copy($TEST{'trg'}{'orig'}, $TEST{'trg'}{'finalunsplit'}) or die "Could not copy unsegmented test set"; - -if ($SPLIT_TRG) { - apply_segmentation_side("trg", $MODEL_TRG_DIR); - } else { - $TEST{'trg'}{'finalsplit'} = "$PROCESSED_DIR/$TEST{'trg'}{'name'}.split"; - apply_segmentation_any($MODEL_TRG_DIR, $TEST{'trg'}{'finalunsplit'}, $TEST{'trg'}{'finalsplit'}); -} - -write_eval_sh("$PROCESSED_DIR/eval-devtest.sh"); - -#copy corpora if they haven't been put in place by splitting operations -place_missing_data_side('src'); -place_missing_data_side('trg'); - -do_align(); - -if ($CORPUS{'src'}{'orig'} && $DEV{'src'}{'orig'} && $TEST{'src'}{'orig'}) { - print STDERR "Putting the config file entry in $PROCESSED_DIR/exp.config\n"; -#format is: - # nlfr100k_unsplit /export/ws10smt/jan/nlfr/morfwork/s100k.w40.sp_0 corpus.nl-fr.al fr-3.lm.gz dev.nl dev.fr test2008.nl eval-devtest.sh - my $line = split_name() . " $PROCESSED_DIR corpus.src-trg.al LMFILE.lm.gz"; - $line = $line . " $DEV{'src'}{'name'} $DEV{'trg'}{'name'}"; - $line = $line . " " . get_basename($TEST{'src'}{$SPLIT_SRC ? "finalsplit" : "finalunsplit"}) . " eval-devtest.sh"; - safesystem("echo '$line' > $PROCESSED_DIR/exp.config"); -} - -system("date"); -print STDERR "All done. You now need to train a language model (if target split), put it in the right dir and update the config file.\n\n"; - -############################## BILINGUAL ################################### - -sub filter_corpus { - print STDERR "\n!!!FILTERING TRAINING COPRUS!!!\n"; - if ( -f $CORPUS{'src'}{'filtered'} && -f $CORPUS{'trg'}{'filtered'}) { - print STDERR "$CORPUS{'src'}{'filtered'} and $CORPUS{'trg'}{'filtered'} exist, reusing...\n"; - return; - } - my $args = "$CORPUS{'src'}{'orig'} $CORPUS{'trg'}{'orig'} $MAX_WORDS"; - if ($SENTENCES) { $args = $args . " $SENTENCES"; } - safesystem("$LINESTRIPPER $args 1> $CORPUS{'src'}{'filtered'} 2> $CORPUS{'trg'}{'filtered'}") or die "Failed to filter training corpus for length."; -} - -sub learn_segmentation -{ - my $WHICH = shift; - my $corpus; my $dev; my $test; my $moddir; my $ppl; - - $corpus = $CORPUS{$WHICH}{'filtered'}; - $dev = $DEV{$WHICH}{'orig'}; - $test = $TEST{$WHICH}{'orig'}; - - if ($WHICH eq "src") { - $moddir = $MODEL_SRC_DIR; - $ppl = $PPL_SRC; - } else { - $moddir = $MODEL_TRG_DIR; - $ppl = $PPL_TRG; - } - my $cmd = "cat $corpus"; - if ($dev) { $cmd = "$cmd $dev"; } - if ($test) { $cmd = "$cmd $test"; } - my $tmpfile = "$CORPUS_DIR/all.tmp.gz"; - safesystem("$cmd | $GZIP > $tmpfile") or die "Failed to concatenate data for model learning.."; - assert_marker($tmpfile); - - learn_segmentation_side($tmpfile, $moddir, $ppl, $WHICH); - safesystem("rm $tmpfile"); -} - -sub do_align { - print STDERR "\n!!!WORD ALIGNMENT!!!\n"; - system("date"); - - my $ALIGNMENTS = "$ALIGNMENT_DIR/training.align"; - if ( -f $ALIGNMENTS ) { - print STDERR "$ALIGNMENTS exists, reusing...\n"; - return; - } - my $conf_file = "$ALIGNMENT_DIR/word-align.conf"; - - #decorate training files with identifiers to stop the aligner from training on dev and test when rerun in future. - safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'src'}{'name'} corpus.src") or die "Failed to symlink: $!"; - safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'trg'}{'name'} corpus.trg") or die "Failed to symlink: $!"; - - write_wconf($conf_file, $PROCESSED_DIR); - system("java -d64 -Xmx24g -jar $ALIGNER ++$conf_file > $ALIGNMENT_DIR/aligner.log"); - - if (! -f $ALIGNMENTS) { die "Failed to run word alignment.";} - - my $cmd = "paste $PROCESSED_DIR/corpus.src $PROCESSED_DIR/corpus.trg $ALIGNMENTS"; - $cmd = $cmd . " | sed 's/\\t/ \|\|\| /g' > $PROCESSED_DIR/corpus.src-trg.al"; - safesystem($cmd) or die "Failed to paste into aligned corpus file."; - -} - -############################# MONOLINGUAL ################################# - -#copy the necessary data files that weren't place by segmentation -sub place_missing_data_side { - my $side = shift; - - ifne_copy($CORPUS{$side}{'filtered'}, "$PROCESSED_DIR/$CORPUS{$side}{'name'}") ; - - if ($DEV{$side}{'orig'} && ! -f "$PROCESSED_DIR/$DEV{$side}{'name'}") { - $DEV{$side}{'final'} = "$PROCESSED_DIR/$DEV{$side}{'name'}"; - copy($DEV{$side}{'orig'}, $DEV{$side}{'final'}) or die "Copy failed: $!"; - } - - if ($TEST{$side}{'orig'} && ! -f "$PROCESSED_DIR/$TEST{$side}{'name'}" && ! $TEST{$side}{'finalunsplit'}) { - $TEST{$side}{'finalunsplit'} = "$PROCESSED_DIR/$TEST{$side}{'name'}"; - copy($TEST{$side}{'orig'}, $TEST{$side}{'finalunsplit'}) or die "Copy failed: $!"; - } - -} - -sub apply_segmentation_side { - my ($side, $moddir) = @_; - - print STDERR "\n!!!APPLYING SEGMENTATION MODEL ($side)!!!\n"; - apply_segmentation_any($moddir, $CORPUS{$side}{'filtered'}, "$PROCESSED_DIR/$CORPUS{$side}{'name'}"); - if ($DEV{$side}{'orig'}) { - $DEV{$side}{'final'} = "$PROCESSED_DIR/$DEV{$side}{'name'}"; - apply_segmentation_any($moddir, $DEV{$side}{'orig'}, "$DEV{$side}{'final'}"); - } - if ($TEST{$side}{'orig'}) { - $TEST{$side}{'finalsplit'} = "$PROCESSED_DIR/$TEST{$side}{'name'}.split"; - apply_segmentation_any($moddir, $TEST{$side}{'orig'}, $TEST{$side}{'finalsplit'} ); - } - -} - -sub learn_segmentation_side { - my($INPUT_FILE, $SEGOUT_DIR, $PPL, $LANG) = @_; - - print STDERR "\n!!!LEARNING SEGMENTATION MODEL ($LANG)!!!\n"; - system("date"); - my $SEG_FILE = $SEGOUT_DIR . "/segmentation.ready"; - if ( -f $SEG_FILE) { - print STDERR "$SEG_FILE exists, reusing...\n"; - return; - } - my $cmd = "$MORF_TRAIN $INPUT_FILE $SEGOUT_DIR $PPL \"$MARKER\""; - safesystem($cmd) or die "Failed to learn segmentation model"; -} - -sub apply_segmentation_any { - my($moddir, $datfile, $outfile) = @_; - if ( -f $outfile) { - print STDERR "$outfile exists, reusing...\n"; - return; - } - - my $args = "$moddir/inputvocab.gz $moddir/segmentation.ready \"$MARKER\""; - safesystem("cat $datfile | $MORF_SEGMENT $args &> $outfile") or die "Could not segment $datfile"; -} - -##################### PATH FUNCTIONS ########################## - -sub beautify_numlines { - return ($SENTENCES ? $SENTENCES : "_all"); -} - -sub corpus_dir { - return "s" . beautify_numlines() . ".w" . $MAX_WORDS; -} - -sub model_dir { - my $lang = shift; - if ($lang eq "src") { - return corpus_dir() . ".PPL" . $PPL_SRC . ".src"; - } elsif ($lang eq "trg") { - return corpus_dir() . ".PPL" . $PPL_TRG . ".trg"; - } else { - return "PPLundef"; - } -} - -sub processed_dir { - return corpus_dir() . "." . split_name(); -} - -########################## HELPER FUNCTIONS ############################ - -sub ifne_copy { - my ($src, $dest) = @_; - if (! -f $dest) { - copy($src, $dest) or die "Copy failed: $!"; - } -} - -sub split_name { - #parses SPLIT_TYPE, which can have the following values - # t|s|ts|st (last 2 are equiv) - # or is undefined when no splitting is done - my $name = ""; - - if ($SPLIT_TYPE) { - $SPLIT_SRC = lc($SPLIT_TYPE) =~ /s/; - $SPLIT_TRG = lc($SPLIT_TYPE) =~ /t/; - $name = $name . ($SPLIT_SRC ? $PPL_SRC : "0"); - $name = $name . "_" . ($SPLIT_TRG ? $PPL_TRG : "0"); - } else { - #no splitting - $name = "0"; - } - - return "sp_" . $name; - -} - -sub usage { - print <> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - -sub get_basename -{ - my $x = shift; - $x = `basename $x`; - $x =~ s/\n//; - return $x; -} - -sub assert_marker { - my $file = shift; - my $result = `zcat $file| grep '$MARKER' | wc -l` or die "Cannot read $file: $!"; - print $result; - if (scalar($result) != 0) { die "Data contains marker '$MARKER'; use something else.";} -} -########################### Dynamic config files ############################## - -sub write_wconf { - my ($filename, $train_dir) = @_; - open WCONF, ">$filename" or die "Can't write $filename: $!"; - - print WCONF <$filename" or die "Can't write $filename: $!"; - - print EVALFILE < "\$1.recombined" - -\$EVAL_MAIN "\$1.recombined" $TEST{'trg'}{'finalunsplit'} -EOT - - } else { - print EVALFILE < "\$1.split" - -\$EVAL_MAIN "\$1.split" $TEST{'trg'}{'finalsplit'} - -echo "DIRECT EVALUATION" -echo "--------------------------" -\$EVAL_MAIN "\$1" $TEST{'trg'}{'finalunsplit'} - -EOT - - } - close EVALFILE; - -} - - - - diff --git a/gi/morf-segmentation/morfsegment.py b/gi/morf-segmentation/morfsegment.py deleted file mode 100755 index 85b9d4fb..00000000 --- a/gi/morf-segmentation/morfsegment.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python - -import sys -import gzip - -#usage: morfsegment.py inputvocab.gz segmentation.ready -# stdin: the data to segment -# stdout: the segmented data - -if len(sys.argv) < 3: - print "usage: morfsegment.py inputvocab.gz segmentation.ready [marker]" - print " stdin: the data to segment" - print " stdout: the segmented data" - sys.exit() - -#read index: -split_index={} - -marker="##" - -if len(sys.argv) > 3: - marker=sys.argv[3] - -word_vocab=gzip.open(sys.argv[1], 'rb') #inputvocab.gz -seg_vocab=open(sys.argv[2], 'r') #segm.ready.. - -for seg in seg_vocab: - #seg = ver# #wonder\n - #wordline = 1 verwonder\n - word = word_vocab.readline().strip().split(' ') - assert(len(word) == 2) - word = word[1] - seg=seg.strip() - - if seg != word: - split_index[word] = seg - -word_vocab.close() -seg_vocab.close() - -for line in sys.stdin: - words = line.strip().split() - - newsent = [] - for word in words: - splitword = split_index.get(word, word) - newsent.append(splitword) - - print ' '.join(newsent) - diff --git a/gi/morf-segmentation/morftrain.sh b/gi/morf-segmentation/morftrain.sh deleted file mode 100755 index 9004922f..00000000 --- a/gi/morf-segmentation/morftrain.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -if [[ $# -lt 3 ]]; then - echo "Trains a morfessor model and places the result in writedir" - echo - echo "Usage: `basename $0` corpus_input_file writedir [PPL] [marker] [lines]" - echo -e "\tcorpus_input_file contains a sentence per line." - exit 1 -fi - -MORFESSOR_DIR="/export/ws10smt/software/morfessor_catmap0.9.2" -SCRIPT_DIR=$(dirname `readlink -f $0`) - -MORFBINDIR="$MORFESSOR_DIR/bin" -MORFMAKEFILE_TRAIN="$MORFESSOR_DIR/train/Makefile" -VOCABEXT="$SCRIPT_DIR/vocabextractor.sh" - -MARKER="#" - -if [[ ! -f $VOCABEXT ]]; then - echo "$VOCABEXT doesn't exist!" - exit 1 -fi -if [[ ! -f $MORFMAKEFILE_TRAIN ]]; then - echo "$MORFMAKEFILE_TRAIN doesn't exist!" - exit 1 -fi - - -CORPUS="$1" -WRITETODIR=$2 - -if [[ ! -f $CORPUS ]]; then - echo "$CORPUS doesn't exist!" - exit 1 -fi - -PPL=10 -LINES=0 -if [[ $# -gt 2 ]]; then - PPL=$3 -fi -if [[ $# -gt 3 ]]; then - MARKER="$4" -fi -if [[ $# -gt 4 ]]; then - LINES=$5 -fi - -mkdir -p $WRITETODIR - -#extract vocabulary to train on -echo "Extracting vocabulary..." -if [[ -f $WRITETODIR/inputvocab.gz ]]; then - echo " ....$WRITETODIR/inputvocab.gz exists, reusing." -else - if [[ $LINES -gt 0 ]]; then - $VOCABEXT $CORPUS $LINES | gzip > $WRITETODIR/inputvocab.gz - else - $VOCABEXT $CORPUS | gzip > $WRITETODIR/inputvocab.gz - fi -fi - - -#train it -echo "Training morf model..." -if [[ -f $WRITETODIR/segmentation.final.gz ]]; then - echo " ....$WRITETODIR/segmentation.final.gz exists, reusing.." -else - OLDPWD=`pwd` - cd $WRITETODIR - - #put the training Makefile in place, with appropriate modifications - sed -e "s/^GZIPPEDINPUTDATA = .*$/GZIPPEDINPUTDATA = inputvocab.gz/" \ - -e "s/^PPLTHRESH = .*$/PPLTHRESH = $PPL/" \ - -e "s;^BINDIR = .*$;BINDIR = $MORFBINDIR;" \ - $MORFMAKEFILE_TRAIN > ./Makefile - - date - make > ./trainmorf.log 2>&1 - cd $OLDPWD - - - echo "Post processing..." - #remove comments, counts and morph types - #mark morphs - - if [[ ! -f $WRITETODIR/segmentation.final.gz ]]; then - echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written" - exit 1 - fi - - zcat $WRITETODIR/segmentation.final.gz | \ - awk '$1 !~ /^#/ {print}' | \ - cut -d ' ' --complement -f 1 | \ - sed -e "s/\/...//g" -e "s/ + /$MARKER $MARKER/g" \ - > $WRITETODIR/segmentation.ready - - if [[ ! -f $WRITETODIR/segmentation.ready ]]; then - echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written" - exit 1 - fi - - - - echo "Done training." - date -fi -echo "Segmentation model is $WRITETODIR/segmentation.ready." - diff --git a/gi/morf-segmentation/vocabextractor.sh b/gi/morf-segmentation/vocabextractor.sh deleted file mode 100755 index 00ae7109..00000000 --- a/gi/morf-segmentation/vocabextractor.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -d=$(dirname `readlink -f $0`) -if [ $# -lt 1 ]; then - echo "Extracts unique words and their frequencies from a subset of a corpus." - echo - echo "Usage: `basename $0` input_file [number_of_lines] > output_file" - echo -e "\tinput_file contains a sentence per line." - echo - echo "Script also removes words from the vocabulary if they contain a digit or a special character. Output is printed to stdout in a format suitable for use with Morfessor." - echo - exit -fi - -srcname=$1 -reallen=0 - -if [[ $# -gt 1 ]]; then - reallen=$2 -fi - -pattern_file=$d/invalid_vocab.patterns - -if [[ ! -f $pattern_file ]]; then - echo "Pattern file missing" - exit 1 -fi - -#this awk strips entries from the vocabulary if they contain invalid characters -#invalid characters are digits and punctuation marks, and words beginning or ending with a dash -#uniq -c extracts the unique words and counts the occurrences - -if [[ $reallen -eq 0 ]]; then - #when a zero is passed, use the whole file - zcat -f $srcname | sed 's/ /\n/g' | egrep -v -f $pattern_file | sort | uniq -c | sed 's/^ *//' - -else - zcat -f $srcname | head -n $reallen | sed 's/ /\n/g' | egrep -v -f $pattern_file | sort | uniq -c | sed 's/^ *//' -fi - diff --git a/gi/pf/Makefile.am b/gi/pf/Makefile.am deleted file mode 100644 index 86f8e07b..00000000 --- a/gi/pf/Makefile.am +++ /dev/null @@ -1,44 +0,0 @@ -bin_PROGRAMS = cbgi brat dpnaive pfbrat pfdist itg pfnaive condnaive align-lexonly-pyp learn_cfg pyp_lm nuisance_test align-tl pf_test bayes_lattice_score - -noinst_LIBRARIES = libpf.a - -libpf_a_SOURCES = base_distributions.cc reachability.cc cfg_wfst_composer.cc corpus.cc unigrams.cc ngram_base.cc transliterations.cc backward.cc hpyp_tm.cc pyp_tm.cc - -bayes_lattice_score_SOURCES = bayes_lattice_score.cc -bayes_lattice_score_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -pf_test_SOURCES = pf_test.cc -pf_test_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -nuisance_test_SOURCES = nuisance_test.cc -nuisance_test_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -align_lexonly_pyp_SOURCES = align-lexonly-pyp.cc -align_lexonly_pyp_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -align_tl_SOURCES = align-tl.cc -align_tl_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -itg_SOURCES = itg.cc - -pyp_lm_SOURCES = pyp_lm.cc - -learn_cfg_SOURCES = learn_cfg.cc - -condnaive_SOURCES = condnaive.cc - -dpnaive_SOURCES = dpnaive.cc - -pfdist_SOURCES = pfdist.cc - -pfnaive_SOURCES = pfnaive.cc - -cbgi_SOURCES = cbgi.cc - -brat_SOURCES = brat.cc - -pfbrat_SOURCES = pfbrat.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -I$(top_srcdir)/decoder -I$(top_srcdir)/klm - -AM_LDFLAGS = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/pf/README b/gi/pf/README deleted file mode 100644 index 62e47541..00000000 --- a/gi/pf/README +++ /dev/null @@ -1,2 +0,0 @@ -Experimental Bayesian alignment tools. Nothing to see here. - diff --git a/gi/pf/align-lexonly-pyp.cc b/gi/pf/align-lexonly-pyp.cc deleted file mode 100644 index e7509f57..00000000 --- a/gi/pf/align-lexonly-pyp.cc +++ /dev/null @@ -1,243 +0,0 @@ -#include -#include - -#include -#include - -#include "tdict.h" -#include "stringlib.h" -#include "filelib.h" -#include "array2d.h" -#include "sampler.h" -#include "corpus.h" -#include "pyp_tm.h" -#include "hpyp_tm.h" -#include "quasi_model2.h" - -using namespace std; -namespace po = boost::program_options; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("infer_alignment_hyperparameters,I", "Infer alpha and p_null, otherwise fixed values will be assumed") - ("p_null,0", po::value()->default_value(0.08), "probability of aligning to null") - ("align_alpha,a", po::value()->default_value(4.0), "how 'tight' is the bias toward be along the diagonal?") - ("input,i",po::value(),"Read parallel data from") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -MT19937* prng; - -struct LexicalAlignment { - unsigned char src_index; - bool is_transliteration; - vector > derivation; -}; - -struct AlignedSentencePair { - vector src; - vector trg; - vector a; - Array2D posterior; -}; - -template -struct Aligner { - Aligner(const vector >& lets, - int vocab_size, - int num_letters, - const po::variables_map& conf, - vector* c) : - corpus(*c), - paj_model(conf["align_alpha"].as(), conf["p_null"].as()), - infer_paj(conf.count("infer_alignment_hyperparameters") > 0), - model(lets, vocab_size, num_letters), - kNULL(TD::Convert("NULL")) { - assert(lets[kNULL].size() == 0); - } - - vector& corpus; - QuasiModel2 paj_model; - const bool infer_paj; - LexicalTranslationModel model; - const WordID kNULL; - - void ResampleHyperparameters() { - model.ResampleHyperparameters(prng); - if (infer_paj) paj_model.ResampleHyperparameters(prng); - } - - void InitializeRandom() { - cerr << "Initializing with random alignments ...\n"; - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - asp.a.resize(asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - unsigned char& a_j = asp.a[j].src_index; - a_j = prng->next() * (1 + asp.src.size()); - const WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Increment(f_a_j, asp.trg[j], &*prng); - paj_model.Increment(a_j, j, asp.src.size(), asp.trg.size()); - } - } - cerr << "Corpus intialized randomly." << endl; - cerr << "LLH = " << Likelihood() << " \t(Amodel=" << paj_model.Likelihood() - << " TModel=" << model.Likelihood() << ") contexts=" << model.UniqueConditioningContexts() << endl; - } - - void ResampleCorpus() { - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - SampleSet ss; ss.resize(asp.src.size() + 1); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - unsigned char& a_j = asp.a[j].src_index; - const WordID e_j = asp.trg[j]; - WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Decrement(f_a_j, e_j, prng); - paj_model.Decrement(a_j, j, asp.src.size(), asp.trg.size()); - - for (unsigned prop_a_j = 0; prop_a_j <= asp.src.size(); ++prop_a_j) { - const WordID prop_f = (prop_a_j ? asp.src[prop_a_j - 1] : kNULL); - ss[prop_a_j] = model.Prob(prop_f, e_j); - ss[prop_a_j] *= paj_model.Prob(prop_a_j, j, asp.src.size(), asp.trg.size()); - } - a_j = prng->SelectSample(ss); - f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Increment(f_a_j, e_j, prng); - paj_model.Increment(a_j, j, asp.src.size(), asp.trg.size()); - } - } - } - - prob_t Likelihood() const { - return model.Likelihood() * paj_model.Likelihood(); - } -}; - -void ExtractLetters(const set& v, vector >* l, set* letset = NULL) { - for (set::const_iterator it = v.begin(); it != v.end(); ++it) { - vector& letters = (*l)[*it]; - if (letters.size()) continue; // if e and f have the same word - - const string& w = TD::Convert(*it); - - size_t cur = 0; - while (cur < w.size()) { - const size_t len = UTF8Len(w[cur]); - letters.push_back(TD::Convert(w.substr(cur, len))); - if (letset) letset->insert(letters.back()); - cur += len; - } - } -} - -void Debug(const AlignedSentencePair& asp) { - cerr << TD::GetString(asp.src) << endl << TD::GetString(asp.trg) << endl; - Array2D a(asp.src.size(), asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - assert(asp.a[j].src_index <= asp.src.size()); - if (asp.a[j].src_index) a(asp.a[j].src_index - 1, j) = true; - } - cerr << a << endl; -} - -void AddSample(AlignedSentencePair* asp) { - for (unsigned j = 0; j < asp->trg.size(); ++j) - asp->posterior(asp->a[j].src_index, j)++; -} - -void WriteAlignments(const AlignedSentencePair& asp) { - bool first = true; - for (unsigned j = 0; j < asp.trg.size(); ++j) { - int src_index = -1; - int mc = -1; - for (unsigned i = 0; i <= asp.src.size(); ++i) { - if (asp.posterior(i, j) > mc) { - mc = asp.posterior(i, j); - src_index = i; - } - } - - if (src_index) { - if (first) first = false; else cout << ' '; - cout << (src_index - 1) << '-' << j; - } - } - cout << endl; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - - if (conf.count("random_seed")) - prng = new MT19937(conf["random_seed"].as()); - else - prng = new MT19937; - - vector > corpuse, corpusf; - set vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - vector corpus(corpuse.size()); - for (unsigned i = 0; i < corpuse.size(); ++i) { - corpus[i].src.swap(corpusf[i]); - corpus[i].trg.swap(corpuse[i]); - corpus[i].posterior.resize(corpus[i].src.size() + 1, corpus[i].trg.size()); - } - corpusf.clear(); corpuse.clear(); - - vocabf.insert(TD::Convert("NULL")); - vector > letters(TD::NumWords()); - set letset; - ExtractLetters(vocabe, &letters, &letset); - ExtractLetters(vocabf, &letters, NULL); - letters[TD::Convert("NULL")].clear(); - - //Aligner aligner(letters, vocabe.size(), letset.size(), conf, &corpus); - Aligner aligner(letters, vocabe.size(), letset.size(), conf, &corpus); - aligner.InitializeRandom(); - - const unsigned samples = conf["samples"].as(); - for (int i = 0; i < samples; ++i) { - for (int j = 65; j < 67; ++j) Debug(corpus[j]); - if (i % 10 == 9) { - aligner.ResampleHyperparameters(); - cerr << "LLH = " << aligner.Likelihood() << " \t(Amodel=" << aligner.paj_model.Likelihood() - << " TModel=" << aligner.model.Likelihood() << ") contexts=" << aligner.model.UniqueConditioningContexts() << endl; - } - aligner.ResampleCorpus(); - if (i > (samples / 5) && (i % 6 == 5)) for (int j = 0; j < corpus.size(); ++j) AddSample(&corpus[j]); - } - for (unsigned i = 0; i < corpus.size(); ++i) - WriteAlignments(corpus[i]); - aligner.model.Summary(); - - return 0; -} diff --git a/gi/pf/align-tl.cc b/gi/pf/align-tl.cc deleted file mode 100644 index f6608f1d..00000000 --- a/gi/pf/align-tl.cc +++ /dev/null @@ -1,339 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "backward.h" -#include "array2d.h" -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "conditional_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "stringlib.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "mfcr.h" -#include "corpus.h" -#include "ngram_base.h" -#include "transliterations.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("s2t", po::value(), "character level source-to-target prior transliteration probabilities") - ("t2s", po::value(), "character level target-to-source prior transliteration probabilities") - ("max_src_chunk", po::value()->default_value(4), "Maximum size of translitered chunk in source") - ("max_trg_chunk", po::value()->default_value(4), "Maximum size of translitered chunk in target") - ("expected_src_to_trg_ratio", po::value()->default_value(1.0), "If a word is transliterated, what is the expected length ratio from source to target?") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr prng; - -struct LexicalAlignment { - unsigned char src_index; - bool is_transliteration; - vector > derivation; -}; - -struct AlignedSentencePair { - vector src; - vector trg; - vector a; - Array2D posterior; -}; - -struct HierarchicalWordBase { - explicit HierarchicalWordBase(const unsigned vocab_e_size) : - base(prob_t::One()), r(1,1,1,1,0.66,50.0), u0(-log(vocab_e_size)), l(1,prob_t::One()), v(1, prob_t::Zero()) {} - - void ResampleHyperparameters(MT19937* rng) { - r.resample_hyperparameters(rng); - } - - inline double logp0(const vector& s) const { - return Md::log_poisson(s.size(), 7.5) + s.size() * u0; - } - - // return p0 of rule.e_ - prob_t operator()(const TRule& rule) const { - v[0].logeq(logp0(rule.e_)); - return r.prob(rule.e_, v.begin(), l.begin()); - } - - void Increment(const TRule& rule) { - v[0].logeq(logp0(rule.e_)); - if (r.increment(rule.e_, v.begin(), l.begin(), &*prng).count) { - base *= v[0] * l[0]; - } - } - - void Decrement(const TRule& rule) { - if (r.decrement(rule.e_, &*prng).count) { - base /= prob_t(exp(logp0(rule.e_))); - } - } - - prob_t Likelihood() const { - prob_t p; p.logeq(r.log_crp_prob()); - p *= base; - return p; - } - - void Summary() const { - cerr << "NUMBER OF CUSTOMERS: " << r.num_customers() << " (d=" << r.discount() << ",s=" << r.strength() << ')' << endl; - for (MFCR<1,vector >::const_iterator it = r.begin(); it != r.end(); ++it) - cerr << " " << it->second.total_dish_count_ << " (on " << it->second.table_counts_.size() << " tables) " << TD::GetString(it->first) << endl; - } - - prob_t base; - MFCR<1,vector > r; - const double u0; - const vector l; - mutable vector v; -}; - -struct BasicLexicalAlignment { - explicit BasicLexicalAlignment(const vector >& lets, - const unsigned words_e, - const unsigned letters_e, - vector* corp) : - letters(lets), - corpus(*corp), - //up0(words_e), - //up0("en.chars.1gram", letters_e), - //up0("en.words.1gram"), - up0(letters_e), - //up0("en.chars.2gram"), - tmodel(up0) { - } - - void InstantiateRule(const WordID src, - const WordID trg, - TRule* rule) const { - static const WordID kX = TD::Convert("X") * -1; - rule->lhs_ = kX; - rule->e_ = letters[trg]; - rule->f_ = letters[src]; - } - - void InitializeRandom() { - const WordID kNULL = TD::Convert("NULL"); - cerr << "Initializing with random alignments ...\n"; - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - asp.a.resize(asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - const unsigned char a_j = prng->next() * (1 + asp.src.size()); - const WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - TRule r; - InstantiateRule(f_a_j, asp.trg[j], &r); - asp.a[j].is_transliteration = false; - asp.a[j].src_index = a_j; - if (tmodel.IncrementRule(r, &*prng)) - up0.Increment(r); - } - } - cerr << " LLH = " << Likelihood() << endl; - } - - prob_t Likelihood() const { - prob_t p = tmodel.Likelihood(); - p *= up0.Likelihood(); - return p; - } - - void ResampleHyperparemeters() { - tmodel.ResampleHyperparameters(&*prng); - up0.ResampleHyperparameters(&*prng); - cerr << " (base d=" << up0.r.discount() << ",s=" << up0.r.strength() << ")\n"; - } - - void ResampleCorpus(); - - const vector >& letters; // spelling dictionary - vector& corpus; - //PhraseConditionalUninformativeBase up0; - //PhraseConditionalUninformativeUnigramBase up0; - //UnigramWordBase up0; - //HierarchicalUnigramBase up0; - HierarchicalWordBase up0; - //CompletelyUniformBase up0; - //FixedNgramBase up0; - //ConditionalTranslationModel tmodel; - //ConditionalTranslationModel tmodel; - //ConditionalTranslationModel tmodel; - //ConditionalTranslationModel tmodel; - MConditionalTranslationModel tmodel; - //ConditionalTranslationModel tmodel; - //ConditionalTranslationModel tmodel; -}; - -void BasicLexicalAlignment::ResampleCorpus() { - static const WordID kNULL = TD::Convert("NULL"); - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - SampleSet ss; ss.resize(asp.src.size() + 1); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - TRule r; - unsigned char& a_j = asp.a[j].src_index; - WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - InstantiateRule(f_a_j, asp.trg[j], &r); - if (tmodel.DecrementRule(r, &*prng)) - up0.Decrement(r); - - for (unsigned prop_a_j = 0; prop_a_j <= asp.src.size(); ++prop_a_j) { - const WordID prop_f = (prop_a_j ? asp.src[prop_a_j - 1] : kNULL); - InstantiateRule(prop_f, asp.trg[j], &r); - ss[prop_a_j] = tmodel.RuleProbability(r); - } - a_j = prng->SelectSample(ss); - f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - InstantiateRule(f_a_j, asp.trg[j], &r); - if (tmodel.IncrementRule(r, &*prng)) - up0.Increment(r); - } - } - cerr << " LLH = " << Likelihood() << endl; -} - -void ExtractLetters(const set& v, vector >* l, set* letset = NULL) { - for (set::const_iterator it = v.begin(); it != v.end(); ++it) { - vector& letters = (*l)[*it]; - if (letters.size()) continue; // if e and f have the same word - - const string& w = TD::Convert(*it); - - size_t cur = 0; - while (cur < w.size()) { - const size_t len = UTF8Len(w[cur]); - letters.push_back(TD::Convert(w.substr(cur, len))); - if (letset) letset->insert(letters.back()); - cur += len; - } - } -} - -void Debug(const AlignedSentencePair& asp) { - cerr << TD::GetString(asp.src) << endl << TD::GetString(asp.trg) << endl; - Array2D a(asp.src.size(), asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) - if (asp.a[j].src_index) a(asp.a[j].src_index - 1, j) = true; - cerr << a << endl; -} - -void AddSample(AlignedSentencePair* asp) { - for (unsigned j = 0; j < asp->trg.size(); ++j) - asp->posterior(asp->a[j].src_index, j)++; -} - -void WriteAlignments(const AlignedSentencePair& asp) { - bool first = true; - for (unsigned j = 0; j < asp.trg.size(); ++j) { - int src_index = -1; - int mc = -1; - for (unsigned i = 0; i <= asp.src.size(); ++i) { - if (asp.posterior(i, j) > mc) { - mc = asp.posterior(i, j); - src_index = i; - } - } - - if (src_index) { - if (first) first = false; else cout << ' '; - cout << (src_index - 1) << '-' << j; - } - } - cout << endl; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - vector corpus(corpuse.size()); - for (unsigned i = 0; i < corpuse.size(); ++i) { - corpus[i].src.swap(corpusf[i]); - corpus[i].trg.swap(corpuse[i]); - corpus[i].posterior.resize(corpus[i].src.size() + 1, corpus[i].trg.size()); - } - corpusf.clear(); corpuse.clear(); - - vocabf.insert(TD::Convert("NULL")); - vector > letters(TD::NumWords() + 1); - set letset; - ExtractLetters(vocabe, &letters, &letset); - ExtractLetters(vocabf, &letters, NULL); - letters[TD::Convert("NULL")].clear(); - - // TODO configure this - const int max_src_chunk = conf["max_src_chunk"].as(); - const int max_trg_chunk = conf["max_trg_chunk"].as(); - const double s2t_rat = conf["expected_src_to_trg_ratio"].as(); - const BackwardEstimator be(conf["s2t"].as(), conf["t2s"].as()); - Transliterations tl(max_src_chunk, max_trg_chunk, s2t_rat, be); - - cerr << "Initializing transliteration graph structures ...\n"; - for (int i = 0; i < corpus.size(); ++i) { - const vector& src = corpus[i].src; - const vector& trg = corpus[i].trg; - for (int j = 0; j < src.size(); ++j) { - const vector& src_let = letters[src[j]]; - for (int k = 0; k < trg.size(); ++k) { - const vector& trg_let = letters[trg[k]]; - tl.Initialize(src[j], src_let, trg[k], trg_let); - //if (src_let.size() < min_trans_src) - // tl.Forbid(src[j], src_let, trg[k], trg_let); - } - } - } - cerr << endl; - tl.GraphSummary(); - - return 0; -} diff --git a/gi/pf/backward.cc b/gi/pf/backward.cc deleted file mode 100644 index b92629fd..00000000 --- a/gi/pf/backward.cc +++ /dev/null @@ -1,89 +0,0 @@ -#include "backward.h" - -#include -#include - -#include "array2d.h" -#include "reachability.h" -#include "base_distributions.h" - -using namespace std; - -BackwardEstimator::BackwardEstimator(const string& s2t, - const string& t2s) : m1(new Model1(s2t)), m1inv(new Model1(t2s)) {} - -BackwardEstimator::~BackwardEstimator() { - delete m1; m1 = NULL; - delete m1inv; m1inv = NULL; -} - -float BackwardEstimator::ComputeBackwardProb(const std::vector& src, - const std::vector& trg, - unsigned src_covered, - unsigned trg_covered, - double s2t_ratio) const { - if (src_covered == src.size() || trg_covered == trg.size()) { - assert(src_covered == src.size()); - assert(trg_covered == trg.size()); - return 0; - } - static const WordID kNULL = TD::Convert(""); - const prob_t uniform_alignment(1.0 / (src.size() - src_covered + 1)); - // TODO factor in expected length ratio - prob_t e; e.logeq(Md::log_poisson(trg.size() - trg_covered, (src.size() - src_covered) * s2t_ratio)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_covered; j < trg.size(); ++j) { - prob_t p = (*m1)(kNULL, trg[j]) + prob_t(1e-12); - for (unsigned i = src_covered; i < src.size(); ++i) - p += (*m1)(src[i], trg[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg[j]) << " | " << TD::GetString(src) << ") = 0!\n"; - assert(!"failed"); - } - p *= uniform_alignment; - e *= p; - } - // TODO factor in expected length ratio - const prob_t inv_uniform(1.0 / (trg.size() - trg_covered + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(src.size() - src_covered, (trg.size() - trg_covered) / s2t_ratio)); - for (unsigned i = src_covered; i < src.size(); ++i) { - prob_t p = (*m1inv)(kNULL, src[i]) + prob_t(1e-12); - for (unsigned j = trg_covered; j < trg.size(); ++j) - p += (*m1inv)(trg[j], src[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(src[i]) << " | " << TD::GetString(trg) << ") = 0!\n"; - assert(!"failed"); - } - p *= inv_uniform; - inv *= p; - } - return (log(e) + log(inv)) / 2; -} - -void BackwardEstimator::InitializeGrid(const vector& src, - const vector& trg, - const Reachability& r, - double s2t_ratio, - float* grid) const { - queue > q; - q.push(make_pair(0,0)); - Array2D done(src.size()+1, trg.size()+1, false); - //cerr << TD::GetString(src) << " ||| " << TD::GetString(trg) << endl; - while(!q.empty()) { - const pair n = q.front(); - q.pop(); - if (done(n.first,n.second)) continue; - done(n.first,n.second) = true; - - float lp = ComputeBackwardProb(src, trg, n.first, n.second, s2t_ratio); - if (n.first == 0 && n.second == 0) grid[0] = lp; - //cerr << " " << n.first << "," << n.second << "\t" << lp << endl; - - if (n.first == src.size() || n.second == trg.size()) continue; - const vector >& edges = r.valid_deltas[n.first][n.second]; - for (int i = 0; i < edges.size(); ++i) - q.push(make_pair(n.first + edges[i].first, n.second + edges[i].second)); - } - //static int cc = 0; ++cc; if (cc == 80) exit(1); -} - diff --git a/gi/pf/backward.h b/gi/pf/backward.h deleted file mode 100644 index e67eff0c..00000000 --- a/gi/pf/backward.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _BACKWARD_H_ -#define _BACKWARD_H_ - -#include -#include -#include "wordid.h" - -struct Reachability; -struct Model1; - -struct BackwardEstimator { - BackwardEstimator(const std::string& s2t, - const std::string& t2s); - ~BackwardEstimator(); - - void InitializeGrid(const std::vector& src, - const std::vector& trg, - const Reachability& r, - double src2trg_ratio, - float* grid) const; - - private: - float ComputeBackwardProb(const std::vector& src, - const std::vector& trg, - unsigned src_covered, - unsigned trg_covered, - double src2trg_ratio) const; - - Model1* m1; - Model1* m1inv; -}; - -#endif diff --git a/gi/pf/base_distributions.cc b/gi/pf/base_distributions.cc deleted file mode 100644 index 57e0bbe1..00000000 --- a/gi/pf/base_distributions.cc +++ /dev/null @@ -1,241 +0,0 @@ -#include "base_distributions.h" - -#include - -#include "filelib.h" - -using namespace std; - -TableLookupBase::TableLookupBase(const string& fname) { - cerr << "TableLookupBase reading from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - vector le, lf; - TRule x; - x.lhs_ = -TD::Convert("X"); - bool flag = false; - while(getline(in, line)) { - ++lc; - if (lc % 1000000 == 0) { cerr << " [" << lc << ']' << endl; flag = false; } - else if (lc % 25000 == 0) { cerr << '.' << flush; flag = true; } - tmp.clear(); - TD::ConvertSentence(line, &tmp); - x.f_.clear(); - x.e_.clear(); - size_t pos = 0; - int cc = 0; - while(pos < tmp.size()) { - const WordID cur = tmp[pos++]; - if (cur == kDIV) { - ++cc; - } else if (cc == 0) { - x.f_.push_back(cur); - } else if (cc == 1) { - x.e_.push_back(cur); - } else if (cc == 2) { - table[x].logeq(atof(TD::Convert(cur).c_str())); - ++cc; - } else { - if (flag) cerr << endl; - cerr << "Bad format in " << lc << ": " << line << endl; abort(); - } - } - if (cc != 3) { - if (flag) cerr << endl; - cerr << "Bad format in " << lc << ": " << line << endl; abort(); - } - } - if (flag) cerr << endl; - cerr << " read " << lc << " entries\n"; -} - -prob_t PhraseConditionalUninformativeUnigramBase::p0(const vector& vsrc, - const vector& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t p; - p.logeq(Md::log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - //p.logeq(log_poisson(elen, 1)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) - p *= u(vtrg[i + start_trg]); // draw e_i ~Uniform - return p; -} - -prob_t PhraseConditionalUninformativeBase::p0(const vector& vsrc, - const vector& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t p; - //p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - p.logeq(Md::log_poisson(elen, 1)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) - p *= kUNIFORM_TARGET; // draw e_i ~Uniform - return p; -} - -void Model1::LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; -} - -prob_t PhraseConditionalBase::p0(const vector& vsrc, - const vector& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(Md::log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - return p; -} - -prob_t PhraseJointBase::p0(const vector& vsrc, - const vector& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(Md::log_poisson(flen, 1.0)); // flen ~Pois(1) - // elen | flen ~Pois(flen + 0.01) - prob_t ptrglen; ptrglen.logeq(Md::log_poisson(elen, flen + 0.01)); - p *= ptrglen; - p *= kUNIFORM_SOURCE.pow(flen); // each f in F ~Uniform - for (int i = 0; i < elen; ++i) { // for each position i in E - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - return p; -} - -prob_t PhraseJointBase_BiDir::p0(const vector& vsrc, - const vector& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t uniform_trg_alignment; uniform_trg_alignment.logeq(-log(elen + 1)); - - prob_t p1; - p1.logeq(Md::log_poisson(flen, 1.0)); // flen ~Pois(1) - // elen | flen ~Pois(flen + 0.01) - prob_t ptrglen; ptrglen.logeq(Md::log_poisson(elen, flen + 0.01)); - p1 *= ptrglen; - p1 *= kUNIFORM_SOURCE.pow(flen); // each f in F ~Uniform - for (int i = 0; i < elen; ++i) { // for each position i in E - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p1 *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p1.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - - prob_t p2; - p2.logeq(Md::log_poisson(elen, 1.0)); // elen ~Pois(1) - // flen | elen ~Pois(flen + 0.01) - prob_t psrclen; psrclen.logeq(Md::log_poisson(flen, elen + 0.01)); - p2 *= psrclen; - p2 *= kUNIFORM_TARGET.pow(elen); // each f in F ~Uniform - for (int i = 0; i < flen; ++i) { // for each position i in E - const WordID src = vsrc[i + start_src]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < elen; ++j) { - const WordID trg = j < 0 ? 0 : vtrg[j + start_trg]; - tp += kM1MIXTURE * invmodel1(trg, src); - tp += kUNIFORM_MIXTURE * kUNIFORM_SOURCE; - } - tp *= uniform_trg_alignment; // draw a_i ~uniform - p2 *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p2.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - - static const prob_t kHALF(0.5); - return (p1 + p2) * kHALF; -} - -JumpBase::JumpBase() : p(200) { - for (unsigned src_len = 1; src_len < 200; ++src_len) { - map& cpd = p[src_len]; - int min_jump = 1 - src_len; - int max_jump = src_len; - prob_t z; - for (int j = min_jump; j <= max_jump; ++j) { - prob_t& cp = cpd[j]; - if (j < 0) - cp.logeq(Md::log_poisson(1.5-j, 1)); - else if (j > 0) - cp.logeq(Md::log_poisson(j, 1)); - cp.poweq(0.2); - z += cp; - } - for (int j = min_jump; j <= max_jump; ++j) { - cpd[j] /= z; - } - } -} - diff --git a/gi/pf/base_distributions.h b/gi/pf/base_distributions.h deleted file mode 100644 index 41b513f8..00000000 --- a/gi/pf/base_distributions.h +++ /dev/null @@ -1,238 +0,0 @@ -#ifndef _BASE_MEASURES_H_ -#define _BASE_MEASURES_H_ - -#include -#include -#include -#include -#include -#include - -#include "unigrams.h" -#include "trule.h" -#include "prob.h" -#include "tdict.h" -#include "sampler.h" -#include "m.h" -#include "os_phrase.h" - -struct Model1 { - explicit Model1(const std::string& fname) : - kNULL(TD::Convert("")), - kZERO() { - LoadModel1(fname); - } - - void LoadModel1(const std::string& fname); - - // returns prob 0 if src or trg is not found - const prob_t& operator()(WordID src, WordID trg) const { - if (src == 0) src = kNULL; - if (src < ttable.size()) { - const std::map& cpd = ttable[src]; - const std::map::const_iterator it = cpd.find(trg); - if (it != cpd.end()) - return it->second; - } - return kZERO; - } - - const WordID kNULL; - const prob_t kZERO; - std::vector > ttable; -}; - -struct PoissonUniformUninformativeBase { - explicit PoissonUniformUninformativeBase(const unsigned ves) : kUNIFORM(1.0 / ves) {} - prob_t operator()(const TRule& r) const { - prob_t p; p.logeq(Md::log_poisson(r.e_.size(), 1.0)); - prob_t q = kUNIFORM; q.poweq(r.e_.size()); - p *= q; - return p; - } - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM; -}; - -struct CompletelyUniformBase { - explicit CompletelyUniformBase(const unsigned ves) : kUNIFORM(1.0 / ves) {} - prob_t operator()(const TRule&) const { - return kUNIFORM; - } - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM; -}; - -struct UnigramWordBase { - explicit UnigramWordBase(const std::string& fname) : un(fname) {} - prob_t operator()(const TRule& r) const { - return un(r.e_); - } - const UnigramWordModel un; -}; - -struct RuleHasher { - size_t operator()(const TRule& r) const { - return hash_value(r); - } -}; - -struct TableLookupBase { - TableLookupBase(const std::string& fname); - - prob_t operator()(const TRule& rule) const { - const std::tr1::unordered_map::const_iterator it = table.find(rule); - if (it == table.end()) { - std::cerr << rule << " not found\n"; - abort(); - } - return it->second; - } - - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - void Summary() const {} - - std::tr1::unordered_map table; -}; - -struct PhraseConditionalUninformativeBase { - explicit PhraseConditionalUninformativeBase(const unsigned vocab_e_size) : - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; - - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseConditionalUninformativeUnigramBase { - explicit PhraseConditionalUninformativeUnigramBase(const std::string& file, const unsigned vocab_e_size) : u(file, vocab_e_size) {} - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; - - const UnigramModel u; -}; - -struct PhraseConditionalBase { - explicit PhraseConditionalBase(const Model1& m1, const double m1mixture, const unsigned vocab_e_size) : - model1(m1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseJointBase { - explicit PhraseJointBase(const Model1& m1, const double m1mixture, const unsigned vocab_e_size, const unsigned vocab_f_size) : - model1(m1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_SOURCE(1.0 / vocab_f_size), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ , rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_SOURCE; - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseJointBase_BiDir { - explicit PhraseJointBase_BiDir(const Model1& m1, - const Model1& im1, - const double m1mixture, - const unsigned vocab_e_size, - const unsigned vocab_f_size) : - model1(m1), - invmodel1(im1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_SOURCE(1.0 / vocab_f_size), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ , rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector& vsrc, const std::vector& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const Model1& invmodel1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_SOURCE; - const prob_t kUNIFORM_TARGET; -}; - -// base distribution for jump size multinomials -// basically p(0) = 0 and then, p(1) is max, and then -// you drop as you move to the max jump distance -struct JumpBase { - JumpBase(); - - const prob_t& operator()(int jump, unsigned src_len) const { - assert(jump != 0); - const std::map::const_iterator it = p[src_len].find(jump); - assert(it != p[src_len].end()); - return it->second; - } - std::vector > p; -}; - - -#endif diff --git a/gi/pf/bayes_lattice_score.cc b/gi/pf/bayes_lattice_score.cc deleted file mode 100644 index 70cb8dc2..00000000 --- a/gi/pf/bayes_lattice_score.cc +++ /dev/null @@ -1,309 +0,0 @@ -#include -#include - -#include -#include -#include - -#include "inside_outside.h" -#include "hg.h" -#include "hg_io.h" -#include "bottom_up_parser.h" -#include "fdict.h" -#include "grammar.h" -#include "m.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -unsigned ReadCorpus(const string& filename, - vector* e, - set* vocab_e) { - e->clear(); - vocab_e->clear(); - ReadFile rf(filename); - istream* in = rf.stream(); - assert(*in); - string line; - unsigned toks = 0; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(Lattice()); - Lattice& le = e->back(); - LatticeTools::ConvertTextOrPLF(line, & le); - for (unsigned i = 0; i < le.size(); ++i) - for (unsigned j = 0; j < le[i].size(); ++j) - vocab_e->insert(le[i][j].label); - toks += le.size(); - } - return toks; -} - -struct BaseModel { - explicit BaseModel(unsigned tc) : - unif(1.0 / tc), p(prob_t::One()) {} - prob_t prob(const TRule& r) const { - return unif; - } - void increment(const TRule& r, MT19937* rng) { - p *= prob(r); - } - void decrement(const TRule& r, MT19937* rng) { - p /= prob(r); - } - prob_t Likelihood() const { - return p; - } - const prob_t unif; - prob_t p; -}; - -struct UnigramModel { - explicit UnigramModel(unsigned tc) : base(tc), crp(1,1,1,1), glue(1,1,1,1) {} - BaseModel base; - CCRP crp; - CCRP glue; - - prob_t Prob(const TRule& r) const { - if (r.Arity() != 0) { - return glue.prob(r, prob_t(0.5)); - } - return crp.prob(r, base.prob(r)); - } - - int Increment(const TRule& r, MT19937* rng) { - if (r.Arity() != 0) { - glue.increment(r, 0.5, rng); - return 0; - } else { - if (crp.increment(r, base.prob(r), rng)) { - base.increment(r, rng); - return 1; - } - return 0; - } - } - - int Decrement(const TRule& r, MT19937* rng) { - if (r.Arity() != 0) { - glue.decrement(r, rng); - return 0; - } else { - if (crp.decrement(r, rng)) { - base.decrement(r, rng); - return -1; - } - return 0; - } - } - - prob_t Likelihood() const { - prob_t p; - p.logeq(crp.log_crp_prob() + glue.log_crp_prob()); - p *= base.Likelihood(); - return p; - } - - void ResampleHyperparameters(MT19937* rng) { - crp.resample_hyperparameters(rng); - glue.resample_hyperparameters(rng); - cerr << " d=" << crp.discount() << ", s=" << crp.strength() << "\t STOP d=" << glue.discount() << ", s=" << glue.strength() << endl; - } -}; - -UnigramModel* plm; - -void SampleDerivation(const Hypergraph& hg, MT19937* rng, vector* sampled_deriv) { - vector node_probs; - Inside(hg, &node_probs); - queue q; - q.push(hg.nodes_.size() - 2); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - //prob_t z; - assert(num_in_edges > 1); - SampleSet ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = edge.edge_prob_; - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - //z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } -// for (unsigned i = 0; i < sampled_deriv->size(); ++i) { -// cerr << *hg.edges_[(*sampled_deriv)[i]].rule_ << endl; -// } -} - -void IncrementDerivation(const Hypergraph& hg, const vector& d, UnigramModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Increment(*hg.edges_[d[i]].rule_, rng); -} - -void DecrementDerivation(const Hypergraph& hg, const vector& d, UnigramModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Decrement(*hg.edges_[d[i]].rule_, rng); -} - -prob_t TotalProb(const Hypergraph& hg) { - return Inside(hg); -} - -void IncrementLatticePath(const Hypergraph& hg, const vector& d, Lattice* pl) { - Lattice& lat = *pl; - for (int i = 0; i < d.size(); ++i) { - const Hypergraph::Edge& edge = hg.edges_[d[i]]; - if (edge.rule_->Arity() != 0) continue; - WordID sym = edge.rule_->e_[0]; - vector& las = lat[edge.i_]; - int dist = edge.j_ - edge.i_; - assert(dist > 0); - for (int j = 0; j < las.size(); ++j) { - if (las[j].dist2next == dist && - las[j].label == sym) { - las[j].cost += 1; - } - } - } -} - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - vector grammars(2); - grammars[0].reset(new GlueGrammar("S","X")); - const unsigned samples = conf["samples"].as(); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector corpuse; - set vocabe; - cerr << "Reading corpus...\n"; - const unsigned toks = ReadCorpus(conf["input"].as(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " lattices\t (" << vocabe.size() << " word types)\n"; - UnigramModel lm(vocabe.size()); - vector hgs(corpuse.size()); - vector > derivs(corpuse.size()); - for (int i = 0; i < corpuse.size(); ++i) { - grammars[1].reset(new PassThroughGrammar(corpuse[i], "X")); - ExhaustiveBottomUpParser parser("S", grammars); - bool res = parser.Parse(corpuse[i], &hgs[i]); // exhaustive parse - assert(res); - } - - double csamples = 0; - for (int SS=0; SS < samples; ++SS) { - const bool is_last = ((samples - 1) == SS); - prob_t dlh = prob_t::One(); - bool record_sample = (SS > (samples * 1 / 3) && (SS % 5 == 3)); - if (record_sample) csamples++; - for (int ci = 0; ci < corpuse.size(); ++ci) { - Lattice& lat = corpuse[ci]; - Hypergraph& hg = hgs[ci]; - vector& d = derivs[ci]; - if (!is_last) DecrementDerivation(hg, d, &lm, &rng); - for (unsigned i = 0; i < hg.edges_.size(); ++i) { - TRule& r = *hg.edges_[i].rule_; - if (r.Arity() != 0) - hg.edges_[i].edge_prob_ = prob_t::One(); - else - hg.edges_[i].edge_prob_ = lm.Prob(r); - } - if (!is_last) { - d.clear(); - SampleDerivation(hg, &rng, &d); - IncrementDerivation(hg, derivs[ci], &lm, &rng); - } else { - prob_t p = TotalProb(hg); - dlh *= p; - cerr << " p(sentence) = " << log(p) << "\t" << log(dlh) << endl; - } - if (record_sample) IncrementLatticePath(hg, derivs[ci], &lat); - } - double llh = log(lm.Likelihood()); - cerr << "LLH=" << llh << "\tENTROPY=" << (-llh / log(2) / toks) << "\tPPL=" << pow(2, -llh / log(2) / toks) << endl; - if (SS % 10 == 9) lm.ResampleHyperparameters(&rng); - if (is_last) { - double z = log(dlh); - cerr << "TOTAL_PROB=" << z << "\tENTROPY=" << (-z / log(2) / toks) << "\tPPL=" << pow(2, -z / log(2) / toks) << endl; - } - } - cerr << lm.crp << endl; - cerr << lm.glue << endl; - for (int i = 0; i < corpuse.size(); ++i) { - for (int j = 0; j < corpuse[i].size(); ++j) - for (int k = 0; k < corpuse[i][j].size(); ++k) { - corpuse[i][j][k].cost /= csamples; - corpuse[i][j][k].cost += 1e-3; - corpuse[i][j][k].cost = log(corpuse[i][j][k].cost); - } - cout << HypergraphIO::AsPLF(corpuse[i]) << endl; - } - return 0; -} - diff --git a/gi/pf/brat.cc b/gi/pf/brat.cc deleted file mode 100644 index 832f22cf..00000000 --- a/gi/pf/brat.cc +++ /dev/null @@ -1,543 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "cfg_wfst_composer.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; -struct FSTState; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -struct ConditionalBase { - explicit ConditionalBase(const double m1mixture, const unsigned vocab_e_size, const string& model1fname) : - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size), - kNULL(TD::Convert("")) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - LoadModel1(model1fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // return logp0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - const int flen = rule.f_.size(); - const int elen = rule.e_.size(); - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = rule.e_[i]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? kNULL : rule.f_[j]; - const map::const_iterator it = ttable[src].find(trg); - if (it != ttable[src].end()) { - tp += kM1MIXTURE * it->second; - } - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - return p; - } - - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; - const WordID kNULL; - vector > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(3),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(3),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UniphraseLM { - UniphraseLM(const vector >& corpus, - const set& vocab, - const po::variables_map& conf) : - phrases_(1,1), - gen_(1,1), - corpus_(corpus), - uniform_word_(1.0 / vocab.size()), - gen_p0_(0.5), - p_end_(0.5), - use_poisson_(conf.count("poisson_length") > 0) {} - - void ResampleHyperparameters(MT19937* rng) { - phrases_.resample_hyperparameters(rng); - gen_.resample_hyperparameters(rng); - cerr << " " << phrases_.alpha(); - } - - CCRP_NoTable > phrases_; - CCRP_NoTable gen_; - vector > z_; // z_[i] is there a phrase boundary after the ith word - const vector >& corpus_; - const double uniform_word_; - const double gen_p0_; - const double p_end_; // in base length distribution, p of the end of a phrase - const bool use_poisson_; -}; - -struct Reachability { - boost::multi_array edges; // edges[src_covered][trg_covered][x][trg_delta] is this edge worth exploring? - boost::multi_array max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; - }; - - struct NState { - NState() : next_src_covered(), next_trg_covered() {} - NState(int i, int j) : next_src_covered(i), next_trg_covered(j) {} - int next_src_covered; - int next_trg_covered; - }; - - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - cerr << "Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - assert(a[srclen][trglen].size() > 0); - - typedef boost::multi_array rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); -// typedef boost::multi_array, 2> narray_type; -// narray_type b(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } -// const NState nstate(i,j); - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; -// b[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(nstate); - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - cerr << " MAX SRC DELTA[0][0] = " << max_src_delta[0][0] << endl; - assert(max_src_delta[0][0] > 0); - //cerr << "First cell contains " << b[0][0].size() << " forward pointers\n"; - //for (int i = 0; i < b[0][0].size(); ++i) { - // cerr << " -> (" << b[0][0][i].next_src_covered << "," << b[0][0][i].next_trg_covered << ")\n"; - //} - } -}; - -ostream& operator<<(ostream& os, const FSTState& q); -struct FSTState { - explicit FSTState(int src_size) : - trg_covered_(), - src_covered_(), - src_coverage_(src_size) {} - - FSTState(short trg_covered, short src_covered, const vector& src_coverage, const vector& src_prefix) : - trg_covered_(trg_covered), - src_covered_(src_covered), - src_coverage_(src_coverage), - src_prefix_(src_prefix) { - if (src_coverage_.size() == src_covered) { - assert(src_prefix.size() == 0); - } - } - - // if we extend by the word at src_position, what are - // the next states that are reachable and lie on a valid - // path to the final state? - vector Extensions(int src_position, int src_len, int trg_len, const Reachability& r) const { - assert(src_position < src_coverage_.size()); - if (src_coverage_[src_position]) { - cerr << "Trying to extend " << *this << " with position " << src_position << endl; - abort(); - } - vector ncvg = src_coverage_; - ncvg[src_position] = true; - - vector res; - const int trg_remaining = trg_len - trg_covered_; - if (trg_remaining <= 0) { - cerr << "Target appears to have been covered: " << *this << " (trg_len=" << trg_len << ",trg_covered=" << trg_covered_ << ")" << endl; - abort(); - } - const int src_remaining = src_len - src_covered_; - if (src_remaining <= 0) { - cerr << "Source appears to have been covered: " << *this << endl; - abort(); - } - - for (int tc = 1; tc <= kMAX_TRG_PHRASE; ++tc) { - if (r.edges[src_covered_][trg_covered_][src_prefix_.size() + 1][tc]) { - int nc = src_prefix_.size() + 1 + src_covered_; - res.push_back(FSTState(trg_covered_ + tc, nc, ncvg, vector())); - } - } - - if ((src_prefix_.size() + 1) < r.max_src_delta[src_covered_][trg_covered_]) { - vector nsp = src_prefix_; - nsp.push_back(src_position); - res.push_back(FSTState(trg_covered_, src_covered_, ncvg, nsp)); - } - - if (res.size() == 0) { - cerr << *this << " can't be extended!\n"; - abort(); - } - return res; - } - - short trg_covered_, src_covered_; - vector src_coverage_; - vector src_prefix_; -}; -bool operator<(const FSTState& q, const FSTState& r) { - if (q.trg_covered_ != r.trg_covered_) return q.trg_covered_ < r.trg_covered_; - if (q.src_covered_!= r.src_covered_) return q.src_covered_ < r.src_covered_; - if (q.src_coverage_ != r.src_coverage_) return q.src_coverage_ < r.src_coverage_; - return q.src_prefix_ < r.src_prefix_; -} - -ostream& operator<<(ostream& os, const FSTState& q) { - os << "[" << q.trg_covered_ << " : "; - for (int i = 0; i < q.src_coverage_.size(); ++i) - os << q.src_coverage_[i]; - os << " : <"; - for (int i = 0; i < q.src_prefix_.size(); ++i) { - if (i != 0) os << ' '; - os << q.src_prefix_[i]; - } - return os << ">]"; -} - -struct MyModel { - MyModel(ConditionalBase& rcp0) : rp0(rcp0) {} - typedef unordered_map, CCRP_NoTable, boost::hash > > SrcToRuleCRPMap; - - void DecrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - it->second.decrement(rule); - if (it->second.num_customers() == 0) rules.erase(it); - } - - void IncrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) { - CCRP_NoTable crp(1,1); - it = rules.insert(make_pair(rule.f_, crp)).first; - } - it->second.increment(rule); - } - - // conditioned on rule.f_ - prob_t RuleConditionalProbability(const TRule& rule) const { - const prob_t base = rp0(rule); - SrcToRuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) { - return base; - } else { - const double lp = it->second.logprob(rule, log(base)); - prob_t q; q.logeq(lp); - return q; - } - } - - const ConditionalBase& rp0; - SrcToRuleCRPMap rules; -}; - -struct MyFST : public WFST { - MyFST(const vector& ssrc, const vector& strg, MyModel* m) : - src(ssrc), trg(strg), - r(src.size(),trg.size(),kMAX_SRC_PHRASE, kMAX_TRG_PHRASE), - model(m) { - FSTState in(src.size()); - cerr << " INIT: " << in << endl; - init = GetNode(in); - for (int i = 0; i < in.src_coverage_.size(); ++i) in.src_coverage_[i] = true; - in.src_covered_ = src.size(); - in.trg_covered_ = trg.size(); - cerr << "FINAL: " << in << endl; - final = GetNode(in); - } - virtual const WFSTNode* Final() const; - virtual const WFSTNode* Initial() const; - - const WFSTNode* GetNode(const FSTState& q); - map > m; - const vector& src; - const vector& trg; - Reachability r; - const WFSTNode* init; - const WFSTNode* final; - MyModel* model; -}; - -struct MyNode : public WFSTNode { - MyNode(const FSTState& q, MyFST* fst) : state(q), container(fst) {} - virtual vector > ExtendInput(unsigned srcindex) const; - const FSTState state; - mutable MyFST* container; -}; - -vector > MyNode::ExtendInput(unsigned srcindex) const { - cerr << "EXTEND " << state << " with " << srcindex << endl; - vector ext = state.Extensions(srcindex, container->src.size(), container->trg.size(), container->r); - vector > res(ext.size()); - for (unsigned i = 0; i < ext.size(); ++i) { - res[i].first = container->GetNode(ext[i]); - if (ext[i].src_prefix_.size() == 0) { - const unsigned trg_from = state.trg_covered_; - const unsigned trg_to = ext[i].trg_covered_; - const unsigned prev_prfx_size = state.src_prefix_.size(); - res[i].second.reset(new TRule); - res[i].second->lhs_ = -TD::Convert("X"); - vector& src = res[i].second->f_; - vector& trg = res[i].second->e_; - src.resize(prev_prfx_size + 1); - for (unsigned j = 0; j < prev_prfx_size; ++j) - src[j] = container->src[state.src_prefix_[j]]; - src[prev_prfx_size] = container->src[srcindex]; - for (unsigned j = trg_from; j < trg_to; ++j) - trg.push_back(container->trg[j]); - res[i].second->scores_.set_value(FD::Convert("Proposal"), log(container->model->RuleConditionalProbability(*res[i].second))); - } - } - return res; -} - -const WFSTNode* MyFST::GetNode(const FSTState& q) { - boost::shared_ptr& res = m[q]; - if (!res) { - res.reset(new MyNode(q, this)); - } - return &*res; -} - -const WFSTNode* MyFST::Final() const { - return final; -} - -const WFSTNode* MyFST::Initial() const { - return init; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - ConditionalBase lp0(conf["model1_interpolation_weight"].as(), - vocabe.size(), - conf["model1"].as()); - MyModel m(lp0); - - TRule x("[X] ||| kAnwntR myN ||| at the convent ||| 0"); - m.IncrementRule(x); - TRule y("[X] ||| nY dyN ||| gave ||| 0"); - m.IncrementRule(y); - - - MyFST fst(corpusf[0], corpuse[0], &m); - ifstream in("./kimura.g"); - assert(in); - CFG_WFSTComposer comp(fst); - Hypergraph hg; - bool succeed = comp.Compose(&in, &hg); - hg.PrintGraphviz(); - if (succeed) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } - -#if 0 - ifstream in2("./amnabooks.g"); - assert(in2); - MyFST fst2(corpusf[1], corpuse[1], &m); - CFG_WFSTComposer comp2(fst2); - Hypergraph hg2; - bool succeed2 = comp2.Compose(&in2, &hg2); - if (succeed2) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } -#endif - - SparseVector w; w.set_value(FD::Convert("Proposal"), 1.0); - hg.Reweight(w); - cerr << ViterbiFTree(hg) << endl; - return 0; -} - diff --git a/gi/pf/cbgi.cc b/gi/pf/cbgi.cc deleted file mode 100644 index 97f1ba34..00000000 --- a/gi/pf/cbgi.cc +++ /dev/null @@ -1,330 +0,0 @@ -#include -#include -#include - -#include -#include - -#include "sampler.h" -#include "filelib.h" -#include "hg_io.h" -#include "hg.h" -#include "ccrp_nt.h" -#include "trule.h" -#include "inside_outside.h" - -using namespace std; -using namespace std::tr1; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -double log_decay(unsigned x, const double& b) { - assert(b > 1.0); - assert(x > 0); - return log(b - 1) - x * log(b); -} - -struct SimpleBase { - SimpleBase(unsigned esize, unsigned fsize, unsigned ntsize = 144) : - uniform_e(-log(esize)), - uniform_f(-log(fsize)), - uniform_nt(-log(ntsize)) { - } - - // binomial coefficient - static double choose(unsigned n, unsigned k) { - return exp(lgamma(n + 1) - lgamma(k + 1) - lgamma(n - k + 1)); - } - - // count the number of patterns of terminals and NTs in the rule, given elen and flen - static double log_number_of_patterns(const unsigned flen, const unsigned elen) { - static vector > counts; - if (elen >= counts.size()) counts.resize(elen + 1); - if (flen >= counts[elen].size()) counts[elen].resize(flen + 1); - double& count = counts[elen][flen]; - if (count) return log(count); - const unsigned max_arity = min(elen, flen); - for (unsigned a = 0; a <= max_arity; ++a) - count += choose(elen, a) * choose(flen, a); - return log(count); - } - - // return logp0 of rule | LHS - double operator()(const TRule& rule) const { - const unsigned flen = rule.f_.size(); - const unsigned elen = rule.e_.size(); -#if 0 - double p = 0; - p += log_poisson(flen, 0.5); // flen ~Pois(0.5) - p += log_poisson(elen, flen); // elen | flen ~Pois(flen) - p -= log_number_of_patterns(flen, elen); // pattern | flen,elen ~Uniform - for (unsigned i = 0; i < flen; ++i) { // for each position in f-RHS - if (rule.f_[i] <= 0) // according to pattern - p += uniform_nt; // draw NT ~Uniform - else - p += uniform_f; // draw f terminal ~Uniform - } - p -= lgamma(rule.Arity() + 1); // draw permutation ~Uniform - for (unsigned i = 0; i < elen; ++i) { // for each position in e-RHS - if (rule.e_[i] > 0) // according to pattern - p += uniform_e; // draw e|f term ~Uniform - // TODO this should prob be model 1 - } -#else - double p = 0; - bool is_abstract = rule.f_[0] <= 0; - p += log(0.5); - if (is_abstract) { - if (flen == 2) p += log(0.99); else p += log(0.01); - } else { - p += log_decay(flen, 3); - } - - for (unsigned i = 0; i < flen; ++i) { // for each position in f-RHS - if (rule.f_[i] <= 0) // according to pattern - p += uniform_nt; // draw NT ~Uniform - else - p += uniform_f; // draw f terminal ~Uniform - } -#endif - return p; - } - const double uniform_e; - const double uniform_f; - const double uniform_nt; - vector arities; -}; - -MT19937* rng = NULL; - -template -struct MHSamplerEdgeProb { - MHSamplerEdgeProb(const Hypergraph& hg, - const map >& rdp, - const Base& logp0, - const bool exclude_multiword_terminals) : edge_probs(hg.edges_.size()) { - for (int i = 0; i < edge_probs.size(); ++i) { - const TRule& rule = *hg.edges_[i].rule_; - const map >::const_iterator it = rdp.find(rule.lhs_); - assert(it != rdp.end()); - const CCRP_NoTable& crp = it->second; - edge_probs[i].logeq(crp.logprob(rule, logp0(rule))); - if (exclude_multiword_terminals && rule.f_[0] > 0 && rule.f_.size() > 1) - edge_probs[i] = prob_t::Zero(); - } - } - inline prob_t operator()(const Hypergraph::Edge& e) const { - return edge_probs[e.id_]; - } - prob_t DerivationProb(const vector& d) const { - prob_t p = prob_t::One(); - for (unsigned i = 0; i < d.size(); ++i) - p *= edge_probs[d[i]]; - return p; - } - vector edge_probs; -}; - -template -struct ModelAndData { - ModelAndData() : - base_lh(prob_t::One()), - logp0(10000, 10000), - mh_samples(), - mh_rejects() {} - - void SampleCorpus(const string& hgpath, int i); - void ResampleHyperparameters() { - for (map >::iterator it = rules.begin(); it != rules.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - CCRP_NoTable& RuleCRP(int lhs) { - map >::iterator it = rules.find(lhs); - if (it == rules.end()) { - rules.insert(make_pair(lhs, CCRP_NoTable(1,1))); - it = rules.find(lhs); - } - return it->second; - } - - void IncrementRule(const TRule& rule) { - CCRP_NoTable& crp = RuleCRP(rule.lhs_); - if (crp.increment(rule)) { - prob_t p; p.logeq(logp0(rule)); - base_lh *= p; - } - } - - void DecrementRule(const TRule& rule) { - CCRP_NoTable& crp = RuleCRP(rule.lhs_); - if (crp.decrement(rule)) { - prob_t p; p.logeq(logp0(rule)); - base_lh /= p; - } - } - - void DecrementDerivation(const Hypergraph& hg, const vector& d) { - for (unsigned i = 0; i < d.size(); ++i) { - const TRule& rule = *hg.edges_[d[i]].rule_; - DecrementRule(rule); - } - } - - void IncrementDerivation(const Hypergraph& hg, const vector& d) { - for (unsigned i = 0; i < d.size(); ++i) { - const TRule& rule = *hg.edges_[d[i]].rule_; - IncrementRule(rule); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (map >::const_iterator it = rules.begin(); it != rules.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - p *= base_lh; - return p; - } - - void ResampleDerivation(const Hypergraph& hg, vector* sampled_derivation); - - map > rules; // [lhs] -> distribution over RHSs - prob_t base_lh; - SimpleBase logp0; - vector > samples; // sampled derivations - unsigned int mh_samples; - unsigned int mh_rejects; -}; - -template -void ModelAndData::SampleCorpus(const string& hgpath, int n) { - vector hgs(n); hgs.clear(); - boost::unordered_map acc; - map tot; - for (int i = 0; i < n; ++i) { - ostringstream os; - os << hgpath << '/' << i << ".json.gz"; - if (!FileExists(os.str())) continue; - hgs.push_back(Hypergraph()); - ReadFile rf(os.str()); - HypergraphIO::ReadFromJSON(rf.stream(), &hgs.back()); - } - cerr << "Read " << hgs.size() << " alignment hypergraphs.\n"; - samples.resize(hgs.size()); - const unsigned SAMPLES = 2000; - const unsigned burnin = 3 * SAMPLES / 4; - const unsigned every = 20; - for (unsigned s = 0; s < SAMPLES; ++s) { - if (s % 10 == 0) { - if (s > 0) { cerr << endl; ResampleHyperparameters(); } - cerr << "[" << s << " LLH=" << log(Likelihood()) << " REJECTS=" << ((double)mh_rejects / mh_samples) << " LHS's=" << rules.size() << " base=" << log(base_lh) << "] "; - } - cerr << '.'; - for (unsigned i = 0; i < hgs.size(); ++i) { - ResampleDerivation(hgs[i], &samples[i]); - if (s > burnin && s % every == 0) { - for (unsigned j = 0; j < samples[i].size(); ++j) { - const TRule& rule = *hgs[i].edges_[samples[i][j]].rule_; - ++acc[rule]; - ++tot[rule.lhs_]; - } - } - } - } - cerr << endl; - for (boost::unordered_map::iterator it = acc.begin(); it != acc.end(); ++it) { - cout << it->first << " MyProb=" << log(it->second)-log(tot[it->first.lhs_]) << endl; - } -} - -template -void ModelAndData::ResampleDerivation(const Hypergraph& hg, vector* sampled_deriv) { - vector cur; - cur.swap(*sampled_deriv); - - const prob_t p_cur = Likelihood(); - DecrementDerivation(hg, cur); - if (cur.empty()) { - // first iteration, create restaurants - for (int i = 0; i < hg.edges_.size(); ++i) - RuleCRP(hg.edges_[i].rule_->lhs_); - } - MHSamplerEdgeProb wf(hg, rules, logp0, cur.empty()); -// MHSamplerEdgeProb wf(hg, rules, logp0, false); - const prob_t q_cur = wf.DerivationProb(cur); - vector node_probs; - Inside >(hg, &node_probs, wf); - queue q; - q.push(hg.nodes_.size() - 3); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - prob_t z; - assert(num_in_edges > 1); - SampleSet ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = wf.edge_probs[edge.id_]; // edge proposal prob - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } - IncrementDerivation(hg, *sampled_deriv); - -// cerr << "sampled derivation contains " << sampled_deriv->size() << " edges\n"; -// cerr << "DERIV:\n"; -// for (int i = 0; i < sampled_deriv->size(); ++i) { -// cerr << " " << hg.edges_[(*sampled_deriv)[i]].rule_->AsString() << endl; -// } - - if (cur.empty()) return; // accept first sample - - ++mh_samples; - // only need to do MH if proposal is different to current state - if (cur != *sampled_deriv) { - const prob_t q_prop = wf.DerivationProb(*sampled_deriv); - const prob_t p_prop = Likelihood(); - if (!rng->AcceptMetropolisHastings(p_prop, p_cur, q_prop, q_cur)) { - ++mh_rejects; - DecrementDerivation(hg, *sampled_deriv); - IncrementDerivation(hg, cur); - swap(cur, *sampled_deriv); - } - } -} - -int main(int argc, char** argv) { - rng = new MT19937; - ModelAndData m; - m.SampleCorpus("./hgs", 50); - // m.SampleCorpus("./btec/hgs", 5000); - return 0; -} - diff --git a/gi/pf/cfg_wfst_composer.cc b/gi/pf/cfg_wfst_composer.cc deleted file mode 100644 index 21d5ec5b..00000000 --- a/gi/pf/cfg_wfst_composer.cc +++ /dev/null @@ -1,731 +0,0 @@ -#include "cfg_wfst_composer.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "fast_lexical_cast.hpp" - -#include "phrasetable_fst.h" -#include "sparse_vector.h" -#include "tdict.h" -#include "hg.h" -#include "hg_remove_eps.h" - -namespace po = boost::program_options; -using namespace std; -using namespace std::tr1; - -WFSTNode::~WFSTNode() {} -WFST::~WFST() {} - -// Define the following macro if you want to see lots of debugging output -// when you run the chart parser -#undef DEBUG_CHART_PARSER - -// A few constants used by the chart parser /////////////// -static const int kMAX_NODES = 2000000; -static const string kPHRASE_STRING = "X"; -static bool constants_need_init = true; -static WordID kUNIQUE_START; -static WordID kPHRASE; -static TRulePtr kX1X2; -static TRulePtr kX1; -static WordID kEPS; -static TRulePtr kEPSRule; - -static void InitializeConstants() { - if (constants_need_init) { - kPHRASE = TD::Convert(kPHRASE_STRING) * -1; - kUNIQUE_START = TD::Convert("S") * -1; - kX1X2.reset(new TRule("[X] ||| [X,1] [X,2] ||| [X,1] [X,2]")); - kX1.reset(new TRule("[X] ||| [X,1] ||| [X,1]")); - kEPSRule.reset(new TRule("[X] ||| ||| ")); - kEPS = TD::Convert(""); - constants_need_init = false; - } -} -//////////////////////////////////////////////////////////// - -class EGrammarNode { - friend bool CFG_WFSTComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest); - friend void AddGrammarRule(const string& r, map* g); - public: -#ifdef DEBUG_CHART_PARSER - string hint; -#endif - EGrammarNode() : is_some_rule_complete(false), is_root(false) {} - const map& GetTerminals() const { return tptr; } - const map& GetNonTerminals() const { return ntptr; } - bool HasNonTerminals() const { return (!ntptr.empty()); } - bool HasTerminals() const { return (!tptr.empty()); } - bool RuleCompletes() const { - return (is_some_rule_complete || (ntptr.empty() && tptr.empty())); - } - bool GrammarContinues() const { - return !(ntptr.empty() && tptr.empty()); - } - bool IsRoot() const { - return is_root; - } - // these are the features associated with the rule from the start - // node up to this point. If you use these features, you must - // not Extend() this rule. - const SparseVector& GetCFGProductionFeatures() const { - return input_features; - } - - const EGrammarNode* Extend(const WordID& t) const { - if (t < 0) { - map::const_iterator it = ntptr.find(t); - if (it == ntptr.end()) return NULL; - return &it->second; - } else { - map::const_iterator it = tptr.find(t); - if (it == tptr.end()) return NULL; - return &it->second; - } - } - - private: - map tptr; - map ntptr; - SparseVector input_features; - bool is_some_rule_complete; - bool is_root; -}; -typedef map EGrammar; // indexed by the rule LHS - -// edges are immutable once created -struct Edge { -#ifdef DEBUG_CHART_PARSER - static int id_count; - const int id; -#endif - const WordID cat; // lhs side of rule proved/being proved - const EGrammarNode* const dot; // dot position - const WFSTNode* const q; // start of span - const WFSTNode* const r; // end of span - const Edge* const active_parent; // back pointer, NULL for PREDICT items - const Edge* const passive_parent; // back pointer, NULL for SCAN and PREDICT items - TRulePtr tps; // translations - boost::shared_ptr > features; // features from CFG rule - - bool IsPassive() const { - // when a rule is completed, this value will be set - return static_cast(features); - } - bool IsActive() const { return !IsPassive(); } - bool IsInitial() const { - return !(active_parent || passive_parent); - } - bool IsCreatedByScan() const { - return active_parent && !passive_parent && !dot->IsRoot(); - } - bool IsCreatedByPredict() const { - return dot->IsRoot(); - } - bool IsCreatedByComplete() const { - return active_parent && passive_parent; - } - - // constructor for PREDICT - Edge(WordID c, const EGrammarNode* d, const WFSTNode* q_and_r) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(NULL), passive_parent(NULL), tps() {} - Edge(WordID c, const EGrammarNode* d, const WFSTNode* q_and_r, const Edge* act_parent) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(act_parent), passive_parent(NULL), tps() {} - - // constructors for SCAN - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const TRulePtr& translations) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations) {} - - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const TRulePtr& translations, - const SparseVector& feats) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations), - features(new SparseVector(feats)) {} - - // constructors for COMPLETE - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const Edge *pas_par) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps() { - assert(pas_par->IsPassive()); - assert(act_par->IsActive()); - } - - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const Edge *pas_par, const SparseVector& feats) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps(), - features(new SparseVector(feats)) { - assert(pas_par->IsPassive()); - assert(act_par->IsActive()); - } - - // constructor for COMPLETE query - Edge(const WFSTNode* _r) : -#ifdef DEBUG_CHART_PARSER - id(0), -#endif - cat(0), dot(NULL), q(NULL), - r(_r), active_parent(NULL), passive_parent(NULL), tps() {} - // constructor for MERGE quere - Edge(const WFSTNode* _q, int) : -#ifdef DEBUG_CHART_PARSER - id(0), -#endif - cat(0), dot(NULL), q(_q), - r(NULL), active_parent(NULL), passive_parent(NULL), tps() {} -}; -#ifdef DEBUG_CHART_PARSER -int Edge::id_count = 0; -#endif - -ostream& operator<<(ostream& os, const Edge& e) { - string type = "PREDICT"; - if (e.IsCreatedByScan()) - type = "SCAN"; - else if (e.IsCreatedByComplete()) - type = "COMPLETE"; - os << "[" -#ifdef DEBUG_CHART_PARSER - << '(' << e.id << ") " -#else - << '(' << &e << ") " -#endif - << "q=" << e.q << ", r=" << e.r - << ", cat="<< TD::Convert(e.cat*-1) << ", dot=" - << e.dot -#ifdef DEBUG_CHART_PARSER - << e.dot->hint -#endif - << (e.IsActive() ? ", Active" : ", Passive") - << ", " << type; -#ifdef DEBUG_CHART_PARSER - if (e.active_parent) { os << ", act.parent=(" << e.active_parent->id << ')'; } - if (e.passive_parent) { os << ", psv.parent=(" << e.passive_parent->id << ')'; } -#endif - if (e.tps) { os << ", tps=" << e.tps->AsString(); } - return os << ']'; -} - -struct Traversal { - const Edge* const edge; // result from the active / passive combination - const Edge* const active; - const Edge* const passive; - Traversal(const Edge* me, const Edge* a, const Edge* p) : edge(me), active(a), passive(p) {} -}; - -struct UniqueTraversalHash { - size_t operator()(const Traversal* t) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast(t->active); - x = ((x << 5) + x) ^ reinterpret_cast(t->passive); - x = ((x << 5) + x) ^ t->edge->IsActive(); - return x; - } -}; - -struct UniqueTraversalEquals { - size_t operator()(const Traversal* a, const Traversal* b) const { - return (a->passive == b->passive && a->active == b->active && a->edge->IsActive() == b->edge->IsActive()); - } -}; - -struct UniqueEdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - if (e->IsActive()) { - x = ((x << 5) + x) ^ reinterpret_cast(e->dot); - x = ((x << 5) + x) ^ reinterpret_cast(e->q); - x = ((x << 5) + x) ^ reinterpret_cast(e->r); - x = ((x << 5) + x) ^ static_cast(e->cat); - x += 13; - } else { // with passive edges, we don't care about the dot - x = ((x << 5) + x) ^ reinterpret_cast(e->q); - x = ((x << 5) + x) ^ reinterpret_cast(e->r); - x = ((x << 5) + x) ^ static_cast(e->cat); - } - return x; - } -}; - -struct UniqueEdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - if (a->IsActive() != b->IsActive()) return false; - if (a->IsActive()) { - return (a->cat == b->cat) && (a->dot == b->dot) && (a->q == b->q) && (a->r == b->r); - } else { - return (a->cat == b->cat) && (a->q == b->q) && (a->r == b->r); - } - } -}; - -struct REdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast(e->r); - return x; - } -}; - -struct REdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - return (a->r == b->r); - } -}; - -struct QEdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast(e->q); - return x; - } -}; - -struct QEdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - return (a->q == b->q); - } -}; - -struct EdgeQueue { - queue q; - EdgeQueue() {} - void clear() { while(!q.empty()) q.pop(); } - bool HasWork() const { return !q.empty(); } - const Edge* Next() { const Edge* res = q.front(); q.pop(); return res; } - void AddEdge(const Edge* s) { q.push(s); } -}; - -class CFG_WFSTComposerImpl { - public: - CFG_WFSTComposerImpl(WordID start_cat, - const WFSTNode* q_0, - const WFSTNode* q_final) : start_cat_(start_cat), q_0_(q_0), q_final_(q_final) {} - - // returns false if the intersection is empty - bool Compose(const EGrammar& g, Hypergraph* forest) { - goal_node = NULL; - EGrammar::const_iterator sit = g.find(start_cat_); - forest->ReserveNodes(kMAX_NODES); - assert(sit != g.end()); - Edge* init = new Edge(start_cat_, &sit->second, q_0_); - assert(IncorporateNewEdge(init)); - while (exp_agenda.HasWork() || agenda.HasWork()) { - while(exp_agenda.HasWork()) { - const Edge* edge = exp_agenda.Next(); - FinishEdge(edge, forest); - } - if (agenda.HasWork()) { - const Edge* edge = agenda.Next(); -#ifdef DEBUG_CHART_PARSER - cerr << "processing (" << edge->id << ')' << endl; -#endif - if (edge->IsActive()) { - if (edge->dot->HasTerminals()) - DoScan(edge); - if (edge->dot->HasNonTerminals()) { - DoMergeWithPassives(edge); - DoPredict(edge, g); - } - } else { - DoComplete(edge); - } - } - } - if (goal_node) { - forest->PruneUnreachable(goal_node->id_); - RemoveEpsilons(forest, kEPS); - } - FreeAll(); - return goal_node; - } - - void FreeAll() { - for (int i = 0; i < free_list_.size(); ++i) - delete free_list_[i]; - free_list_.clear(); - for (int i = 0; i < traversal_free_list_.size(); ++i) - delete traversal_free_list_[i]; - traversal_free_list_.clear(); - all_traversals.clear(); - exp_agenda.clear(); - agenda.clear(); - tps2node.clear(); - edge2node.clear(); - all_edges.clear(); - passive_edges.clear(); - active_edges.clear(); - } - - ~CFG_WFSTComposerImpl() { - FreeAll(); - } - - // returns the total number of edges created during composition - int EdgesCreated() const { - return free_list_.size(); - } - - private: - void DoScan(const Edge* edge) { - // here, we assume that the FST will potentially have many more outgoing - // edges than the grammar, which will be just a couple. If you want to - // efficiently handle the case where both are relatively large, this code - // will need to change how the intersection is done. The best general - // solution would probably be the Baeza-Yates double binary search. - - const EGrammarNode* dot = edge->dot; - const WFSTNode* r = edge->r; - const map& terms = dot->GetTerminals(); - for (map::const_iterator git = terms.begin(); - git != terms.end(); ++git) { - - if (!(TD::Convert(git->first)[0] >= '0' && TD::Convert(git->first)[0] <= '9')) { - std::cerr << "TERMINAL SYMBOL: " << TD::Convert(git->first) << endl; - abort(); - } - std::vector > extensions = r->ExtendInput(atoi(TD::Convert(git->first).c_str())); - for (unsigned nsi = 0; nsi < extensions.size(); ++nsi) { - const WFSTNode* next_r = extensions[nsi].first; - const EGrammarNode* next_dot = &git->second; - const bool grammar_continues = next_dot->GrammarContinues(); - const bool rule_completes = next_dot->RuleCompletes(); - if (extensions[nsi].second) - cerr << "!!! " << extensions[nsi].second->AsString() << endl; - // cerr << " rule completes: " << rule_completes << " after consuming " << TD::Convert(git->first) << endl; - assert(grammar_continues || rule_completes); - const SparseVector& input_features = next_dot->GetCFGProductionFeatures(); - if (rule_completes) - IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, extensions[nsi].second, input_features)); - if (grammar_continues) - IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, extensions[nsi].second)); - } - } - } - - void DoPredict(const Edge* edge, const EGrammar& g) { - const EGrammarNode* dot = edge->dot; - const map& non_terms = dot->GetNonTerminals(); - for (map::const_iterator git = non_terms.begin(); - git != non_terms.end(); ++git) { - const WordID nt_to_predict = git->first; - //cerr << edge->id << " -- " << TD::Convert(nt_to_predict*-1) << endl; - EGrammar::const_iterator egi = g.find(nt_to_predict); - if (egi == g.end()) { - cerr << "[ERROR] Can't find any grammar rules with a LHS of type " - << TD::Convert(-1*nt_to_predict) << '!' << endl; - continue; - } - assert(edge->IsActive()); - const EGrammarNode* new_dot = &egi->second; - Edge* new_edge = new Edge(nt_to_predict, new_dot, edge->r, edge); - IncorporateNewEdge(new_edge); - } - } - - void DoComplete(const Edge* passive) { -#ifdef DEBUG_CHART_PARSER - cerr << " complete: " << *passive << endl; -#endif - const WordID completed_nt = passive->cat; - const WFSTNode* q = passive->q; - const WFSTNode* next_r = passive->r; - const Edge query(q); - const pair::iterator, - unordered_multiset::iterator > p = - active_edges.equal_range(&query); - for (unordered_multiset::iterator it = p.first; - it != p.second; ++it) { - const Edge* active = *it; -#ifdef DEBUG_CHART_PARSER - cerr << " pos: " << *active << endl; -#endif - const EGrammarNode* next_dot = active->dot->Extend(completed_nt); - if (!next_dot) continue; - const SparseVector& input_features = next_dot->GetCFGProductionFeatures(); - // add up to 2 rules - if (next_dot->RuleCompletes()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features)); - if (next_dot->GrammarContinues()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive)); - } - } - - void DoMergeWithPassives(const Edge* active) { - // edge is active, has non-terminals, we need to find the passives that can extend it - assert(active->IsActive()); - assert(active->dot->HasNonTerminals()); -#ifdef DEBUG_CHART_PARSER - cerr << " merge active with passives: ACT=" << *active << endl; -#endif - const Edge query(active->r, 1); - const pair::iterator, - unordered_multiset::iterator > p = - passive_edges.equal_range(&query); - for (unordered_multiset::iterator it = p.first; - it != p.second; ++it) { - const Edge* passive = *it; - const EGrammarNode* next_dot = active->dot->Extend(passive->cat); - if (!next_dot) continue; - const WFSTNode* next_r = passive->r; - const SparseVector& input_features = next_dot->GetCFGProductionFeatures(); - if (next_dot->RuleCompletes()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features)); - if (next_dot->GrammarContinues()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive)); - } - } - - // take ownership of edge memory, add to various indexes, etc - // returns true if this edge is new - bool IncorporateNewEdge(Edge* edge) { - free_list_.push_back(edge); - if (edge->passive_parent && edge->active_parent) { - Traversal* t = new Traversal(edge, edge->active_parent, edge->passive_parent); - traversal_free_list_.push_back(t); - if (all_traversals.find(t) != all_traversals.end()) { - return false; - } else { - all_traversals.insert(t); - } - } - exp_agenda.AddEdge(edge); - return true; - } - - bool FinishEdge(const Edge* edge, Hypergraph* hg) { - bool is_new = false; - if (all_edges.find(edge) == all_edges.end()) { -#ifdef DEBUG_CHART_PARSER - cerr << *edge << " is NEW\n"; -#endif - all_edges.insert(edge); - is_new = true; - if (edge->IsPassive()) passive_edges.insert(edge); - if (edge->IsActive()) active_edges.insert(edge); - agenda.AddEdge(edge); - } else { -#ifdef DEBUG_CHART_PARSER - cerr << *edge << " is NOT NEW.\n"; -#endif - } - AddEdgeToTranslationForest(edge, hg); - return is_new; - } - - // build the translation forest - void AddEdgeToTranslationForest(const Edge* edge, Hypergraph* hg) { - assert(hg->nodes_.size() < kMAX_NODES); - Hypergraph::Node* tps = NULL; - // first add any target language rules - if (edge->tps) { - Hypergraph::Node*& node = tps2node[(size_t)edge->tps.get()]; - if (!node) { - // cerr << "Creating phrases for " << edge->tps << endl; - const TRulePtr& rule = edge->tps; - node = hg->AddNode(kPHRASE); - Hypergraph::Edge* hg_edge = hg->AddEdge(rule, Hypergraph::TailNodeVector()); - hg_edge->feature_values_ += rule->GetFeatureValues(); - hg->ConnectEdgeToHeadNode(hg_edge, node); - } - tps = node; - } - Hypergraph::Node*& head_node = edge2node[edge]; - if (!head_node) - head_node = hg->AddNode(kPHRASE); - if (edge->cat == start_cat_ && edge->q == q_0_ && edge->r == q_final_ && edge->IsPassive()) { - assert(goal_node == NULL || goal_node == head_node); - goal_node = head_node; - } - Hypergraph::TailNodeVector tail; - SparseVector extra; - if (edge->IsCreatedByPredict()) { - // extra.set_value(FD::Convert("predict"), 1); - } else if (edge->IsCreatedByScan()) { - tail.push_back(edge2node[edge->active_parent]->id_); - if (tps) { - tail.push_back(tps->id_); - } - //extra.set_value(FD::Convert("scan"), 1); - } else if (edge->IsCreatedByComplete()) { - tail.push_back(edge2node[edge->active_parent]->id_); - tail.push_back(edge2node[edge->passive_parent]->id_); - //extra.set_value(FD::Convert("complete"), 1); - } else { - assert(!"unexpected edge type!"); - } - //cerr << head_node->id_ << "<--" << *edge << endl; - -#ifdef DEBUG_CHART_PARSER - for (int i = 0; i < tail.size(); ++i) - if (tail[i] == head_node->id_) { - cerr << "ERROR: " << *edge << "\n i=" << i << endl; - if (i == 1) { cerr << "\tP: " << *edge->passive_parent << endl; } - if (i == 0) { cerr << "\tA: " << *edge->active_parent << endl; } - assert(!"self-loop found!"); - } -#endif - Hypergraph::Edge* hg_edge = NULL; - if (tail.size() == 0) { - hg_edge = hg->AddEdge(kEPSRule, tail); - } else if (tail.size() == 1) { - hg_edge = hg->AddEdge(kX1, tail); - } else if (tail.size() == 2) { - hg_edge = hg->AddEdge(kX1X2, tail); - } - if (edge->features) - hg_edge->feature_values_ += *edge->features; - hg_edge->feature_values_ += extra; - hg->ConnectEdgeToHeadNode(hg_edge, head_node); - } - - Hypergraph::Node* goal_node; - EdgeQueue exp_agenda; - EdgeQueue agenda; - unordered_map tps2node; - unordered_map edge2node; - unordered_set all_traversals; - unordered_set all_edges; - unordered_multiset passive_edges; - unordered_multiset active_edges; - vector free_list_; - vector traversal_free_list_; - const WordID start_cat_; - const WFSTNode* const q_0_; - const WFSTNode* const q_final_; -}; - -#ifdef DEBUG_CHART_PARSER -static string TrimRule(const string& r) { - size_t start = r.find(" |||") + 5; - size_t end = r.rfind(" |||"); - return r.substr(start, end - start); -} -#endif - -void AddGrammarRule(const string& r, EGrammar* g) { - const size_t pos = r.find(" ||| "); - if (pos == string::npos || r[0] != '[') { - cerr << "Bad rule: " << r << endl; - return; - } - const size_t rpos = r.rfind(" ||| "); - string feats; - string rs = r; - if (rpos != pos) { - feats = r.substr(rpos + 5); - rs = r.substr(0, rpos); - } - string rhs = rs.substr(pos + 5); - string trule = rs + " ||| " + rhs + " ||| " + feats; - TRule tr(trule); - cerr << "X: " << tr.e_[0] << endl; -#ifdef DEBUG_CHART_PARSER - string hint_last_rule; -#endif - EGrammarNode* cur = &(*g)[tr.GetLHS()]; - cur->is_root = true; - for (int i = 0; i < tr.FLength(); ++i) { - WordID sym = tr.f()[i]; -#ifdef DEBUG_CHART_PARSER - hint_last_rule = TD::Convert(sym < 0 ? -sym : sym); - cur->hint += " <@@> (*" + hint_last_rule + ") " + TrimRule(tr.AsString()); -#endif - if (sym < 0) - cur = &cur->ntptr[sym]; - else - cur = &cur->tptr[sym]; - } -#ifdef DEBUG_CHART_PARSER - cur->hint += " <@@> (" + hint_last_rule + "*) " + TrimRule(tr.AsString()); -#endif - cur->is_some_rule_complete = true; - cur->input_features = tr.GetFeatureValues(); -} - -CFG_WFSTComposer::~CFG_WFSTComposer() { - delete pimpl_; -} - -CFG_WFSTComposer::CFG_WFSTComposer(const WFST& wfst) { - InitializeConstants(); - pimpl_ = new CFG_WFSTComposerImpl(kUNIQUE_START, wfst.Initial(), wfst.Final()); -} - -bool CFG_WFSTComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest) { - // first, convert the src forest into an EGrammar - EGrammar g; - const int nedges = src_forest.edges_.size(); - const int nnodes = src_forest.nodes_.size(); - vector cats(nnodes); - bool assign_cats = false; - for (int i = 0; i < nnodes; ++i) - if (assign_cats) { - cats[i] = TD::Convert("CAT_" + boost::lexical_cast(i)) * -1; - } else { - cats[i] = src_forest.nodes_[i].cat_; - } - // construct the grammar - for (int i = 0; i < nedges; ++i) { - const Hypergraph::Edge& edge = src_forest.edges_[i]; - const vector& src = edge.rule_->f(); - EGrammarNode* cur = &g[cats[edge.head_node_]]; - cur->is_root = true; - int ntc = 0; - for (int j = 0; j < src.size(); ++j) { - WordID sym = src[j]; - if (sym <= 0) { - sym = cats[edge.tail_nodes_[ntc]]; - ++ntc; - cur = &cur->ntptr[sym]; - } else { - cur = &cur->tptr[sym]; - } - } - cur->is_some_rule_complete = true; - cur->input_features = edge.feature_values_; - } - EGrammarNode& goal_rule = g[kUNIQUE_START]; - assert((goal_rule.ntptr.size() == 1 && goal_rule.tptr.size() == 0) || - (goal_rule.ntptr.size() == 0 && goal_rule.tptr.size() == 1)); - - return pimpl_->Compose(g, trg_forest); -} - -bool CFG_WFSTComposer::Compose(istream* in, Hypergraph* trg_forest) { - EGrammar g; - while(*in) { - string line; - getline(*in, line); - if (line.empty()) continue; - AddGrammarRule(line, &g); - } - - return pimpl_->Compose(g, trg_forest); -} diff --git a/gi/pf/cfg_wfst_composer.h b/gi/pf/cfg_wfst_composer.h deleted file mode 100644 index cf47f459..00000000 --- a/gi/pf/cfg_wfst_composer.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef _CFG_WFST_COMPOSER_H_ -#define _CFG_WFST_COMPOSER_H_ - -#include -#include -#include - -#include "trule.h" -#include "wordid.h" - -class CFG_WFSTComposerImpl; -class Hypergraph; - -struct WFSTNode { - virtual ~WFSTNode(); - // returns the next states reachable by consuming srcindex (which identifies a word) - // paired with the output string generated by taking that transition. - virtual std::vector > ExtendInput(unsigned srcindex) const = 0; -}; - -struct WFST { - virtual ~WFST(); - virtual const WFSTNode* Final() const = 0; - virtual const WFSTNode* Initial() const = 0; -}; - -class CFG_WFSTComposer { - public: - ~CFG_WFSTComposer(); - explicit CFG_WFSTComposer(const WFST& wfst); - bool Compose(const Hypergraph& in_forest, Hypergraph* trg_forest); - - // reads the grammar from a file. There must be a single top-level - // S -> X rule. Anything else is possible. Format is: - // [S] ||| [SS,1] - // [SS] ||| [NP,1] [VP,2] ||| Feature1=0.2 Feature2=-2.3 - // [SS] ||| [VP,1] [NP,2] ||| Feature1=0.8 - // [NP] ||| [DET,1] [N,2] ||| Feature3=2 - // ... - bool Compose(std::istream* grammar_file, Hypergraph* trg_forest); - - private: - CFG_WFSTComposerImpl* pimpl_; -}; - -#endif diff --git a/gi/pf/conditional_pseg.h b/gi/pf/conditional_pseg.h deleted file mode 100644 index 81ddb206..00000000 --- a/gi/pf/conditional_pseg.h +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef _CONDITIONAL_PSEG_H_ -#define _CONDITIONAL_PSEG_H_ - -#include -#include -#include -#include - -#include "m.h" -#include "prob.h" -#include "ccrp_nt.h" -#include "mfcr.h" -#include "trule.h" -#include "base_distributions.h" -#include "tdict.h" - -template -struct MConditionalTranslationModel { - explicit MConditionalTranslationModel(ConditionalBaseMeasure& rcp0) : - rp0(rcp0), d(0.5), strength(1.0), lambdas(1, prob_t::One()), p0s(1) {} - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << std::endl; - for (MFCR<1,TRule>::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second.total_dish_count_ << '\t' << i2->first << std::endl; - } - } - - double log_likelihood(const double& dd, const double& aa) const { - if (aa <= -dd) return -std::numeric_limits::infinity(); - //double llh = Md::log_beta_density(dd, 10, 3) + Md::log_gamma_density(aa, 1, 1); - double llh = Md::log_beta_density(dd, 1, 1) + - Md::log_gamma_density(dd + aa, 1, 1); - typename std::tr1::unordered_map, MFCR<1,TRule>, boost::hash > >::const_iterator it; - for (it = r.begin(); it != r.end(); ++it) - llh += it->second.log_crp_prob(dd, aa); - return llh; - } - - struct DiscountResampler { - DiscountResampler(const MConditionalTranslationModel& m) : m_(m) {} - const MConditionalTranslationModel& m_; - double operator()(const double& proposed_discount) const { - return m_.log_likelihood(proposed_discount, m_.strength); - } - }; - - struct AlphaResampler { - AlphaResampler(const MConditionalTranslationModel& m) : m_(m) {} - const MConditionalTranslationModel& m_; - double operator()(const double& proposed_strength) const { - return m_.log_likelihood(m_.d, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng) { - typename std::tr1::unordered_map, MFCR<1,TRule>, boost::hash > >::iterator it; -#if 1 - for (it = r.begin(); it != r.end(); ++it) { - it->second.resample_hyperparameters(rng); - } -#else - const unsigned nloop = 5; - const unsigned niterations = 10; - DiscountResampler dr(*this); - AlphaResampler ar(*this); - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -d + std::numeric_limits::min(), - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits::min(); - if (strength < 0.0) min_discount -= strength; - d = slice_sampler1d(dr, d, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } - strength = slice_sampler1d(ar, strength, *rng, -d, - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "MConditionalTranslationModel(d=" << d << ",s=" << strength << ") = " << log_likelihood(d, strength) << std::endl; - for (it = r.begin(); it != r.end(); ++it) { - it->second.set_discount(d); - it->second.set_strength(strength); - } -#endif - } - - int DecrementRule(const TRule& rule, MT19937* rng) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - const TableCount delta = it->second.decrement(rule, rng); - if (delta.count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return delta.count; - } - - int IncrementRule(const TRule& rule, MT19937* rng) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - //it = r.insert(make_pair(rule.f_, MFCR<1,TRule>(d, strength))).first; - it = r.insert(make_pair(rule.f_, MFCR<1,TRule>(1,1,1,1,0.6, -0.12))).first; - } - p0s[0] = rp0(rule); - TableCount delta = it->second.increment(rule, p0s.begin(), lambdas.begin(), rng); - return delta.count; - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p = rp0(rule); - } else { - p0s[0] = rp0(rule); - p = it->second.prob(rule, p0s.begin(), lambdas.begin()); - } - return p; - } - - prob_t Likelihood() const { - prob_t p; p.logeq(log_likelihood(d, strength)); - return p; - } - - const ConditionalBaseMeasure& rp0; - typedef std::tr1::unordered_map, - MFCR<1, TRule>, - boost::hash > > RuleModelHash; - RuleModelHash r; - double d, strength; - std::vector lambdas; - mutable std::vector p0s; -}; - -template -struct ConditionalTranslationModel { - explicit ConditionalTranslationModel(ConditionalBaseMeasure& rcp0) : - rp0(rcp0) {} - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(\\alpha = " << it->second.alpha() << ") --------------------------" << std::endl; - for (CCRP_NoTable::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second << '\t' << i2->first << std::endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - for (RuleModelHash::iterator it = r.begin(); it != r.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - int DecrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - int count = it->second.decrement(rule); - if (count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return count; - } - - int IncrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - it = r.insert(make_pair(rule.f_, CCRP_NoTable(1.0, 1.0, 8.0))).first; - } - int count = it->second.increment(rule); - return count; - } - - void IncrementRules(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p.logeq(log(rp0(rule))); - } else { - p.logeq(it->second.logprob(rule, log(rp0(rule)))); - } - return p; - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - for (CCRP_NoTable::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - p *= rp0(i2->first); - } - return p; - } - - const ConditionalBaseMeasure& rp0; - typedef std::tr1::unordered_map, - CCRP_NoTable, - boost::hash > > RuleModelHash; - RuleModelHash r; -}; - -template -struct ConditionalParallelSegementationModel { - explicit ConditionalParallelSegementationModel(ConditionalBaseMeasure& rcp0) : - tmodel(rcp0), base(prob_t::One()), aligns(1,1) {} - - ConditionalTranslationModel tmodel; - - void DecrementRule(const TRule& rule) { - tmodel.DecrementRule(rule); - } - - void IncrementRule(const TRule& rule) { - tmodel.IncrementRule(rule); - } - - void IncrementRulesAndAlignments(const std::vector& rules) { - tmodel.IncrementRules(rules); - for (int i = 0; i < rules.size(); ++i) { - IncrementAlign(rules[i]->f_.size()); - } - } - - void DecrementRulesAndAlignments(const std::vector& rules) { - tmodel.DecrementRules(rules); - for (int i = 0; i < rules.size(); ++i) { - DecrementAlign(rules[i]->f_.size()); - } - } - - prob_t RuleProbability(const TRule& rule) const { - return tmodel.RuleProbability(rule); - } - - void IncrementAlign(unsigned span) { - if (aligns.increment(span)) { - // TODO - } - } - - void DecrementAlign(unsigned span) { - if (aligns.decrement(span)) { - // TODO - } - } - - prob_t AlignProbability(unsigned span) const { - prob_t p; - p.logeq(aligns.logprob(span, Md::log_poisson(span, 1.0))); - return p; - } - - prob_t Likelihood() const { - prob_t p; p.logeq(aligns.log_crp_prob()); - p *= base; - p *= tmodel.Likelihood(); - return p; - } - - prob_t base; - CCRP_NoTable aligns; -}; - -#endif - diff --git a/gi/pf/condnaive.cc b/gi/pf/condnaive.cc deleted file mode 100644 index 419731ac..00000000 --- a/gi/pf/condnaive.cc +++ /dev/null @@ -1,298 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "conditional_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "corpus.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(4),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(4),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr prng; - -struct ModelAndData { - explicit ModelAndData(ConditionalParallelSegementationModel& m, const vector >& ce, const vector >& cf, const set& ve, const set& vf) : - model(m), - rng(&*prng), - corpuse(ce), - corpusf(cf), - vocabe(ve), - vocabf(vf), - mh_samples(), - mh_rejects(), - kX(-TD::Convert("X")), - derivations(corpuse.size()) {} - - void ResampleHyperparameters() { - } - - void InstantiateRule(const pair& from, - const pair& to, - const vector& sentf, - const vector& sente, - TRule* rule) const { - rule->f_.clear(); - rule->e_.clear(); - rule->lhs_ = kX; - for (short i = from.first; i < to.first; ++i) - rule->f_.push_back(sentf[i]); - for (short i = from.second; i < to.second; ++i) - rule->e_.push_back(sente[i]); - } - - void DecrementDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.DecrementRule(x); - model.DecrementAlign(x.f_.size()); - } - } - - void PrintDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - cerr << i << '/' << (d.size() - 1) << ": " << x << endl; - } - } - - void IncrementDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.IncrementRule(x); - model.IncrementAlign(x.f_.size()); - } - } - - prob_t Likelihood() const { - return model.Likelihood(); - } - - prob_t DerivationProposalProbability(const vector >& d, const vector& sentf, const vector& sente) const { - prob_t p = prob_t::One(); - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - p *= model.RuleProbability(x); - p *= model.AlignProbability(x.f_.size()); - } - return p; - } - - void Sample(); - - ConditionalParallelSegementationModel& model; - MT19937* rng; - const vector >& corpuse, corpusf; - const set& vocabe, vocabf; - unsigned mh_samples, mh_rejects; - const int kX; - vector > > derivations; -}; - -void ModelAndData::Sample() { - unsigned MAXK = kMAX_SRC_PHRASE; - unsigned MAXL = kMAX_TRG_PHRASE; - TRule x; - x.lhs_ = -TD::Convert("X"); - - for (int samples = 0; samples < 1000; ++samples) { - if (samples % 1 == 0 && samples > 0) { - //ResampleHyperparameters(); - cerr << " [" << samples << " LLH=" << log(Likelihood()) << " MH=" << ((double)mh_rejects / mh_samples) << "]\n"; - for (int i = 0; i < 10; ++i) { - cerr << "SENTENCE: " << TD::GetString(corpusf[i]) << " ||| " << TD::GetString(corpuse[i]) << endl; - PrintDerivation(derivations[i], corpusf[i], corpuse[i]); - } - static TRule xx("[X] ||| w n ||| s h ||| X=0"); - const CCRP_NoTable& dcrp = model.tmodel.r.find(xx.f_)->second; - for (CCRP_NoTable::const_iterator it = dcrp.begin(); it != dcrp.end(); ++it) { - cerr << "\t" << it->second << "\t" << it->first << endl; - } - } - cerr << '.' << flush; - for (int s = 0; s < corpuse.size(); ++s) { - const vector& sentf = corpusf[s]; - const vector& sente = corpuse[s]; -// cerr << " CUSTOMERS: " << rules.num_customers() << endl; -// cerr << "SENTENCE: " << TD::GetString(sentf) << " ||| " << TD::GetString(sente) << endl; - - vector >& deriv = derivations[s]; - const prob_t p_cur = Likelihood(); - DecrementDerivation(deriv, sentf, sente); - - boost::multi_array a(boost::extents[sentf.size() + 1][sente.size() + 1]); - boost::multi_array trans(boost::extents[sentf.size() + 1][sente.size() + 1][MAXK][MAXL]); - a[0][0] = prob_t::One(); - for (int i = 0; i < sentf.size(); ++i) { - for (int j = 0; j < sente.size(); ++j) { - const prob_t src_a = a[i][j]; - x.f_.clear(); - for (int k = 1; k <= MAXK; ++k) { - if (i + k > sentf.size()) break; - x.f_.push_back(sentf[i + k - 1]); - x.e_.clear(); - const prob_t p_span = model.AlignProbability(k); // prob of consuming this much source - for (int l = 1; l <= MAXL; ++l) { - if (j + l > sente.size()) break; - x.e_.push_back(sente[j + l - 1]); - trans[i][j][k - 1][l - 1] = model.RuleProbability(x) * p_span; - a[i + k][j + l] += src_a * trans[i][j][k - 1][l - 1]; - } - } - } - } -// cerr << "Inside: " << log(a[sentf.size()][sente.size()]) << endl; - const prob_t q_cur = DerivationProposalProbability(deriv, sentf, sente); - - vector > newderiv; - int cur_i = sentf.size(); - int cur_j = sente.size(); - while(cur_i > 0 && cur_j > 0) { - newderiv.push_back(pair(cur_i, cur_j)); -// cerr << "NODE: (" << cur_i << "," << cur_j << ")\n"; - SampleSet ss; - vector > nexts; - for (int k = 1; k <= MAXK; ++k) { - const int hyp_i = cur_i - k; - if (hyp_i < 0) break; - for (int l = 1; l <= MAXL; ++l) { - const int hyp_j = cur_j - l; - if (hyp_j < 0) break; - const prob_t& inside = a[hyp_i][hyp_j]; - if (inside == prob_t::Zero()) continue; - const prob_t& transp = trans[hyp_i][hyp_j][k - 1][l - 1]; - if (transp == prob_t::Zero()) continue; - const prob_t p = inside * transp; - ss.add(p); - nexts.push_back(pair(hyp_i, hyp_j)); -// cerr << " (" << hyp_i << "," << hyp_j << ") <--- " << log(p) << endl; - } - } -// cerr << " sample set has " << nexts.size() << " elements.\n"; - const int selected = rng->SelectSample(ss); - cur_i = nexts[selected].first; - cur_j = nexts[selected].second; - } - newderiv.push_back(pair(0,0)); - const prob_t q_new = DerivationProposalProbability(newderiv, sentf, sente); - IncrementDerivation(newderiv, sentf, sente); -// cerr << "SANITY: " << q_new << " " <(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - Model1 m1(conf["model1"].as()); - - PhraseConditionalBase pcb0(m1, conf["model1_interpolation_weight"].as(), vocabe.size()); - ConditionalParallelSegementationModel x(pcb0); - - ModelAndData posterior(x, corpuse, corpusf, vocabe, vocabf); - posterior.Sample(); - - TRule r1("[X] ||| x ||| l e ||| X=0"); - TRule r2("[X] ||| A ||| a d ||| X=0"); - TRule r3("[X] ||| n ||| e r ||| X=0"); - TRule r4("[X] ||| x A n ||| b l a g ||| X=0"); - - PhraseConditionalUninformativeBase u0(vocabe.size()); - - cerr << (pcb0(r1)*pcb0(r2)*pcb0(r3)) << endl; - cerr << (u0(r4)) << endl; - - return 0; -} - diff --git a/gi/pf/corpus.cc b/gi/pf/corpus.cc deleted file mode 100644 index cb6e4ed7..00000000 --- a/gi/pf/corpus.cc +++ /dev/null @@ -1,62 +0,0 @@ -#include "corpus.h" - -#include -#include -#include - -#include "tdict.h" -#include "filelib.h" - -using namespace std; - -namespace corpus { - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - ReadFile rf(filename); - istream* in = rf.stream(); - assert(*in); - string line; - unsigned lc = 0; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(getline(*in, line)) { - ++lc; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { - isf = false; - } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - if (cur == kDIV) { - cerr << "ERROR in " << lc << ": " << line << endl << endl; - abort(); - } - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } -} - -} - diff --git a/gi/pf/corpus.h b/gi/pf/corpus.h deleted file mode 100644 index e7febdb7..00000000 --- a/gi/pf/corpus.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _CORPUS_H_ -#define _CORPUS_H_ - -#include -#include -#include -#include "wordid.h" - -namespace corpus { - -void ReadParallelCorpus(const std::string& filename, - std::vector >* f, - std::vector >* e, - std::set* vocab_f, - std::set* vocab_e); - -} - -#endif diff --git a/gi/pf/dpnaive.cc b/gi/pf/dpnaive.cc deleted file mode 100644 index 75ccad72..00000000 --- a/gi/pf/dpnaive.cc +++ /dev/null @@ -1,301 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "corpus.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(4),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(4),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr prng; - -template -struct ModelAndData { - explicit ModelAndData(MonotonicParallelSegementationModel& m, const Base& b, const vector >& ce, const vector >& cf, const set& ve, const set& vf) : - model(m), - rng(&*prng), - p0(b), - baseprob(prob_t::One()), - corpuse(ce), - corpusf(cf), - vocabe(ve), - vocabf(vf), - mh_samples(), - mh_rejects(), - kX(-TD::Convert("X")), - derivations(corpuse.size()) {} - - void ResampleHyperparameters() { - } - - void InstantiateRule(const pair& from, - const pair& to, - const vector& sentf, - const vector& sente, - TRule* rule) const { - rule->f_.clear(); - rule->e_.clear(); - rule->lhs_ = kX; - for (short i = from.first; i < to.first; ++i) - rule->f_.push_back(sentf[i]); - for (short i = from.second; i < to.second; ++i) - rule->e_.push_back(sente[i]); - } - - void DecrementDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.DecrementRule(x); - model.DecrementContinue(); - } - model.DecrementStop(); - } - - void PrintDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - cerr << i << '/' << (d.size() - 1) << ": " << x << endl; - } - } - - void IncrementDerivation(const vector >& d, const vector& sentf, const vector& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.IncrementRule(x); - model.IncrementContinue(); - } - model.IncrementStop(); - } - - prob_t Likelihood() const { - return model.Likelihood(); - } - - prob_t DerivationProposalProbability(const vector >& d, const vector& sentf, const vector& sente) const { - prob_t p = model.StopProbability(); - if (d.size() < 2) return p; - TRule x; - const prob_t p_cont = model.ContinueProbability(); - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - p *= p_cont; - p *= model.RuleProbability(x); - } - return p; - } - - void Sample(); - - MonotonicParallelSegementationModel& model; - MT19937* rng; - const Base& p0; - prob_t baseprob; // cached value of generating the table table labels from p0 - // this can't be used if we go to a hierarchical prior! - const vector >& corpuse, corpusf; - const set& vocabe, vocabf; - unsigned mh_samples, mh_rejects; - const int kX; - vector > > derivations; -}; - -template -void ModelAndData::Sample() { - unsigned MAXK = kMAX_SRC_PHRASE; - unsigned MAXL = kMAX_TRG_PHRASE; - TRule x; - x.lhs_ = -TD::Convert("X"); - for (int samples = 0; samples < 1000; ++samples) { - if (samples % 1 == 0 && samples > 0) { - //ResampleHyperparameters(); - cerr << " [" << samples << " LLH=" << log(Likelihood()) << " MH=" << ((double)mh_rejects / mh_samples) << "]\n"; - for (int i = 0; i < 10; ++i) { - cerr << "SENTENCE: " << TD::GetString(corpusf[i]) << " ||| " << TD::GetString(corpuse[i]) << endl; - PrintDerivation(derivations[i], corpusf[i], corpuse[i]); - } - } - cerr << '.' << flush; - for (int s = 0; s < corpuse.size(); ++s) { - const vector& sentf = corpusf[s]; - const vector& sente = corpuse[s]; -// cerr << " CUSTOMERS: " << rules.num_customers() << endl; -// cerr << "SENTENCE: " << TD::GetString(sentf) << " ||| " << TD::GetString(sente) << endl; - - vector >& deriv = derivations[s]; - const prob_t p_cur = Likelihood(); - DecrementDerivation(deriv, sentf, sente); - - boost::multi_array a(boost::extents[sentf.size() + 1][sente.size() + 1]); - boost::multi_array trans(boost::extents[sentf.size() + 1][sente.size() + 1][MAXK][MAXL]); - a[0][0] = prob_t::One(); - const prob_t q_stop = model.StopProbability(); - const prob_t q_cont = model.ContinueProbability(); - for (int i = 0; i < sentf.size(); ++i) { - for (int j = 0; j < sente.size(); ++j) { - const prob_t src_a = a[i][j]; - x.f_.clear(); - for (int k = 1; k <= MAXK; ++k) { - if (i + k > sentf.size()) break; - x.f_.push_back(sentf[i + k - 1]); - x.e_.clear(); - for (int l = 1; l <= MAXL; ++l) { - if (j + l > sente.size()) break; - x.e_.push_back(sente[j + l - 1]); - const bool stop_now = ((j + l) == sente.size()) && ((i + k) == sentf.size()); - const prob_t& cp = stop_now ? q_stop : q_cont; - trans[i][j][k - 1][l - 1] = model.RuleProbability(x) * cp; - a[i + k][j + l] += src_a * trans[i][j][k - 1][l - 1]; - } - } - } - } -// cerr << "Inside: " << log(a[sentf.size()][sente.size()]) << endl; - const prob_t q_cur = DerivationProposalProbability(deriv, sentf, sente); - - vector > newderiv; - int cur_i = sentf.size(); - int cur_j = sente.size(); - while(cur_i > 0 && cur_j > 0) { - newderiv.push_back(pair(cur_i, cur_j)); -// cerr << "NODE: (" << cur_i << "," << cur_j << ")\n"; - SampleSet ss; - vector > nexts; - for (int k = 1; k <= MAXK; ++k) { - const int hyp_i = cur_i - k; - if (hyp_i < 0) break; - for (int l = 1; l <= MAXL; ++l) { - const int hyp_j = cur_j - l; - if (hyp_j < 0) break; - const prob_t& inside = a[hyp_i][hyp_j]; - if (inside == prob_t::Zero()) continue; - const prob_t& transp = trans[hyp_i][hyp_j][k - 1][l - 1]; - if (transp == prob_t::Zero()) continue; - const prob_t p = inside * transp; - ss.add(p); - nexts.push_back(pair(hyp_i, hyp_j)); -// cerr << " (" << hyp_i << "," << hyp_j << ") <--- " << log(p) << endl; - } - } -// cerr << " sample set has " << nexts.size() << " elements.\n"; - const int selected = rng->SelectSample(ss); - cur_i = nexts[selected].first; - cur_j = nexts[selected].second; - } - newderiv.push_back(pair(0,0)); - const prob_t q_new = DerivationProposalProbability(newderiv, sentf, sente); - IncrementDerivation(newderiv, sentf, sente); -// cerr << "SANITY: " << q_new << " " <(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (!conf.count("inverse_model1")) { - cerr << argv[0] << "Please use --inverse_model1 to specify inverse model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - Model1 m1(conf["model1"].as()); - Model1 invm1(conf["inverse_model1"].as()); -// PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel m(alp0); - - ModelAndData posterior(m, alp0, corpuse, corpusf, vocabe, vocabf); - posterior.Sample(); - - return 0; -} - diff --git a/gi/pf/guess-translits.pl b/gi/pf/guess-translits.pl deleted file mode 100755 index d00c2168..00000000 --- a/gi/pf/guess-translits.pl +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use utf8; - -my $MIN_PMI = -3; - -my %fs; -my %es; -my %ef; - -die "Usage: $0 < input.utf8.txt\n" if scalar @ARGV > 0; - -binmode(STDIN,":utf8"); -binmode(STDOUT,":utf8"); -binmode(STDERR,":utf8"); - -my $tot = 0; -print STDERR "Reading alignments from STDIN ...\n"; -while() { - chomp; - my ($fsent, $esent, $alsent) = split / \|\|\| /; - die "Format should be 'foreign sentence ||| english sentence ||| 0-0 1-1 ...'\n" unless defined $fsent && defined $esent && defined $alsent; - - my @fws = split /\s+/, $fsent; - my @ews = split /\s+/, $esent; - my @as = split /\s+/, $alsent; - my %a2b; - my %b2a; - for my $ap (@as) { - my ($a,$b) = split /-/, $ap; - die "BAD INPUT: $_\n" unless defined $a && defined $b; - $a2b{$a}->{$b} = 1; - $b2a{$b}->{$a} = 1; - } - for my $a (keys %a2b) { - my $bref = $a2b{$a}; - next unless scalar keys %$bref < 2; - my $b = (keys %$bref)[0]; - next unless scalar keys %{$b2a{$b}} < 2; - my $f = $fws[$a]; - next unless defined $f; - next unless length($f) > 3; - my $e = $ews[$b]; - next unless defined $e; - next unless length($e) > 3; - - $ef{$f}->{$e}++; - $es{$e}++; - $fs{$f}++; - $tot++; - } -} -my $ltot = log($tot); -my $num = 0; -print STDERR "Extracting pairs for PMI > $MIN_PMI ...\n"; -for my $f (keys %fs) { - my $logf = log($fs{$f}); - my $esref = $ef{$f}; - for my $e (keys %$esref) { - my $loge = log($es{$e}); - my $ef = $esref->{$e}; - my $logef = log($ef); - my $pmi = $logef - ($loge + $logf); - next if $pmi < $MIN_PMI; - my @flets = split //, $f; - my @elets = split //, $e; - print "@flets ||| @elets\n"; - $num++; - } -} -print STDERR "Extracted $num pairs.\n"; -print STDERR "Recommend running:\n ../../training/model1 -v -d -t -99999 output.txt\n"; diff --git a/gi/pf/hpyp_tm.cc b/gi/pf/hpyp_tm.cc deleted file mode 100644 index f362d3f8..00000000 --- a/gi/pf/hpyp_tm.cc +++ /dev/null @@ -1,133 +0,0 @@ -#include "hpyp_tm.h" - -#include -#include -#include - -#include "tdict.h" -#include "ccrp.h" -#include "pyp_word_model.h" -#include "tied_resampler.h" - -using namespace std; -using namespace std::tr1; - -struct FreqBinner { - FreqBinner(const std::string& fname) { fd_.Load(fname); } - unsigned NumberOfBins() const { return fd_.Max() + 1; } - unsigned Bin(const WordID& w) const { return fd_.LookUp(w); } - FreqDict fd_; -}; - -template -struct ConditionalPYPWordModel { - ConditionalPYPWordModel(Base* b, const Binner* bnr = NULL) : - base(*b), - binner(bnr), - btr(binner ? binner->NumberOfBins() + 1u : 2u) {} - - void Summary() const { - cerr << "Number of conditioning contexts: " << r.size() << endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - cerr << TD::Convert(it->first) << " \tPYP(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << endl; - for (CCRP >::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - cerr << " " << i2->second << endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - btr.ResampleHyperparameters(rng); - } - - prob_t Prob(const WordID src, const vector& trglets) const { - RuleModelHash::const_iterator it = r.find(src); - if (it == r.end()) { - return base(trglets); - } else { - return it->second.prob(trglets, base(trglets)); - } - } - - void Increment(const WordID src, const vector& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - if (it == r.end()) { - it = r.insert(make_pair(src, CCRP >(0.5,1.0))).first; - static const WordID kNULL = TD::Convert("NULL"); - unsigned bin = (src == kNULL ? 0 : 1); - if (binner && bin) { bin = binner->Bin(src) + 1; } - btr.Add(bin, &it->second); - } - if (it->second.increment(trglets, base(trglets), rng)) - base.Increment(trglets, rng); - } - - void Decrement(const WordID src, const vector& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - assert(it != r.end()); - if (it->second.decrement(trglets, rng)) { - base.Decrement(trglets, rng); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - return p; - } - - unsigned UniqueConditioningContexts() const { - return r.size(); - } - - // TODO tie PYP hyperparameters based on source word frequency bins - Base& base; - const Binner* binner; - BinTiedResampler > > btr; - typedef unordered_map > > RuleModelHash; - RuleModelHash r; -}; - -HPYPLexicalTranslation::HPYPLexicalTranslation(const vector >& lets, - const unsigned vocab_size, - const unsigned num_letters) : - letters(lets), - base(vocab_size, num_letters, 5), - up0(new PYPWordModel(&base)), - tmodel(new ConditionalPYPWordModel >(up0, new FreqBinner("10k.freq"))), - kX(-TD::Convert("X")) {} - -void HPYPLexicalTranslation::Summary() const { - tmodel->Summary(); - up0->Summary(); -} - -prob_t HPYPLexicalTranslation::Likelihood() const { - prob_t p = up0->Likelihood(); - p *= tmodel->Likelihood(); - return p; -} - -void HPYPLexicalTranslation::ResampleHyperparameters(MT19937* rng) { - tmodel->ResampleHyperparameters(rng); - up0->ResampleHyperparameters(rng); -} - -unsigned HPYPLexicalTranslation::UniqueConditioningContexts() const { - return tmodel->UniqueConditioningContexts(); -} - -prob_t HPYPLexicalTranslation::Prob(WordID src, WordID trg) const { - return tmodel->Prob(src, letters[trg]); -} - -void HPYPLexicalTranslation::Increment(WordID src, WordID trg, MT19937* rng) { - tmodel->Increment(src, letters[trg], rng); -} - -void HPYPLexicalTranslation::Decrement(WordID src, WordID trg, MT19937* rng) { - tmodel->Decrement(src, letters[trg], rng); -} - diff --git a/gi/pf/hpyp_tm.h b/gi/pf/hpyp_tm.h deleted file mode 100644 index af3215ba..00000000 --- a/gi/pf/hpyp_tm.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef HPYP_LEX_TRANS -#define HPYP_LEX_TRANS - -#include -#include "wordid.h" -#include "prob.h" -#include "sampler.h" -#include "freqdict.h" -#include "poisson_uniform_word_model.h" - -struct FreqBinner; -template struct PYPWordModel; -template struct ConditionalPYPWordModel; - -struct HPYPLexicalTranslation { - explicit HPYPLexicalTranslation(const std::vector >& lets, - const unsigned vocab_size, - const unsigned num_letters); - - prob_t Likelihood() const; - - void ResampleHyperparameters(MT19937* rng); - prob_t Prob(WordID src, WordID trg) const; // return p(trg | src) - void Summary() const; - void Increment(WordID src, WordID trg, MT19937* rng); - void Decrement(WordID src, WordID trg, MT19937* rng); - unsigned UniqueConditioningContexts() const; - - private: - const std::vector >& letters; // spelling dictionary - PoissonUniformWordModel base; // "generator" of English types - PYPWordModel* up0; // model English lexicon - ConditionalPYPWordModel, FreqBinner>* tmodel; // translation distributions - // (model English word | French word) - const WordID kX; -}; - -#endif diff --git a/gi/pf/itg.cc b/gi/pf/itg.cc deleted file mode 100644 index 29ec3860..00000000 --- a/gi/pf/itg.cc +++ /dev/null @@ -1,275 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -ostream& operator<<(ostream& os, const vector& p) { - os << '['; - for (int i = 0; i < p.size(); ++i) - os << (i==0 ? "" : " ") << TD::Convert(p[i]); - return os << ']'; -} - -struct UnigramModel { - explicit UnigramModel(const string& fname, unsigned vocab_size, double p0null = 0.05) : - use_uniform_(fname.size() == 0), - p0null_(p0null), - uniform_((1.0 - p0null) / vocab_size), - probs_(TD::NumWords() + 1) { - if (fname.size() > 0) LoadUnigrams(fname); - probs_[0] = p0null_; - } - -// -// \data\ -// ngram 1=9295 -// -// \1-grams: -// -3.191193 " - - void LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - const WordID w = TD::Convert(line.substr(pos + 1)); - line[pos] = 0; - float p = atof(&line[0]); - const prob_t pnon_null(1.0 - p0null_.as_float()); - if (w < probs_.size()) probs_[w].logeq(p * log(10) + log(pnon_null)); else abort(); - } - } - - const prob_t& operator()(const WordID& w) const { - if (!w) return p0null_; - if (use_uniform_) return uniform_; - return probs_[w]; - } - - const bool use_uniform_; - const prob_t p0null_; - const prob_t uniform_; - vector probs_; -}; - -struct Model1 { - explicit Model1(const string& fname) : - kNULL(TD::Convert("")), - kZERO() { - LoadModel1(fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // returns prob 0 if src or trg is not found! - const prob_t& operator()(WordID src, WordID trg) const { - if (src == 0) src = kNULL; - if (src < ttable.size()) { - const map& cpd = ttable[src]; - const map::const_iterator it = cpd.find(trg); - if (it != cpd.end()) - return it->second; - } - return kZERO; - } - - const WordID kNULL; - const prob_t kZERO; - vector > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("particles,p",po::value()->default_value(25),"Number of particles") - ("input,i",po::value(),"Read parallel data from") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("src_unigram,u",po::value()->default_value(""),"Source unigram distribution; empty for uniform") - ("trg_unigram,U",po::value()->default_value(""),"Target unigram distribution; empty for uniform") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned particles = conf["particles"].as(); - const unsigned samples = conf["samples"].as(); - TD::Convert(""); - TD::Convert(""); - TD::Convert(""); - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - UnigramModel src_unigram(conf["src_unigram"].as(), vocabf.size()); - UnigramModel trg_unigram(conf["trg_unigram"].as(), vocabe.size()); - const prob_t kHALF(0.5); - - const string kEMPTY = "NULL"; - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as()); - Model1 invm1(conf["inverse_model1"].as()); - for (int si = 0; si < conf["samples"].as(); ++si) { - cerr << '.' << flush; - for (int ci = 0; ci < corpusf.size(); ++ci) { - const vector& trg = corpuse[ci]; - const vector& src = corpusf[ci]; - for (int i = 0; i <= trg.size(); ++i) { - const WordID e_i = i > 0 ? trg[i-1] : 0; - for (int j = 0; j <= src.size(); ++j) { - const WordID f_j = j > 0 ? src[j-1] : 0; - if (e_i == 0 && f_j == 0) continue; - prob_t je = kHALF * src_unigram(f_j) * m1(f_j,e_i) + kHALF * trg_unigram(e_i) * invm1(e_i,f_j); - cerr << "p( " << (e_i ? TD::Convert(e_i) : kEMPTY) << " , " << (f_j ? TD::Convert(f_j) : kEMPTY) << " ) = " << log(je) << endl; - if (e_i && f_j) - cout << "[X] ||| " << TD::Convert(f_j) << " ||| " << TD::Convert(e_i) << " ||| LogProb=" << log(je) << endl; - } - } - } - } -} - diff --git a/gi/pf/learn_cfg.cc b/gi/pf/learn_cfg.cc deleted file mode 100644 index 1d5126e4..00000000 --- a/gi/pf/learn_cfg.cc +++ /dev/null @@ -1,428 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "inside_outside.h" -#include "hg.h" -#include "bottom_up_parser.h" -#include "fdict.h" -#include "grammar.h" -#include "m.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; -vector nt_vocab; -vector nt_id_to_index; -static unsigned kMAX_RULE_SIZE = 0; -static unsigned kMAX_ARITY = 0; -static bool kALLOW_MIXED = true; // allow rules with mixed terminals and NTs -static bool kHIERARCHICAL_PRIOR = false; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("max_rule_size,m", po::value()->default_value(0), "Maximum rule size (0 for unlimited)") - ("max_arity,a", po::value()->default_value(0), "Maximum number of nonterminals in a rule (0 for unlimited)") - ("no_mixed_rules,M", "Do not mix terminals and nonterminals in a rule RHS") - ("nonterminals,n", po::value()->default_value(1), "Size of nonterminal vocabulary") - ("hierarchical_prior,h", "Use hierarchical prior") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -unsigned ReadCorpus(const string& filename, - vector >* e, - set* vocab_e) { - e->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - unsigned toks = 0; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - vector& le = e->back(); - TD::ConvertSentence(line, &le); - for (unsigned i = 0; i < le.size(); ++i) - vocab_e->insert(le[i]); - toks += le.size(); - } - if (in != &cin) delete in; - return toks; -} - -struct Grid { - // a b c d e - // 0 - 0 - - - vector grid; -}; - -struct BaseRuleModel { - explicit BaseRuleModel(unsigned term_size, - unsigned nonterm_size = 1) : - unif_term(1.0 / term_size), - unif_nonterm(1.0 / nonterm_size) {} - prob_t operator()(const TRule& r) const { - prob_t p; p.logeq(Md::log_poisson(1.0, r.f_.size())); - const prob_t term_prob((2.0 + 0.01*r.f_.size()) / (r.f_.size() + 2)); - const prob_t nonterm_prob(1.0 - term_prob.as_float()); - for (unsigned i = 0; i < r.f_.size(); ++i) { - if (r.f_[i] <= 0) { // nonterminal - if (kALLOW_MIXED) p *= nonterm_prob; - p *= unif_nonterm; - } else { // terminal - if (kALLOW_MIXED) p *= term_prob; - p *= unif_term; - } - } - return p; - } - const prob_t unif_term, unif_nonterm; -}; - -struct HieroLMModel { - explicit HieroLMModel(unsigned vocab_size, unsigned num_nts = 1) : - base(vocab_size, num_nts), - q0(1,1,1,1), - nts(num_nts, CCRP(1,1,1,1)) {} - - prob_t Prob(const TRule& r) const { - return nts[nt_id_to_index[-r.lhs_]].prob(r, p0(r)); - } - - inline prob_t p0(const TRule& r) const { - if (kHIERARCHICAL_PRIOR) - return q0.prob(r, base(r)); - else - return base(r); - } - - int Increment(const TRule& r, MT19937* rng) { - const int delta = nts[nt_id_to_index[-r.lhs_]].increment(r, p0(r), rng); - if (kHIERARCHICAL_PRIOR && delta) - q0.increment(r, base(r), rng); - return delta; - // return x.increment(r); - } - - int Decrement(const TRule& r, MT19937* rng) { - const int delta = nts[nt_id_to_index[-r.lhs_]].decrement(r, rng); - if (kHIERARCHICAL_PRIOR && delta) - q0.decrement(r, rng); - return delta; - //return x.decrement(r); - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (unsigned i = 0; i < nts.size(); ++i) { - prob_t q; q.logeq(nts[i].log_crp_prob()); - p *= q; - for (CCRP::const_iterator it = nts[i].begin(); it != nts[i].end(); ++it) { - prob_t tp = p0(it->first); - tp.poweq(it->second.num_tables()); - p *= tp; - } - } - if (kHIERARCHICAL_PRIOR) { - prob_t q; q.logeq(q0.log_crp_prob()); - p *= q; - for (CCRP::const_iterator it = q0.begin(); it != q0.end(); ++it) { - prob_t tp = base(it->first); - tp.poweq(it->second.num_tables()); - p *= tp; - } - } - //for (CCRP_OneTable::const_iterator it = x.begin(); it != x.end(); ++it) - // p *= base(it->first); - return p; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < nts.size(); ++i) - nts[i].resample_hyperparameters(rng); - if (kHIERARCHICAL_PRIOR) { - q0.resample_hyperparameters(rng); - cerr << "[base d=" << q0.discount() << ", s=" << q0.strength() << "]"; - } - cerr << " d=" << nts[0].discount() << ", s=" << nts[0].strength() << endl; - } - - const BaseRuleModel base; - CCRP q0; - vector > nts; - //CCRP_OneTable x; -}; - -vector tofreelist; - -HieroLMModel* plm; - -struct NPGrammarIter : public GrammarIter, public RuleBin { - NPGrammarIter() : arity() { tofreelist.push_back(this); } - NPGrammarIter(const TRulePtr& inr, const int a, int symbol) : arity(a) { - if (inr) { - r.reset(new TRule(*inr)); - } else { - r.reset(new TRule); - } - TRule& rr = *r; - rr.lhs_ = nt_vocab[0]; - rr.f_.push_back(symbol); - rr.e_.push_back(symbol < 0 ? (1-int(arity)) : symbol); - tofreelist.push_back(this); - } - inline static unsigned NextArity(int cur_a, int symbol) { - return cur_a + (symbol <= 0 ? 1 : 0); - } - virtual int GetNumRules() const { - if (r) return nt_vocab.size(); else return 0; - } - virtual TRulePtr GetIthRule(int i) const { - if (i == 0) return r; - TRulePtr nr(new TRule(*r)); - nr->lhs_ = nt_vocab[i]; - return nr; - } - virtual int Arity() const { - return arity; - } - virtual const RuleBin* GetRules() const { - if (!r) return NULL; else return this; - } - virtual const GrammarIter* Extend(int symbol) const { - const int next_arity = NextArity(arity, symbol); - if (kMAX_ARITY && next_arity > kMAX_ARITY) - return NULL; - if (!kALLOW_MIXED && r) { - bool t1 = r->f_.front() <= 0; - bool t2 = symbol <= 0; - if (t1 != t2) return NULL; - } - if (!kMAX_RULE_SIZE || !r || (r->f_.size() < kMAX_RULE_SIZE)) - return new NPGrammarIter(r, next_arity, symbol); - else - return NULL; - } - const unsigned char arity; - TRulePtr r; -}; - -struct NPGrammar : public Grammar { - virtual const GrammarIter* GetRoot() const { - return new NPGrammarIter; - } -}; - -prob_t TotalProb(const Hypergraph& hg) { - return Inside(hg); -} - -void SampleDerivation(const Hypergraph& hg, MT19937* rng, vector* sampled_deriv) { - vector node_probs; - Inside(hg, &node_probs); - queue q; - q.push(hg.nodes_.size() - 2); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - //prob_t z; - assert(num_in_edges > 1); - SampleSet ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = edge.edge_prob_; - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - //z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } - for (unsigned i = 0; i < sampled_deriv->size(); ++i) { - cerr << *hg.edges_[(*sampled_deriv)[i]].rule_ << endl; - } -} - -void IncrementDerivation(const Hypergraph& hg, const vector& d, HieroLMModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Increment(*hg.edges_[d[i]].rule_, rng); -} - -void DecrementDerivation(const Hypergraph& hg, const vector& d, HieroLMModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Decrement(*hg.edges_[d[i]].rule_, rng); -} - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - nt_vocab.resize(conf["nonterminals"].as()); - assert(nt_vocab.size() > 0); - assert(nt_vocab.size() < 26); - { - string nt = "X"; - for (unsigned i = 0; i < nt_vocab.size(); ++i) { - if (nt_vocab.size() > 1) nt[0] = ('A' + i); - int pid = TD::Convert(nt); - nt_vocab[i] = -pid; - if (pid >= nt_id_to_index.size()) { - nt_id_to_index.resize(pid + 1, -1); - } - nt_id_to_index[pid] = i; - } - } - vector grammars; - grammars.push_back(GrammarPtr(new NPGrammar)); - - const unsigned samples = conf["samples"].as(); - kMAX_RULE_SIZE = conf["max_rule_size"].as(); - if (kMAX_RULE_SIZE == 1) { - cerr << "Invalid maximum rule size: must be 0 or >1\n"; - return 1; - } - kMAX_ARITY = conf["max_arity"].as(); - if (kMAX_ARITY == 1) { - cerr << "Invalid maximum arity: must be 0 or >1\n"; - return 1; - } - kALLOW_MIXED = !conf.count("no_mixed_rules"); - - kHIERARCHICAL_PRIOR = conf.count("hierarchical_prior"); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector > corpuse; - set vocabe; - cerr << "Reading corpus...\n"; - const unsigned toks = ReadCorpus(conf["input"].as(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - HieroLMModel lm(vocabe.size(), nt_vocab.size()); - - plm = &lm; - ExhaustiveBottomUpParser parser(TD::Convert(-nt_vocab[0]), grammars); - - Hypergraph hg; - const int kGoal = -TD::Convert("Goal"); - const int kLP = FD::Convert("LogProb"); - SparseVector v; v.set_value(kLP, 1.0); - vector > derivs(corpuse.size()); - vector cl(corpuse.size()); - for (int ci = 0; ci < corpuse.size(); ++ci) { - vector& src = corpuse[ci]; - Lattice& lat = cl[ci]; - lat.resize(src.size()); - for (unsigned i = 0; i < src.size(); ++i) - lat[i].push_back(LatticeArc(src[i], 0.0, 1)); - } - for (int SS=0; SS < samples; ++SS) { - const bool is_last = ((samples - 1) == SS); - prob_t dlh = prob_t::One(); - for (int ci = 0; ci < corpuse.size(); ++ci) { - const vector& src = corpuse[ci]; - const Lattice& lat = cl[ci]; - cerr << TD::GetString(src) << endl; - hg.clear(); - parser.Parse(lat, &hg); // exhaustive parse - vector& d = derivs[ci]; - if (!is_last) DecrementDerivation(hg, d, &lm, &rng); - for (unsigned i = 0; i < hg.edges_.size(); ++i) { - TRule& r = *hg.edges_[i].rule_; - if (r.lhs_ == kGoal) - hg.edges_[i].edge_prob_ = prob_t::One(); - else - hg.edges_[i].edge_prob_ = lm.Prob(r); - } - if (!is_last) { - d.clear(); - SampleDerivation(hg, &rng, &d); - IncrementDerivation(hg, derivs[ci], &lm, &rng); - } else { - prob_t p = TotalProb(hg); - dlh *= p; - cerr << " p(sentence) = " << log(p) << "\t" << log(dlh) << endl; - } - if (tofreelist.size() > 200000) { - cerr << "Freeing ... "; - for (unsigned i = 0; i < tofreelist.size(); ++i) - delete tofreelist[i]; - tofreelist.clear(); - cerr << "Freed.\n"; - } - } - double llh = log(lm.Likelihood()); - cerr << "LLH=" << llh << "\tENTROPY=" << (-llh / log(2) / toks) << "\tPPL=" << pow(2, -llh / log(2) / toks) << endl; - if (SS % 10 == 9) lm.ResampleHyperparameters(&rng); - if (is_last) { - double z = log(dlh); - cerr << "TOTAL_PROB=" << z << "\tENTROPY=" << (-z / log(2) / toks) << "\tPPL=" << pow(2, -z / log(2) / toks) << endl; - } - } - for (unsigned i = 0; i < nt_vocab.size(); ++i) - cerr << lm.nts[i] << endl; - return 0; -} - diff --git a/gi/pf/make-freq-bins.pl b/gi/pf/make-freq-bins.pl deleted file mode 100755 index fdcd3555..00000000 --- a/gi/pf/make-freq-bins.pl +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $BASE = 6; -my $CUTOFF = 3; - -my %d; -my $num = 0; -while(<>){ - chomp; - my @words = split /\s+/; - for my $w (@words) {$d{$w}++; $num++;} -} - -my @vocab = sort {$d{$b} <=> $d{$a}} keys %d; - -for (my $i=0; $i -#include - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -static bool verbose = false; - -struct Model { - - Model() : bp(), base(0.2, 0.6) , ccrps(5, CCRP(0.8, 0.5)) {} - - double p0(int x) const { - assert(x > 0); - assert(x < 5); - return 1.0/4.0; - } - - double llh() const { - double lh = bp + base.log_crp_prob(); - for (int ctx = 1; ctx < 5; ++ctx) - lh += ccrps[ctx].log_crp_prob(); - return lh; - } - - double prob(int ctx, int x) const { - assert(ctx > 0 && ctx < 5); - return ccrps[ctx].prob(x, base.prob(x, p0(x))); - } - - void increment(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].increment(x, base.prob(x, p0(x)), &rng)) { - if (base.increment(x, p0(x), &rng)) { - bp += log(1.0 / 4.0); - } - } - } - - // this is just a biased estimate - double est_base_prob(int x) { - return (x + 1) * x / 40.0; - } - - void increment_is(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - SampleSet ss; - const int PARTICLES = 25; - vector > s1s(PARTICLES, CCRP(0.5,0.5)); - vector > sbs(PARTICLES, CCRP(0.5,0.5)); - vector sp0s(PARTICLES); - - CCRP s1 = ccrps[ctx]; - CCRP sb = base; - double sp0 = bp; - for (int pp = 0; pp < PARTICLES; ++pp) { - if (pp > 0) { - ccrps[ctx] = s1; - base = sb; - bp = sp0; - } - - double q = 1; - double gamma = 1; - double est_p = est_base_prob(x); - //base.prob(x, p0(x)) + rng.next() * 0.1; - if (ccrps[ctx].increment(x, est_p, &rng, &q)) { - gamma = q * base.prob(x, p0(x)); - q *= est_p; - if (verbose) cerr << "(DP-base draw) "; - double qq = -1; - if (base.increment(x, p0(x), &rng, &qq)) { - if (verbose) cerr << "(G0 draw) "; - bp += log(p0(x)); - qq *= p0(x); - } - } else { gamma = q; } - double w = gamma / q; - if (verbose) - cerr << "gamma=" << gamma << " q=" << q << "\tw=" << w << endl; - ss.add(w); - s1s[pp] = ccrps[ctx]; - sbs[pp] = base; - sp0s[pp] = bp; - } - int ps = rng.SelectSample(ss); - ccrps[ctx] = s1s[ps]; - base = sbs[ps]; - bp = sp0s[ps]; - if (verbose) { - cerr << "SELECTED: " << ps << endl; - static int cc = 0; cc++; if (cc ==10) exit(1); - } - } - - void decrement(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].decrement(x, &rng)) { - if (base.decrement(x, &rng)) { - bp -= log(p0(x)); - } - } - } - - double bp; - CCRP base; - vector > ccrps; - -}; - -int main(int argc, char** argv) { - if (argc > 1) { verbose = true; } - vector counts(15, 0); - vector tcounts(15, 0); - int points[] = {1,2, 2,2, 3,2, 4,1, 3, 4, 3, 3, 2, 3, 4, 1, 4, 1, 3, 2, 1, 3, 1, 4, 0, 0}; - double tlh = 0; - double tt = 0; - for (int n = 0; n < 1000; ++n) { - if (n % 10 == 0) cerr << '.'; - if ((n+1) % 400 == 0) cerr << " [" << (n+1) << "]\n"; - Model m; - for (int *x = points; *x; x += 2) - m.increment(x[0], x[1]); - - for (int j = 0; j < 24; ++j) { - for (int *x = points; *x; x += 2) { - if (rng.next() < 0.8) { - m.decrement(x[0], x[1]); - m.increment_is(x[0], x[1]); - } - } - } - counts[m.base.num_customers()]++; - tcounts[m.base.num_tables()]++; - tlh += m.llh(); - tt += 1.0; - } - cerr << "mean LLH = " << (tlh / tt) << endl; - for (int i = 0; i < 15; ++i) - cerr << i << ": " << (counts[i] / tt) << "\t" << (tcounts[i] / tt) << endl; -} - diff --git a/gi/pf/monotonic_pseg.h b/gi/pf/monotonic_pseg.h deleted file mode 100644 index 10d171fe..00000000 --- a/gi/pf/monotonic_pseg.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef _MONOTONIC_PSEG_H_ -#define _MONOTONIC_PSEG_H_ - -#include - -#include "prob.h" -#include "ccrp_nt.h" -#include "trule.h" -#include "base_distributions.h" - -template -struct MonotonicParallelSegementationModel { - explicit MonotonicParallelSegementationModel(BaseMeasure& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), stop(1.0) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRulesAndStops(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - if (rules.size()) IncrementContinue(rules.size() - 1); - IncrementStop(); - } - - void DecrementRulesAndStops(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - if (rules.size()) { - DecrementContinue(rules.size() - 1); - DecrementStop(); - } - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - q.logeq(stop.log_crp_prob()); - p *= q; - return p; - } - - void IncrementStop() { - stop.increment(true); - } - - void IncrementContinue(int n = 1) { - for (int i = 0; i < n; ++i) - stop.increment(false); - } - - void DecrementStop() { - stop.decrement(true); - } - - void DecrementContinue(int n = 1) { - for (int i = 0; i < n; ++i) - stop.decrement(false); - } - - prob_t StopProbability() const { - return prob_t(stop.prob(true, 0.5)); - } - - prob_t ContinueProbability() const { - return prob_t(stop.prob(false, 0.5)); - } - - const BaseMeasure& rp0; - prob_t base; - CCRP_NoTable rules; - CCRP_NoTable stop; -}; - -#endif - diff --git a/gi/pf/ngram_base.cc b/gi/pf/ngram_base.cc deleted file mode 100644 index 1299f06f..00000000 --- a/gi/pf/ngram_base.cc +++ /dev/null @@ -1,69 +0,0 @@ -#include "ngram_base.h" - -#include "lm/model.hh" -#include "tdict.h" - -using namespace std; - -namespace { -struct GICSVMapper : public lm::EnumerateVocab { - GICSVMapper(vector* out) : out_(out), kLM_UNKNOWN_TOKEN(0) { out_->clear(); } - void Add(lm::WordIndex index, const StringPiece &str) { - const WordID cdec_id = TD::Convert(str.as_string()); - if (cdec_id >= out_->size()) - out_->resize(cdec_id + 1, kLM_UNKNOWN_TOKEN); - (*out_)[cdec_id] = index; - } - vector* out_; - const lm::WordIndex kLM_UNKNOWN_TOKEN; -}; -} - -struct FixedNgramBaseImpl { - FixedNgramBaseImpl(const string& param) { - GICSVMapper vm(&cdec2klm_map_); - lm::ngram::Config conf; - conf.enumerate_vocab = &vm; - cerr << "Reading character LM from " << param << endl; - model = new lm::ngram::ProbingModel(param.c_str(), conf); - order = model->Order(); - kEOS = MapWord(TD::Convert("")); - assert(kEOS > 0); - } - - lm::WordIndex MapWord(const WordID w) const { - if (w < cdec2klm_map_.size()) return cdec2klm_map_[w]; - return 0; - } - - ~FixedNgramBaseImpl() { delete model; } - - prob_t StringProbability(const vector& s) const { - lm::ngram::State state = model->BeginSentenceState(); - double prob = 0; - for (unsigned i = 0; i < s.size(); ++i) { - const lm::ngram::State scopy(state); - prob += model->Score(scopy, MapWord(s[i]), state); - } - const lm::ngram::State scopy(state); - prob += model->Score(scopy, kEOS, state); - prob_t p; p.logeq(prob * log(10)); - return p; - } - - lm::ngram::ProbingModel* model; - unsigned order; - vector cdec2klm_map_; - lm::WordIndex kEOS; -}; - -FixedNgramBase::~FixedNgramBase() { delete impl; } - -FixedNgramBase::FixedNgramBase(const string& lmfname) { - impl = new FixedNgramBaseImpl(lmfname); -} - -prob_t FixedNgramBase::StringProbability(const vector& s) const { - return impl->StringProbability(s); -} - diff --git a/gi/pf/ngram_base.h b/gi/pf/ngram_base.h deleted file mode 100644 index 4ea999f3..00000000 --- a/gi/pf/ngram_base.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef _NGRAM_BASE_H_ -#define _NGRAM_BASE_H_ - -#include -#include -#include "trule.h" -#include "wordid.h" -#include "prob.h" - -struct FixedNgramBaseImpl; -struct FixedNgramBase { - FixedNgramBase(const std::string& lmfname); - ~FixedNgramBase(); - prob_t StringProbability(const std::vector& s) const; - - prob_t operator()(const TRule& rule) const { - return StringProbability(rule.e_); - } - - private: - FixedNgramBaseImpl* impl; - -}; - -#endif diff --git a/gi/pf/nuisance_test.cc b/gi/pf/nuisance_test.cc deleted file mode 100644 index fc0af9cb..00000000 --- a/gi/pf/nuisance_test.cc +++ /dev/null @@ -1,161 +0,0 @@ -#include "ccrp.h" - -#include -#include - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -ostream& operator<<(ostream&os, const vector& v) { - os << '[' << v[0]; - if (v.size() == 2) os << ' ' << v[1]; - return os << ']'; -} - -struct Base { - Base() : llh(), v(2), v1(1), v2(1), crp(0.25, 0.5) {} - inline double p0(const vector& x) const { - double p = 0.75; - if (x.size() == 2) p = 0.25; - p *= 1.0 / 3.0; - if (x.size() == 2) p *= 1.0 / 3.0; - return p; - } - double est_deriv_prob(int a, int b, int seg) const { - assert(a > 0 && a < 4); // a \in {1,2,3} - assert(b > 0 && b < 4); // b \in {1,2,3} - assert(seg == 0 || seg == 1); // seg \in {0,1} - if (seg == 0) { - v[0] = a; - v[1] = b; - return crp.prob(v, p0(v)); - } else { - v1[0] = a; - v2[0] = b; - return crp.prob(v1, p0(v1)) * crp.prob(v2, p0(v2)); - } - } - double est_marginal_prob(int a, int b) const { - return est_deriv_prob(a,b,0) + est_deriv_prob(a,b,1); - } - int increment(int a, int b, double* pw = NULL) { - double p1 = est_deriv_prob(a, b, 0); - double p2 = est_deriv_prob(a, b, 1); - //p1 = 0.5; p2 = 0.5; - int seg = rng.SelectSample(p1,p2); - double tmp = 0; - if (!pw) pw = &tmp; - double& w = *pw; - if (seg == 0) { - v[0] = a; - v[1] = b; - w = crp.prob(v, p0(v)) / p1; - if (crp.increment(v, p0(v), &rng)) { - llh += log(p0(v)); - } - } else { - v1[0] = a; - w = crp.prob(v1, p0(v1)) / p2; - if (crp.increment(v1, p0(v1), &rng)) { - llh += log(p0(v1)); - } - v2[0] = b; - w *= crp.prob(v2, p0(v2)); - if (crp.increment(v2, p0(v2), &rng)) { - llh += log(p0(v2)); - } - } - return seg; - } - void increment(int a, int b, int seg) { - if (seg == 0) { - v[0] = a; - v[1] = b; - if (crp.increment(v, p0(v), &rng)) { - llh += log(p0(v)); - } - } else { - v1[0] = a; - if (crp.increment(v1, p0(v1), &rng)) { - llh += log(p0(v1)); - } - v2[0] = b; - if (crp.increment(v2, p0(v2), &rng)) { - llh += log(p0(v2)); - } - } - } - void decrement(int a, int b, int seg) { - if (seg == 0) { - v[0] = a; - v[1] = b; - if (crp.decrement(v, &rng)) { - llh -= log(p0(v)); - } - } else { - v1[0] = a; - if (crp.decrement(v1, &rng)) { - llh -= log(p0(v1)); - } - v2[0] = b; - if (crp.decrement(v2, &rng)) { - llh -= log(p0(v2)); - } - } - } - double log_likelihood() const { - return llh + crp.log_crp_prob(); - } - double llh; - mutable vector v, v1, v2; - CCRP > crp; -}; - -int main(int argc, char** argv) { - double tl = 0; - const int ITERS = 1000; - const int PARTICLES = 20; - const int DATAPOINTS = 50; - WordID x = TD::Convert("souvenons"); - WordID y = TD::Convert("remember"); - vector src; TD::ConvertSentence("s o u v e n o n s", &src); - vector trg; TD::ConvertSentence("r e m e m b e r", &trg); -// Transliterations xx; -// xx.Initialize(x, src, y, trg); -// return 1; - - for (int j = 0; j < ITERS; ++j) { - Base b; - vector segs(DATAPOINTS); - SampleSet ss; - vector sss; - for (int i = 0; i < DATAPOINTS; i++) { - ss.clear(); - sss.clear(); - int x = ((i / 10) % 3) + 1; - int y = (i % 3) + 1; - //double ep = b.est_marginal_prob(x,y); - //cerr << "est p(" << x << "," << y << ") = " << ep << endl; - for (int n = 0; n < PARTICLES; ++n) { - double w; - int seg = b.increment(x,y,&w); - //cerr << seg << " w=" << w << endl; - ss.add(w); - sss.push_back(seg); - b.decrement(x,y,seg); - } - int seg = sss[rng.SelectSample(ss)]; - b.increment(x, y, seg); - //cerr << "Selected: " << seg << endl; - //return 1; - segs[i] = seg; - } - tl += b.log_likelihood(); - } - cerr << "LLH=" << tl / ITERS << endl; -} - diff --git a/gi/pf/os_phrase.h b/gi/pf/os_phrase.h deleted file mode 100644 index dfe40cb1..00000000 --- a/gi/pf/os_phrase.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef _OS_PHRASE_H_ -#define _OS_PHRASE_H_ - -#include -#include -#include "tdict.h" - -inline std::ostream& operator<<(std::ostream& os, const std::vector& p) { - os << '['; - for (int i = 0; i < p.size(); ++i) - os << (i==0 ? "" : " ") << TD::Convert(p[i]); - return os << ']'; -} - -#endif diff --git a/gi/pf/pf.h b/gi/pf/pf.h deleted file mode 100644 index ede7cda8..00000000 --- a/gi/pf/pf.h +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef _PF_H_ -#define _PF_H_ - -#include -#include -#include "sampler.h" -#include "prob.h" - -template -struct ParticleRenormalizer { - void operator()(std::vector* pv) const { - if (pv->empty()) return; - prob_t z = prob_t::Zero(); - for (unsigned i = 0; i < pv->size(); ++i) - z += (*pv)[i].weight; - assert(z > prob_t::Zero()); - for (unsigned i = 0; i < pv->size(); ++i) - (*pv)[i].weight /= z; - } -}; - -template -struct MultinomialResampleFilter { - explicit MultinomialResampleFilter(MT19937* rng) : rng_(rng) {} - - void operator()(std::vector* pv) { - if (pv->empty()) return; - std::vector& ps = *pv; - SampleSet ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - std::vector nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[rng_->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); - } - - private: - MT19937* rng_; -}; - -template -struct SystematicResampleFilter { - explicit SystematicResampleFilter(MT19937* rng) : rng_(rng), renorm_() {} - - void operator()(std::vector* pv) { - if (pv->empty()) return; - renorm_(pv); - std::vector& ps = *pv; - std::vector nps; nps.reserve(ps.size()); - double lower = 0, upper = 0; - const double skip = 1.0 / ps.size(); - double u_j = rng_->next() * skip; - //std::cerr << "u_0: " << u_j << std::endl; - int j = 0; - for (unsigned i = 0; i < ps.size(); ++i) { - upper += ps[i].weight.as_float(); - //std::cerr << "lower: " << lower << " upper: " << upper << std::endl; - // how many children does ps[i] have? - while (u_j < lower) { u_j += skip; ++j; } - while (u_j >= lower && u_j <= upper) { - assert(j < ps.size()); - nps.push_back(ps[i]); - u_j += skip; - //std::cerr << " add u_j=" << u_j << std::endl; - ++j; - } - lower = upper; - } - //std::cerr << ps.size() << " " << nps.size() << "\n"; - assert(ps.size() == nps.size()); - //exit(1); - ps.swap(nps); - } - - private: - MT19937* rng_; - ParticleRenormalizer renorm_; -}; - -#endif diff --git a/gi/pf/pf_test.cc b/gi/pf/pf_test.cc deleted file mode 100644 index 296e7285..00000000 --- a/gi/pf/pf_test.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include "ccrp.h" - -#include -#include - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -static bool verbose = false; - -struct Model { - - Model() : bp(), base(0.2, 0.6) , ccrps(5, CCRP(0.8, 0.5)) {} - - double p0(int x) const { - assert(x > 0); - assert(x < 5); - return 1.0/4.0; - } - - double llh() const { - double lh = bp + base.log_crp_prob(); - for (int ctx = 1; ctx < 5; ++ctx) - lh += ccrps[ctx].log_crp_prob(); - return lh; - } - - double prob(int ctx, int x) const { - assert(ctx > 0 && ctx < 5); - return ccrps[ctx].prob(x, base.prob(x, p0(x))); - } - - void increment(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].increment(x, base.prob(x, p0(x)), &rng)) { - if (base.increment(x, p0(x), &rng)) { - bp += log(1.0 / 4.0); - } - } - } - - // this is just a biased estimate - double est_base_prob(int x) { - return (x + 1) * x / 40.0; - } - - void increment_is(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - SampleSet ss; - const int PARTICLES = 25; - vector > s1s(PARTICLES, CCRP(0.5,0.5)); - vector > sbs(PARTICLES, CCRP(0.5,0.5)); - vector sp0s(PARTICLES); - - CCRP s1 = ccrps[ctx]; - CCRP sb = base; - double sp0 = bp; - for (int pp = 0; pp < PARTICLES; ++pp) { - if (pp > 0) { - ccrps[ctx] = s1; - base = sb; - bp = sp0; - } - - double q = 1; - double gamma = 1; - double est_p = est_base_prob(x); - //base.prob(x, p0(x)) + rng.next() * 0.1; - if (ccrps[ctx].increment(x, est_p, &rng, &q)) { - gamma = q * base.prob(x, p0(x)); - q *= est_p; - if (verbose) cerr << "(DP-base draw) "; - double qq = -1; - if (base.increment(x, p0(x), &rng, &qq)) { - if (verbose) cerr << "(G0 draw) "; - bp += log(p0(x)); - qq *= p0(x); - } - } else { gamma = q; } - double w = gamma / q; - if (verbose) - cerr << "gamma=" << gamma << " q=" << q << "\tw=" << w << endl; - ss.add(w); - s1s[pp] = ccrps[ctx]; - sbs[pp] = base; - sp0s[pp] = bp; - } - int ps = rng.SelectSample(ss); - ccrps[ctx] = s1s[ps]; - base = sbs[ps]; - bp = sp0s[ps]; - if (verbose) { - cerr << "SELECTED: " << ps << endl; - static int cc = 0; cc++; if (cc ==10) exit(1); - } - } - - void decrement(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].decrement(x, &rng)) { - if (base.decrement(x, &rng)) { - bp -= log(p0(x)); - } - } - } - - double bp; - CCRP base; - vector > ccrps; - -}; - -int main(int argc, char** argv) { - if (argc > 1) { verbose = true; } - vector counts(15, 0); - vector tcounts(15, 0); - int points[] = {1,2, 2,2, 3,2, 4,1, 3, 4, 3, 3, 2, 3, 4, 1, 4, 1, 3, 2, 1, 3, 1, 4, 0, 0}; - double tlh = 0; - double tt = 0; - for (int n = 0; n < 1000; ++n) { - if (n % 10 == 0) cerr << '.'; - if ((n+1) % 400 == 0) cerr << " [" << (n+1) << "]\n"; - Model m; - for (int *x = points; *x; x += 2) - m.increment(x[0], x[1]); - - for (int j = 0; j < 24; ++j) { - for (int *x = points; *x; x += 2) { - if (rng.next() < 0.8) { - m.decrement(x[0], x[1]); - m.increment_is(x[0], x[1]); - } - } - } - counts[m.base.num_customers()]++; - tcounts[m.base.num_tables()]++; - tlh += m.llh(); - tt += 1.0; - } - cerr << "mean LLH = " << (tlh / tt) << endl; - for (int i = 0; i < 15; ++i) - cerr << i << ": " << (counts[i] / tt) << "\t" << (tcounts[i] / tt) << endl; -} - diff --git a/gi/pf/pfbrat.cc b/gi/pf/pfbrat.cc deleted file mode 100644 index 832f22cf..00000000 --- a/gi/pf/pfbrat.cc +++ /dev/null @@ -1,543 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "cfg_wfst_composer.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; -struct FSTState; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -struct ConditionalBase { - explicit ConditionalBase(const double m1mixture, const unsigned vocab_e_size, const string& model1fname) : - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size), - kNULL(TD::Convert("")) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - LoadModel1(model1fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // return logp0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - const int flen = rule.f_.size(); - const int elen = rule.e_.size(); - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = rule.e_[i]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? kNULL : rule.f_[j]; - const map::const_iterator it = ttable[src].find(trg); - if (it != ttable[src].end()) { - tp += kM1MIXTURE * it->second; - } - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - return p; - } - - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; - const WordID kNULL; - vector > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(3),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(3),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UniphraseLM { - UniphraseLM(const vector >& corpus, - const set& vocab, - const po::variables_map& conf) : - phrases_(1,1), - gen_(1,1), - corpus_(corpus), - uniform_word_(1.0 / vocab.size()), - gen_p0_(0.5), - p_end_(0.5), - use_poisson_(conf.count("poisson_length") > 0) {} - - void ResampleHyperparameters(MT19937* rng) { - phrases_.resample_hyperparameters(rng); - gen_.resample_hyperparameters(rng); - cerr << " " << phrases_.alpha(); - } - - CCRP_NoTable > phrases_; - CCRP_NoTable gen_; - vector > z_; // z_[i] is there a phrase boundary after the ith word - const vector >& corpus_; - const double uniform_word_; - const double gen_p0_; - const double p_end_; // in base length distribution, p of the end of a phrase - const bool use_poisson_; -}; - -struct Reachability { - boost::multi_array edges; // edges[src_covered][trg_covered][x][trg_delta] is this edge worth exploring? - boost::multi_array max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; - }; - - struct NState { - NState() : next_src_covered(), next_trg_covered() {} - NState(int i, int j) : next_src_covered(i), next_trg_covered(j) {} - int next_src_covered; - int next_trg_covered; - }; - - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - cerr << "Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - assert(a[srclen][trglen].size() > 0); - - typedef boost::multi_array rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); -// typedef boost::multi_array, 2> narray_type; -// narray_type b(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } -// const NState nstate(i,j); - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; -// b[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(nstate); - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - cerr << " MAX SRC DELTA[0][0] = " << max_src_delta[0][0] << endl; - assert(max_src_delta[0][0] > 0); - //cerr << "First cell contains " << b[0][0].size() << " forward pointers\n"; - //for (int i = 0; i < b[0][0].size(); ++i) { - // cerr << " -> (" << b[0][0][i].next_src_covered << "," << b[0][0][i].next_trg_covered << ")\n"; - //} - } -}; - -ostream& operator<<(ostream& os, const FSTState& q); -struct FSTState { - explicit FSTState(int src_size) : - trg_covered_(), - src_covered_(), - src_coverage_(src_size) {} - - FSTState(short trg_covered, short src_covered, const vector& src_coverage, const vector& src_prefix) : - trg_covered_(trg_covered), - src_covered_(src_covered), - src_coverage_(src_coverage), - src_prefix_(src_prefix) { - if (src_coverage_.size() == src_covered) { - assert(src_prefix.size() == 0); - } - } - - // if we extend by the word at src_position, what are - // the next states that are reachable and lie on a valid - // path to the final state? - vector Extensions(int src_position, int src_len, int trg_len, const Reachability& r) const { - assert(src_position < src_coverage_.size()); - if (src_coverage_[src_position]) { - cerr << "Trying to extend " << *this << " with position " << src_position << endl; - abort(); - } - vector ncvg = src_coverage_; - ncvg[src_position] = true; - - vector res; - const int trg_remaining = trg_len - trg_covered_; - if (trg_remaining <= 0) { - cerr << "Target appears to have been covered: " << *this << " (trg_len=" << trg_len << ",trg_covered=" << trg_covered_ << ")" << endl; - abort(); - } - const int src_remaining = src_len - src_covered_; - if (src_remaining <= 0) { - cerr << "Source appears to have been covered: " << *this << endl; - abort(); - } - - for (int tc = 1; tc <= kMAX_TRG_PHRASE; ++tc) { - if (r.edges[src_covered_][trg_covered_][src_prefix_.size() + 1][tc]) { - int nc = src_prefix_.size() + 1 + src_covered_; - res.push_back(FSTState(trg_covered_ + tc, nc, ncvg, vector())); - } - } - - if ((src_prefix_.size() + 1) < r.max_src_delta[src_covered_][trg_covered_]) { - vector nsp = src_prefix_; - nsp.push_back(src_position); - res.push_back(FSTState(trg_covered_, src_covered_, ncvg, nsp)); - } - - if (res.size() == 0) { - cerr << *this << " can't be extended!\n"; - abort(); - } - return res; - } - - short trg_covered_, src_covered_; - vector src_coverage_; - vector src_prefix_; -}; -bool operator<(const FSTState& q, const FSTState& r) { - if (q.trg_covered_ != r.trg_covered_) return q.trg_covered_ < r.trg_covered_; - if (q.src_covered_!= r.src_covered_) return q.src_covered_ < r.src_covered_; - if (q.src_coverage_ != r.src_coverage_) return q.src_coverage_ < r.src_coverage_; - return q.src_prefix_ < r.src_prefix_; -} - -ostream& operator<<(ostream& os, const FSTState& q) { - os << "[" << q.trg_covered_ << " : "; - for (int i = 0; i < q.src_coverage_.size(); ++i) - os << q.src_coverage_[i]; - os << " : <"; - for (int i = 0; i < q.src_prefix_.size(); ++i) { - if (i != 0) os << ' '; - os << q.src_prefix_[i]; - } - return os << ">]"; -} - -struct MyModel { - MyModel(ConditionalBase& rcp0) : rp0(rcp0) {} - typedef unordered_map, CCRP_NoTable, boost::hash > > SrcToRuleCRPMap; - - void DecrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - it->second.decrement(rule); - if (it->second.num_customers() == 0) rules.erase(it); - } - - void IncrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) { - CCRP_NoTable crp(1,1); - it = rules.insert(make_pair(rule.f_, crp)).first; - } - it->second.increment(rule); - } - - // conditioned on rule.f_ - prob_t RuleConditionalProbability(const TRule& rule) const { - const prob_t base = rp0(rule); - SrcToRuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) { - return base; - } else { - const double lp = it->second.logprob(rule, log(base)); - prob_t q; q.logeq(lp); - return q; - } - } - - const ConditionalBase& rp0; - SrcToRuleCRPMap rules; -}; - -struct MyFST : public WFST { - MyFST(const vector& ssrc, const vector& strg, MyModel* m) : - src(ssrc), trg(strg), - r(src.size(),trg.size(),kMAX_SRC_PHRASE, kMAX_TRG_PHRASE), - model(m) { - FSTState in(src.size()); - cerr << " INIT: " << in << endl; - init = GetNode(in); - for (int i = 0; i < in.src_coverage_.size(); ++i) in.src_coverage_[i] = true; - in.src_covered_ = src.size(); - in.trg_covered_ = trg.size(); - cerr << "FINAL: " << in << endl; - final = GetNode(in); - } - virtual const WFSTNode* Final() const; - virtual const WFSTNode* Initial() const; - - const WFSTNode* GetNode(const FSTState& q); - map > m; - const vector& src; - const vector& trg; - Reachability r; - const WFSTNode* init; - const WFSTNode* final; - MyModel* model; -}; - -struct MyNode : public WFSTNode { - MyNode(const FSTState& q, MyFST* fst) : state(q), container(fst) {} - virtual vector > ExtendInput(unsigned srcindex) const; - const FSTState state; - mutable MyFST* container; -}; - -vector > MyNode::ExtendInput(unsigned srcindex) const { - cerr << "EXTEND " << state << " with " << srcindex << endl; - vector ext = state.Extensions(srcindex, container->src.size(), container->trg.size(), container->r); - vector > res(ext.size()); - for (unsigned i = 0; i < ext.size(); ++i) { - res[i].first = container->GetNode(ext[i]); - if (ext[i].src_prefix_.size() == 0) { - const unsigned trg_from = state.trg_covered_; - const unsigned trg_to = ext[i].trg_covered_; - const unsigned prev_prfx_size = state.src_prefix_.size(); - res[i].second.reset(new TRule); - res[i].second->lhs_ = -TD::Convert("X"); - vector& src = res[i].second->f_; - vector& trg = res[i].second->e_; - src.resize(prev_prfx_size + 1); - for (unsigned j = 0; j < prev_prfx_size; ++j) - src[j] = container->src[state.src_prefix_[j]]; - src[prev_prfx_size] = container->src[srcindex]; - for (unsigned j = trg_from; j < trg_to; ++j) - trg.push_back(container->trg[j]); - res[i].second->scores_.set_value(FD::Convert("Proposal"), log(container->model->RuleConditionalProbability(*res[i].second))); - } - } - return res; -} - -const WFSTNode* MyFST::GetNode(const FSTState& q) { - boost::shared_ptr& res = m[q]; - if (!res) { - res.reset(new MyNode(q, this)); - } - return &*res; -} - -const WFSTNode* MyFST::Final() const { - return final; -} - -const WFSTNode* MyFST::Initial() const { - return init; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - ConditionalBase lp0(conf["model1_interpolation_weight"].as(), - vocabe.size(), - conf["model1"].as()); - MyModel m(lp0); - - TRule x("[X] ||| kAnwntR myN ||| at the convent ||| 0"); - m.IncrementRule(x); - TRule y("[X] ||| nY dyN ||| gave ||| 0"); - m.IncrementRule(y); - - - MyFST fst(corpusf[0], corpuse[0], &m); - ifstream in("./kimura.g"); - assert(in); - CFG_WFSTComposer comp(fst); - Hypergraph hg; - bool succeed = comp.Compose(&in, &hg); - hg.PrintGraphviz(); - if (succeed) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } - -#if 0 - ifstream in2("./amnabooks.g"); - assert(in2); - MyFST fst2(corpusf[1], corpuse[1], &m); - CFG_WFSTComposer comp2(fst2); - Hypergraph hg2; - bool succeed2 = comp2.Compose(&in2, &hg2); - if (succeed2) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } -#endif - - SparseVector w; w.set_value(FD::Convert("Proposal"), 1.0); - hg.Reweight(w); - cerr << ViterbiFTree(hg) << endl; - return 0; -} - diff --git a/gi/pf/pfdist.cc b/gi/pf/pfdist.cc deleted file mode 100644 index a3e46064..00000000 --- a/gi/pf/pfdist.cc +++ /dev/null @@ -1,598 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "pf.h" -#include "base_distributions.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("particles,p",po::value()->default_value(30),"Number of particles") - ("filter_frequency,f",po::value()->default_value(5),"Number of time steps between filterings") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -#if 0 -struct MyConditionalModel { - MyConditionalModel(PhraseConditionalBase& rcp0) : rp0(&rcp0), base(prob_t::One()), src_phrases(1,1), src_jumps(200, CCRP_NoTable(1,1)) {} - - prob_t srcp0(const vector& src) const { - prob_t p(1.0 / 3000.0); - p.poweq(src.size()); - prob_t lenp; lenp.logeq(log_poisson(src.size(), 1.0)); - p *= lenp; - return p; - } - - void DecrementRule(const TRule& rule) { - const RuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - if (it->second.decrement(rule)) { - base /= (*rp0)(rule); - if (it->second.num_customers() == 0) - rules.erase(it); - } - if (src_phrases.decrement(rule.f_)) - base /= srcp0(rule.f_); - } - - void IncrementRule(const TRule& rule) { - RuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) - it = rules.insert(make_pair(rule.f_, CCRP_NoTable(1,1))).first; - if (it->second.increment(rule)) { - base *= (*rp0)(rule); - } - if (src_phrases.increment(rule.f_)) - base *= srcp0(rule.f_); - } - - void IncrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - const prob_t p0 = (*rp0)(rule); - prob_t srcp; srcp.logeq(src_phrases.logprob(rule.f_, log(srcp0(rule.f_)))); - const RuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) return srcp * p0; - const double lp = it->second.logprob(rule, log(p0)); - prob_t q; q.logeq(lp); - return q * srcp; - } - - prob_t Likelihood() const { - prob_t p = base; - for (RuleCRPMap::const_iterator it = rules.begin(); - it != rules.end(); ++it) { - prob_t cl; cl.logeq(it->second.log_crp_prob()); - p *= cl; - } - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseConditionalBase* rp0; - prob_t base; - typedef unordered_map, CCRP_NoTable, boost::hash > > RuleCRPMap; - RuleCRPMap rules; - CCRP_NoTable > src_phrases; - vector > src_jumps; -}; - -#endif - -struct MyJointModel { - MyJointModel(PhraseJointBase& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), src_jumps(200, CCRP_NoTable(1,1)) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseJointBase& rp0; - prob_t base; - CCRP_NoTable rules; - vector > src_jumps; -}; - -struct BackwardEstimate { - BackwardEstimate(const Model1& m1, const vector& src, const vector& trg) : - model1_(m1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector r(src_.size() + 1); r.clear(); - r.push_back(0); // NULL word - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - } - return e; - } - const Model1& model1_; - const vector& src_; - const vector& trg_; - mutable unordered_map, map, boost::hash > > cache_; -}; - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector& src, const vector& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector r(src_.size() + 1); r.clear(); - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector& src_; - const vector& trg_; - mutable unordered_map, map, boost::hash > > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov(), prev_pos(-1) {} - prob_t weight; - prob_t gamma_last; - vector src_jumps; - vector rules; - vector src_cv; - int src_cov; - int trg_cov; - int prev_pos; -}; - -ostream& operator<<(ostream& o, const vector& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[cv=" << p.src_cv << " src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " last_pos=" << p.prev_pos << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - const unsigned particles = conf["particles"].as(); - const unsigned samples = conf["samples"].as(); - const unsigned rejuv_freq = conf["filter_frequency"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as()); - Model1 invm1(conf["inverse_model1"].as()); - -#if 0 - PhraseConditionalBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size()); - MyConditionalModel m(lp0); -#else - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MyJointModel m(lp0); -#endif - - MultinomialResampleFilter filter(&rng); - cerr << "Initializing reachability limits...\n"; - vector ps(corpusf.size()); - vector reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector tmp_p(10000); // work space - SampleSet pfss; - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector& src = corpusf[ci]; - vector& trg = corpuse[ci]; - m.DecrementRules(ps[ci].rules); - m.DecrementJumps(ps[ci].src_jumps, src.size()); - - //BackwardEstimate be(m1, src, trg); - BackwardEstimateSym be(m1, invm1, src, trg); - const Reachability& r = reaches[ci]; - vector lps(particles); - - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - p.src_cv.resize(src.size(), false); - } - - bool all_complete = false; - while(!all_complete) { - SampleSet ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - filter(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - int first_uncovered = src.size(); - int last_uncovered = -1; - for (int i = 0; i < src.size(); ++i) { - const bool is_uncovered = !p.src_cv[i]; - if (i < first_uncovered && is_uncovered) first_uncovered = i; - if (is_uncovered && i > last_uncovered) last_uncovered = i; - } - assert(last_uncovered > -1); - assert(first_uncovered < src.size()); - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - const int last_possible_start = last_uncovered - src_len + 1; - assert(last_possible_start >= 0); - //cerr << src_len << "," << trg_len << " is allowed. E=" << TD::GetString(x.e_) << endl; - //cerr << " first_uncovered=" << first_uncovered << " last_possible_start=" << last_possible_start << endl; - for (int i = first_uncovered; i <= last_possible_start; ++i) { - if (p.src_cv[i]) continue; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - int gap_add = 0; - bool bad = false; - prob_t jp = prob_t::One(); - int prev_pos = p.prev_pos; - for (int j = 0; j < src_len; ++j) { - if ((j + i + gap_add) == src.size()) { bad = true; break; } - while ((i+j+gap_add) < src.size() && p.src_cv[i + j + gap_add]) { ++gap_add; } - if ((j + i + gap_add) == src.size()) { bad = true; break; } - np.src_cv[i + j + gap_add] = true; - x.f_.push_back(src[i + j + gap_add]); - jp *= m.JumpProbability(i + j + gap_add - prev_pos, src.size()); - int jump = i + j + gap_add - prev_pos; - assert(jump != 0); - np.src_jumps.push_back(jump); - prev_pos = i + j + gap_add; - } - if (bad) continue; - np.prev_pos = prev_pos; - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - if (x.f_.size() != src_len) continue; - prob_t rp = m.RuleProbability(x); - np.gamma_last = rp * jp; - const prob_t u = pow(np.gamma_last * be(np.src_cv, np.trg_cov), 0.2); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << " jump=" << jp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - - const bool completed = (p.trg_cov == trg.size()); - if (completed) { - int last_jump = src.size() - p.prev_pos; - assert(last_jump > 0); - p.src_jumps.push_back(last_jump); - p.weight *= m.JumpProbability(last_jump, src.size()); - } - } - } - } - cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRules(lps[sampled].rules); - m.IncrementJumps(lps[sampled].src_jumps, src.size()); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - for (int sni = 0; sni < 5; ++sni) { - for (int i = 0; i < ps[sni].rules.size(); ++i) { cerr << "\t" << ps[sni].rules[i]->AsString() << endl; } - } - } - return 0; -} - diff --git a/gi/pf/pfdist.new.cc b/gi/pf/pfdist.new.cc deleted file mode 100644 index 3169eb75..00000000 --- a/gi/pf/pfdist.new.cc +++ /dev/null @@ -1,620 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "base_measures.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -shared_ptr prng; - -size_t hash_value(const TRule& r) { - size_t h = boost::hash_value(r.e_); - boost::hash_combine(h, -r.lhs_); - boost::hash_combine(h, boost::hash_value(r.f_)); - return h; -} - -bool operator==(const TRule& a, const TRule& b) { - return (a.lhs_ == b.lhs_ && a.e_ == b.e_ && a.f_ == b.f_); -} - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("particles,p",po::value()->default_value(25),"Number of particles") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector >* f, - vector >* e, - set* vocab_f, - set* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector()); - f->push_back(vector()); - vector& le = e->back(); - vector& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -#if 0 -struct MyConditionalModel { - MyConditionalModel(PhraseConditionalBase& rcp0) : rp0(&rcp0), base(prob_t::One()), src_phrases(1,1), src_jumps(200, CCRP_NoTable(1,1)) {} - - prob_t srcp0(const vector& src) const { - prob_t p(1.0 / 3000.0); - p.poweq(src.size()); - prob_t lenp; lenp.logeq(log_poisson(src.size(), 1.0)); - p *= lenp; - return p; - } - - void DecrementRule(const TRule& rule) { - const RuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - if (it->second.decrement(rule)) { - base /= (*rp0)(rule); - if (it->second.num_customers() == 0) - rules.erase(it); - } - if (src_phrases.decrement(rule.f_)) - base /= srcp0(rule.f_); - } - - void IncrementRule(const TRule& rule) { - RuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) - it = rules.insert(make_pair(rule.f_, CCRP_NoTable(1,1))).first; - if (it->second.increment(rule)) { - base *= (*rp0)(rule); - } - if (src_phrases.increment(rule.f_)) - base *= srcp0(rule.f_); - } - - void IncrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - const prob_t p0 = (*rp0)(rule); - prob_t srcp; srcp.logeq(src_phrases.logprob(rule.f_, log(srcp0(rule.f_)))); - const RuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) return srcp * p0; - const double lp = it->second.logprob(rule, log(p0)); - prob_t q; q.logeq(lp); - return q * srcp; - } - - prob_t Likelihood() const { - prob_t p = base; - for (RuleCRPMap::const_iterator it = rules.begin(); - it != rules.end(); ++it) { - prob_t cl; cl.logeq(it->second.log_crp_prob()); - p *= cl; - } - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseConditionalBase* rp0; - prob_t base; - typedef unordered_map, CCRP_NoTable, boost::hash > > RuleCRPMap; - RuleCRPMap rules; - CCRP_NoTable > src_phrases; - vector > src_jumps; -}; - -#endif - -struct MyJointModel { - MyJointModel(PhraseJointBase& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), src_jumps(200, CCRP_NoTable(1,1)) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseJointBase& rp0; - prob_t base; - CCRP_NoTable rules; - vector > src_jumps; -}; - -struct BackwardEstimate { - BackwardEstimate(const Model1& m1, const vector& src, const vector& trg) : - model1_(m1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector r(src_.size() + 1); r.clear(); - r.push_back(0); // NULL word - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - } - return e; - } - const Model1& model1_; - const vector& src_; - const vector& trg_; - mutable unordered_map, map, boost::hash > > cache_; -}; - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector& src, const vector& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector r(src_.size() + 1); r.clear(); - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector& src_; - const vector& trg_; - mutable unordered_map, map, boost::hash > > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov(), prev_pos(-1) {} - prob_t weight; - prob_t gamma_last; - vector src_jumps; - vector rules; - vector src_cv; - int src_cov; - int trg_cov; - int prev_pos; -}; - -ostream& operator<<(ostream& o, const vector& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[cv=" << p.src_cv << " src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " last_pos=" << p.prev_pos << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -void FilterCrapParticlesAndReweight(vector* pps) { - vector& ps = *pps; - SampleSet ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - vector nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[prng->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - const unsigned particles = conf["particles"].as(); - const unsigned samples = conf["samples"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as()); - Model1 invm1(conf["inverse_model1"].as()); - -#if 0 - PhraseConditionalBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size()); - MyConditionalModel m(lp0); -#else - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MyJointModel m(lp0); -#endif - - cerr << "Initializing reachability limits...\n"; - vector ps(corpusf.size()); - vector reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector tmp_p(10000); // work space - SampleSet pfss; - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector& src = corpusf[ci]; - vector& trg = corpuse[ci]; - m.DecrementRules(ps[ci].rules); - m.DecrementJumps(ps[ci].src_jumps, src.size()); - - //BackwardEstimate be(m1, src, trg); - BackwardEstimateSym be(m1, invm1, src, trg); - const Reachability& r = reaches[ci]; - vector lps(particles); - - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - p.src_cv.resize(src.size(), false); - } - - bool all_complete = false; - while(!all_complete) { - SampleSet ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - FilterCrapParticlesAndReweight(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - const int rejuv_freq = 1; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - int first_uncovered = src.size(); - int last_uncovered = -1; - for (int i = 0; i < src.size(); ++i) { - const bool is_uncovered = !p.src_cv[i]; - if (i < first_uncovered && is_uncovered) first_uncovered = i; - if (is_uncovered && i > last_uncovered) last_uncovered = i; - } - assert(last_uncovered > -1); - assert(first_uncovered < src.size()); - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - const int last_possible_start = last_uncovered - src_len + 1; - assert(last_possible_start >= 0); - //cerr << src_len << "," << trg_len << " is allowed. E=" << TD::GetString(x.e_) << endl; - //cerr << " first_uncovered=" << first_uncovered << " last_possible_start=" << last_possible_start << endl; - for (int i = first_uncovered; i <= last_possible_start; ++i) { - if (p.src_cv[i]) continue; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - int gap_add = 0; - bool bad = false; - prob_t jp = prob_t::One(); - int prev_pos = p.prev_pos; - for (int j = 0; j < src_len; ++j) { - if ((j + i + gap_add) == src.size()) { bad = true; break; } - while ((i+j+gap_add) < src.size() && p.src_cv[i + j + gap_add]) { ++gap_add; } - if ((j + i + gap_add) == src.size()) { bad = true; break; } - np.src_cv[i + j + gap_add] = true; - x.f_.push_back(src[i + j + gap_add]); - jp *= m.JumpProbability(i + j + gap_add - prev_pos, src.size()); - int jump = i + j + gap_add - prev_pos; - assert(jump != 0); - np.src_jumps.push_back(jump); - prev_pos = i + j + gap_add; - } - if (bad) continue; - np.prev_pos = prev_pos; - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - if (x.f_.size() != src_len) continue; - prob_t rp = m.RuleProbability(x); - np.gamma_last = rp * jp; - const prob_t u = pow(np.gamma_last * be(np.src_cv, np.trg_cov), 0.2); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << " jump=" << jp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - - const bool completed = (p.trg_cov == trg.size()); - if (completed) { - int last_jump = src.size() - p.prev_pos; - assert(last_jump > 0); - p.src_jumps.push_back(last_jump); - p.weight *= m.JumpProbability(last_jump, src.size()); - } - } - } - } - cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRules(lps[sampled].rules); - m.IncrementJumps(lps[sampled].src_jumps, src.size()); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - for (int sni = 0; sni < 5; ++sni) { - for (int i = 0; i < ps[sni].rules.size(); ++i) { cerr << "\t" << ps[sni].rules[i]->AsString() << endl; } - } - } - return 0; -} - diff --git a/gi/pf/pfnaive.cc b/gi/pf/pfnaive.cc deleted file mode 100644 index 958ec4e2..00000000 --- a/gi/pf/pfnaive.cc +++ /dev/null @@ -1,284 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "pf.h" -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" -#include "corpus.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value()->default_value(1000),"Number of samples") - ("particles,p",po::value()->default_value(30),"Number of particles") - ("filter_frequency,f",po::value()->default_value(5),"Number of time steps between filterings") - ("input,i",po::value(),"Read parallel data from") - ("max_src_phrase",po::value()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector& src, const vector& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(unsigned src_cov, unsigned trg_cov) const { - assert(src_cov <= src_.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector r(src_.size() + 1); r.clear(); - for (int i = src_cov; i < src_.size(); ++i) - r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector& src_; - const vector& trg_; - mutable unordered_map > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov() {} - prob_t weight; - prob_t gamma_last; - vector rules; - int src_cov; - int trg_cov; -}; - -ostream& operator<<(ostream& o, const vector& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as(); - const unsigned particles = conf["particles"].as(); - const unsigned samples = conf["samples"].as(); - const unsigned rejuv_freq = conf["filter_frequency"].as(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector > corpuse, corpusf; - set vocabe, vocabf; - cerr << "Reading corpus...\n"; - corpus::ReadParallelCorpus(conf["input"].as(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as()); - Model1 invm1(conf["inverse_model1"].as()); - - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel m(alp0); - TRule xx("[X] ||| ms. kimura ||| MS. KIMURA ||| X=0"); - cerr << xx << endl << lp0(xx) << " " << alp0(xx) << endl; - TRule xx12("[X] ||| . ||| PHARMACY . ||| X=0"); - TRule xx21("[X] ||| pharmacy . ||| . ||| X=0"); -// TRule xx22("[X] ||| . ||| . ||| X=0"); - TRule xx22("[X] ||| . ||| THE . ||| X=0"); - cerr << xx12 << "\t" << lp0(xx12) << " " << alp0(xx12) << endl; - cerr << xx21 << "\t" << lp0(xx21) << " " << alp0(xx21) << endl; - cerr << xx22 << "\t" << lp0(xx22) << " " << alp0(xx22) << endl; - - cerr << "Initializing reachability limits...\n"; - vector ps(corpusf.size()); - vector reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector tmp_p(10000); // work space - SampleSet pfss; - SystematicResampleFilter filter(&rng); - // MultinomialResampleFilter filter(&rng); - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector& src = corpusf[ci]; - vector& trg = corpuse[ci]; - m.DecrementRulesAndStops(ps[ci].rules); - const prob_t q_stop = m.StopProbability(); - const prob_t q_cont = m.ContinueProbability(); - cerr << "P(stop)=" << q_stop << "\tP(continue)=" < lps(particles); - - bool all_complete = false; - while(!all_complete) { - SampleSet ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - filter(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - int i = p.src_cov; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - for (int j = 0; j < src_len; ++j) - x.f_.push_back(src[i + j]); - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - const bool stop_now = (np.src_cov == src_len && np.trg_cov == trg_len); - prob_t rp = m.RuleProbability(x) * (stop_now ? q_stop : q_cont); - np.gamma_last = rp; - const prob_t u = pow(np.gamma_last * pow(be(np.src_cov, np.trg_cov), 1.2), 0.1); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - } - } - //cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - //cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - //cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - prob_t wv = prob_t::Zero(); - for (int pp = 0; pp < lps.size(); ++pp) - wv += lps[pp].weight; - for (int pp = 0; pp < lps.size(); ++pp) - lps[pp].weight /= wv; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRulesAndStops(lps[sampled].rules); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - } - return 0; -} - diff --git a/gi/pf/poisson_uniform_word_model.h b/gi/pf/poisson_uniform_word_model.h deleted file mode 100644 index 76204a0e..00000000 --- a/gi/pf/poisson_uniform_word_model.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef _POISSON_UNIFORM_WORD_MODEL_H_ -#define _POISSON_UNIFORM_WORD_MODEL_H_ - -#include -#include -#include "prob.h" -#include "m.h" - -// len ~ Poisson(lambda) -// for (1..len) -// e_i ~ Uniform({Vocabulary}) -struct PoissonUniformWordModel { - explicit PoissonUniformWordModel(const unsigned vocab_size, - const unsigned alphabet_size, - const double mean_len = 5) : - lh(prob_t::One()), - v0(-std::log(vocab_size)), - u0(-std::log(alphabet_size)), - mean_length(mean_len) {} - - void ResampleHyperparameters(MT19937*) {} - - inline prob_t operator()(const std::vector& s) const { - prob_t p; - p.logeq(Md::log_poisson(s.size(), mean_length) + s.size() * u0); - //p.logeq(v0); - return p; - } - - inline void Increment(const std::vector& w, MT19937*) { - lh *= (*this)(w); - } - - inline void Decrement(const std::vector& w, MT19937 *) { - lh /= (*this)(w); - } - - inline prob_t Likelihood() const { return lh; } - - void Summary() const {} - - private: - - prob_t lh; // keeps track of the draws from the base distribution - const double v0; // uniform log prob of generating a word - const double u0; // uniform log prob of generating a letter - const double mean_length; // mean length of a word in the base distribution -}; - -#endif diff --git a/gi/pf/pyp_lm.cc b/gi/pf/pyp_lm.cc deleted file mode 100644 index 605d8206..00000000 --- a/gi/pf/pyp_lm.cc +++ /dev/null @@ -1,273 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -#include "gamma_poisson.h" -#include "corpus_tools.h" -#include "m.h" -#include "tdict.h" -#include "sampler.h" -#include "ccrp.h" -#include "tied_resampler.h" - -// A not very memory-efficient implementation of an N-gram LM based on PYPs -// as described in Y.-W. Teh. (2006) A Hierarchical Bayesian Language Model -// based on Pitman-Yor Processes. In Proc. ACL. - -// I use templates to handle the recursive formalation of the prior, so -// the order of the model has to be specified here, at compile time: -#define kORDER 3 - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,n",po::value()->default_value(300),"Number of samples") - ("train,i",po::value(),"Training data file") - ("test,T",po::value(),"Test data file") - ("discount_prior_a,a",po::value()->default_value(1.0), "discount ~ Beta(a,b): a=this") - ("discount_prior_b,b",po::value()->default_value(1.0), "discount ~ Beta(a,b): b=this") - ("strength_prior_s,s",po::value()->default_value(1.0), "strength ~ Gamma(s,r): s=this") - ("strength_prior_r,r",po::value()->default_value(1.0), "strength ~ Gamma(s,r): r=this") - ("random_seed,S",po::value(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("train") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -// uniform distribution over a fixed vocabulary -struct UniformVocabulary { - UniformVocabulary(unsigned vs, double, double, double, double) : p0(1.0 / vs), draws() {} - void increment(WordID, const vector&, MT19937*) { ++draws; } - void decrement(WordID, const vector&, MT19937*) { --draws; assert(draws >= 0); } - double prob(WordID, const vector&) const { return p0; } - void resample_hyperparameters(MT19937*) {} - double log_likelihood() const { return draws * log(p0); } - const double p0; - int draws; -}; - -// Lord Rothschild. 1986. THE DISTRIBUTION OF ENGLISH DICTIONARY WORD LENGTHS. -// Journal of Statistical Planning and Inference 14 (1986) 311-322 -struct PoissonLengthUniformCharWordModel { - explicit PoissonLengthUniformCharWordModel(unsigned vocab_size, double, double, double, double) : plen(5,5), uc(-log(95)), llh() {} - void increment(WordID w, const vector& v, MT19937*) { - llh += log(prob(w, v)); // this isn't quite right - plen.increment(TD::Convert(w).size() - 1); - } - void decrement(WordID w, const vector& v, MT19937*) { - plen.decrement(TD::Convert(w).size() - 1); - llh -= log(prob(w, v)); // this isn't quite right - } - double prob(WordID w, const vector&) const { - const unsigned len = TD::Convert(w).size(); - return plen.prob(len - 1) * exp(uc * len); - } - double log_likelihood() const { return llh; } - void resample_hyperparameters(MT19937*) {} - GammaPoisson plen; - const double uc; - double llh; -}; - -struct PYPAdaptedPoissonLengthUniformCharWordModel { - explicit PYPAdaptedPoissonLengthUniformCharWordModel(unsigned vocab_size, double, double, double, double) : - base(vocab_size,1,1,1,1), - crp(1,1,1,1) {} - void increment(WordID w, const vector& v, MT19937* rng) { - double p0 = base.prob(w, v); - if (crp.increment(w, p0, rng)) - base.increment(w, v, rng); - } - void decrement(WordID w, const vector& v, MT19937* rng) { - if (crp.decrement(w, rng)) - base.decrement(w, v, rng); - } - double prob(WordID w, const vector& v) const { - double p0 = base.prob(w, v); - return crp.prob(w, p0); - } - double log_likelihood() const { return crp.log_crp_prob() + base.log_likelihood(); } - void resample_hyperparameters(MT19937* rng) { crp.resample_hyperparameters(rng); } - PoissonLengthUniformCharWordModel base; - CCRP crp; -}; - -template struct PYPLM; - -#if 1 -template<> struct PYPLM<0> : public UniformVocabulary { - PYPLM(unsigned vs, double a, double b, double c, double d) : - UniformVocabulary(vs, a, b, c, d) {} -}; -#else -#if 0 -template<> struct PYPLM<0> : public PoissonLengthUniformCharWordModel { - PYPLM(unsigned vs, double a, double b, double c, double d) : - PoissonLengthUniformCharWordModel(vs, a, b, c, d) {} -}; -#else -template<> struct PYPLM<0> : public PYPAdaptedPoissonLengthUniformCharWordModel { - PYPLM(unsigned vs, double a, double b, double c, double d) : - PYPAdaptedPoissonLengthUniformCharWordModel(vs, a, b, c, d) {} -}; -#endif -#endif - -// represents an N-gram LM -template struct PYPLM { - PYPLM(unsigned vs, double da, double db, double ss, double sr) : - backoff(vs, da, db, ss, sr), - tr(da, db, ss, sr, 0.8, 1.0), - lookup(N-1) {} - void increment(WordID w, const vector& context, MT19937* rng) { - const double bo = backoff.prob(w, context); - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map, CCRP, boost::hash > >::iterator it = p.find(lookup); - if (it == p.end()) { - it = p.insert(make_pair(lookup, CCRP(0.5,1))).first; - tr.Add(&it->second); // add to resampler - } - if (it->second.increment(w, bo, rng)) - backoff.increment(w, context, rng); - } - void decrement(WordID w, const vector& context, MT19937* rng) { - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map, CCRP, boost::hash > >::iterator it = p.find(lookup); - assert(it != p.end()); - if (it->second.decrement(w, rng)) - backoff.decrement(w, context, rng); - } - double prob(WordID w, const vector& context) const { - const double bo = backoff.prob(w, context); - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map, CCRP, boost::hash > >::const_iterator it = p.find(lookup); - if (it == p.end()) return bo; - return it->second.prob(w, bo); - } - - double log_likelihood() const { - double llh = backoff.log_likelihood(); - typename unordered_map, CCRP, boost::hash > >::const_iterator it; - for (it = p.begin(); it != p.end(); ++it) - llh += it->second.log_crp_prob(); - llh += tr.LogLikelihood(); - return llh; - } - - void resample_hyperparameters(MT19937* rng) { - tr.ResampleHyperparameters(rng); - backoff.resample_hyperparameters(rng); - } - - PYPLM backoff; - TiedResampler > tr; - double discount_a, discount_b, strength_s, strength_r; - double d, strength; - mutable vector lookup; // thread-local - unordered_map, CCRP, boost::hash > > p; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - const unsigned samples = conf["samples"].as(); - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector > corpuse; - set vocabe; - const WordID kEOS = TD::Convert(""); - cerr << "Reading corpus...\n"; - CorpusTools::ReadFromFile(conf["train"].as(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - vector > test; - if (conf.count("test")) - CorpusTools::ReadFromFile(conf["test"].as(), &test); - else - test = corpuse; - PYPLM lm(vocabe.size(), - conf["discount_prior_a"].as(), - conf["discount_prior_b"].as(), - conf["strength_prior_s"].as(), - conf["strength_prior_r"].as()); - vector ctx(kORDER - 1, TD::Convert("")); - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpuse.size(); ++ci) { - ctx.resize(kORDER - 1); - const vector& s = corpuse[ci]; - for (int i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - if (SS > 0) lm.decrement(w, ctx, &rng); - lm.increment(w, ctx, &rng); - ctx.push_back(w); - } - } - if (SS % 10 == 9) { - cerr << " [LLH=" << lm.log_likelihood() << "]" << endl; - if (SS % 30 == 29) lm.resample_hyperparameters(&rng); - } else { cerr << '.' << flush; } - } - double llh = 0; - unsigned cnt = 0; - unsigned oovs = 0; - for (int ci = 0; ci < test.size(); ++ci) { - ctx.resize(kORDER - 1); - const vector& s = test[ci]; - for (int i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - double lp = log(lm.prob(w, ctx)) / log(2); - if (i < s.size() && vocabe.count(w) == 0) { - cerr << "**OOV "; - ++oovs; - lp = 0; - } - cerr << "p(" << TD::Convert(w) << " |"; - for (int j = ctx.size() + 1 - kORDER; j < ctx.size(); ++j) - cerr << ' ' << TD::Convert(ctx[j]); - cerr << ") = " << lp << endl; - ctx.push_back(w); - llh -= lp; - cnt++; - } - } - cerr << " Log_10 prob: " << (-llh * log(2) / log(10)) << endl; - cerr << " Count: " << cnt << endl; - cerr << " OOVs: " << oovs << endl; - cerr << "Cross-entropy: " << (llh / cnt) << endl; - cerr << " Perplexity: " << pow(2, llh / cnt) << endl; - return 0; -} - - diff --git a/gi/pf/pyp_tm.cc b/gi/pf/pyp_tm.cc deleted file mode 100644 index 37b9a604..00000000 --- a/gi/pf/pyp_tm.cc +++ /dev/null @@ -1,128 +0,0 @@ -#include "pyp_tm.h" - -#include -#include -#include - -#include "tdict.h" -#include "ccrp.h" -#include "pyp_word_model.h" -#include "tied_resampler.h" - -using namespace std; -using namespace std::tr1; - -struct FreqBinner { - FreqBinner(const std::string& fname) { fd_.Load(fname); } - unsigned NumberOfBins() const { return fd_.Max() + 1; } - unsigned Bin(const WordID& w) const { return fd_.LookUp(w); } - FreqDict fd_; -}; - -template -struct ConditionalPYPWordModel { - ConditionalPYPWordModel(Base* b, const Binner* bnr = NULL) : - base(*b), - binner(bnr), - btr(binner ? binner->NumberOfBins() + 1u : 2u) {} - - void Summary() const { - cerr << "Number of conditioning contexts: " << r.size() << endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - cerr << TD::Convert(it->first) << " \tPYP(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << endl; - for (CCRP >::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - cerr << " " << i2->second << '\t' << TD::GetString(i2->first) << endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - btr.ResampleHyperparameters(rng); - } - - prob_t Prob(const WordID src, const vector& trglets) const { - RuleModelHash::const_iterator it = r.find(src); - if (it == r.end()) { - return base(trglets); - } else { - return it->second.prob(trglets, base(trglets)); - } - } - - void Increment(const WordID src, const vector& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - if (it == r.end()) { - it = r.insert(make_pair(src, CCRP >(0.5,1.0))).first; - static const WordID kNULL = TD::Convert("NULL"); - unsigned bin = (src == kNULL ? 0 : 1); - if (binner && bin) { bin = binner->Bin(src) + 1; } - btr.Add(bin, &it->second); - } - if (it->second.increment(trglets, base(trglets), rng)) - base.Increment(trglets, rng); - } - - void Decrement(const WordID src, const vector& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - assert(it != r.end()); - if (it->second.decrement(trglets, rng)) { - base.Decrement(trglets, rng); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - return p; - } - - unsigned UniqueConditioningContexts() const { - return r.size(); - } - - // TODO tie PYP hyperparameters based on source word frequency bins - Base& base; - const Binner* binner; - BinTiedResampler > > btr; - typedef unordered_map > > RuleModelHash; - RuleModelHash r; -}; - -PYPLexicalTranslation::PYPLexicalTranslation(const vector >& lets, - const unsigned vocab_size, - const unsigned num_letters) : - letters(lets), - base(vocab_size, num_letters, 5), - tmodel(new ConditionalPYPWordModel(&base, new FreqBinner("10k.freq"))), - kX(-TD::Convert("X")) {} - -void PYPLexicalTranslation::Summary() const { - tmodel->Summary(); -} - -prob_t PYPLexicalTranslation::Likelihood() const { - return tmodel->Likelihood() * base.Likelihood(); -} - -void PYPLexicalTranslation::ResampleHyperparameters(MT19937* rng) { - tmodel->ResampleHyperparameters(rng); -} - -unsigned PYPLexicalTranslation::UniqueConditioningContexts() const { - return tmodel->UniqueConditioningContexts(); -} - -prob_t PYPLexicalTranslation::Prob(WordID src, WordID trg) const { - return tmodel->Prob(src, letters[trg]); -} - -void PYPLexicalTranslation::Increment(WordID src, WordID trg, MT19937* rng) { - tmodel->Increment(src, letters[trg], rng); -} - -void PYPLexicalTranslation::Decrement(WordID src, WordID trg, MT19937* rng) { - tmodel->Decrement(src, letters[trg], rng); -} - diff --git a/gi/pf/pyp_tm.h b/gi/pf/pyp_tm.h deleted file mode 100644 index 2b076a25..00000000 --- a/gi/pf/pyp_tm.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef PYP_LEX_TRANS -#define PYP_LEX_TRANS - -#include -#include "wordid.h" -#include "prob.h" -#include "sampler.h" -#include "freqdict.h" -#include "poisson_uniform_word_model.h" - -struct FreqBinner; -template struct ConditionalPYPWordModel; - -struct PYPLexicalTranslation { - explicit PYPLexicalTranslation(const std::vector >& lets, - const unsigned vocab_size, - const unsigned num_letters); - - prob_t Likelihood() const; - - void ResampleHyperparameters(MT19937* rng); - prob_t Prob(WordID src, WordID trg) const; // return p(trg | src) - void Summary() const; - void Increment(WordID src, WordID trg, MT19937* rng); - void Decrement(WordID src, WordID trg, MT19937* rng); - unsigned UniqueConditioningContexts() const; - - private: - const std::vector >& letters; // spelling dictionary - PoissonUniformWordModel base; // "generator" of English types - ConditionalPYPWordModel* tmodel; // translation distributions - // (model English word | French word) - const WordID kX; -}; - -#endif diff --git a/gi/pf/pyp_word_model.h b/gi/pf/pyp_word_model.h deleted file mode 100644 index 0bebb751..00000000 --- a/gi/pf/pyp_word_model.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef _PYP_WORD_MODEL_H_ -#define _PYP_WORD_MODEL_H_ - -#include -#include -#include -#include "prob.h" -#include "ccrp.h" -#include "m.h" -#include "tdict.h" -#include "os_phrase.h" - -// PYP(d,s,poisson-uniform) represented as a CRP -template -struct PYPWordModel { - explicit PYPWordModel(Base* b) : - base(*b), - r(1,1,1,1,0.66,50.0) - {} - - void ResampleHyperparameters(MT19937* rng) { - r.resample_hyperparameters(rng); - std::cerr << " PYPWordModel(d=" << r.discount() << ",s=" << r.strength() << ")\n"; - } - - inline prob_t operator()(const std::vector& s) const { - return r.prob(s, base(s)); - } - - inline void Increment(const std::vector& s, MT19937* rng) { - if (r.increment(s, base(s), rng)) - base.Increment(s, rng); - } - - inline void Decrement(const std::vector& s, MT19937 *rng) { - if (r.decrement(s, rng)) - base.Decrement(s, rng); - } - - inline prob_t Likelihood() const { - prob_t p; p.logeq(r.log_crp_prob()); - p *= base.Likelihood(); - return p; - } - - void Summary() const { - std::cerr << "PYPWordModel: generations=" << r.num_customers() - << " PYP(d=" << r.discount() << ",s=" << r.strength() << ')' << std::endl; - for (typename CCRP >::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << " " << it->second - << TD::GetString(it->first) << std::endl; - } - } - - private: - - Base& base; // keeps track of the draws from the base distribution - CCRP > r; -}; - -#endif diff --git a/gi/pf/quasi_model2.h b/gi/pf/quasi_model2.h deleted file mode 100644 index 4075affe..00000000 --- a/gi/pf/quasi_model2.h +++ /dev/null @@ -1,177 +0,0 @@ -#ifndef _QUASI_MODEL2_H_ -#define _QUASI_MODEL2_H_ - -#include -#include -#include -#include "boost/functional.hpp" -#include "prob.h" -#include "array2d.h" -#include "slice_sampler.h" -#include "m.h" -#include "have_64_bits.h" - -struct AlignmentObservation { - AlignmentObservation() : src_len(), trg_len(), j(), a_j() {} - AlignmentObservation(unsigned sl, unsigned tl, unsigned tw, unsigned sw) : - src_len(sl), trg_len(tl), j(tw), a_j(sw) {} - unsigned short src_len; - unsigned short trg_len; - unsigned short j; - unsigned short a_j; -}; - -#ifdef HAVE_64_BITS -inline size_t hash_value(const AlignmentObservation& o) { - return reinterpret_cast(o); -} -inline bool operator==(const AlignmentObservation& a, const AlignmentObservation& b) { - return hash_value(a) == hash_value(b); -} -#else -inline size_t hash_value(const AlignmentObservation& o) { - size_t h = 1; - boost::hash_combine(h, o.src_len); - boost::hash_combine(h, o.trg_len); - boost::hash_combine(h, o.j); - boost::hash_combine(h, o.a_j); - return h; -} -#endif - -struct QuasiModel2 { - explicit QuasiModel2(double alpha, double pnull = 0.1) : - alpha_(alpha), - pnull_(pnull), - pnotnull_(1 - pnull) {} - - // a_j = 0 => NULL; src_len does *not* include null - prob_t Prob(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) const { - if (!a_j) return pnull_; - return pnotnull_ * - prob_t(UnnormalizedProb(a_j, j, src_len, trg_len, alpha_) / GetOrComputeZ(j, src_len, trg_len)); - } - - void Increment(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) { - assert(a_j <= src_len); - assert(j < trg_len); - ++obs_[AlignmentObservation(src_len, trg_len, j, a_j)]; - } - - void Decrement(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) { - const AlignmentObservation ao(src_len, trg_len, j, a_j); - int &cc = obs_[ao]; - assert(cc > 0); - --cc; - if (!cc) obs_.erase(ao); - } - - struct PNullResampler { - PNullResampler(const QuasiModel2& m) : m_(m) {} - const QuasiModel2& m_; - double operator()(const double& proposed_pnull) const { - return log(m_.Likelihood(m_.alpha_, proposed_pnull)); - } - }; - - struct AlphaResampler { - AlphaResampler(const QuasiModel2& m) : m_(m) {} - const QuasiModel2& m_; - double operator()(const double& proposed_alpha) const { - return log(m_.Likelihood(proposed_alpha, m_.pnull_.as_float())); - } - }; - - void ResampleHyperparameters(MT19937* rng, const unsigned nloop = 5, const unsigned niterations = 10) { - const PNullResampler dr(*this); - const AlphaResampler ar(*this); - for (unsigned i = 0; i < nloop; ++i) { - double pnull = slice_sampler1d(dr, pnull_.as_float(), *rng, 0.00000001, - 1.0, 0.0, niterations, 100*niterations); - pnull_ = prob_t(pnull); - alpha_ = slice_sampler1d(ar, alpha_, *rng, 0.00000001, - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - } - std::cerr << "QuasiModel2(alpha=" << alpha_ << ",p_null=" - << pnull_.as_float() << ") = " << Likelihood() << std::endl; - zcache_.clear(); - } - - prob_t Likelihood() const { - return Likelihood(alpha_, pnull_.as_float()); - } - - prob_t Likelihood(double alpha, double ppnull) const { - const prob_t pnull(ppnull); - const prob_t pnotnull(1 - ppnull); - - prob_t p; - p.logeq(Md::log_gamma_density(alpha, 0.1, 25)); // TODO configure - assert(!p.is_0()); - prob_t prob_of_ppnull; prob_of_ppnull.logeq(Md::log_beta_density(ppnull, 2, 10)); - assert(!prob_of_ppnull.is_0()); - p *= prob_of_ppnull; - for (ObsCount::const_iterator it = obs_.begin(); it != obs_.end(); ++it) { - const AlignmentObservation& ao = it->first; - if (ao.a_j) { - prob_t u = XUnnormalizedProb(ao.a_j, ao.j, ao.src_len, ao.trg_len, alpha); - prob_t z = XComputeZ(ao.j, ao.src_len, ao.trg_len, alpha); - prob_t pa(u / z); - pa *= pnotnull; - pa.poweq(it->second); - p *= pa; - } else { - p *= pnull.pow(it->second); - } - } - return p; - } - - private: - static prob_t XUnnormalizedProb(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - prob_t p; - p.logeq(-fabs(double(a_j - 1) / src_len - double(j) / trg_len) * alpha); - return p; - } - - static prob_t XComputeZ(unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - prob_t z = prob_t::Zero(); - for (int a_j = 1; a_j <= src_len; ++a_j) - z += XUnnormalizedProb(a_j, j, src_len, trg_len, alpha); - return z; - } - - static double UnnormalizedProb(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - return exp(-fabs(double(a_j - 1) / src_len - double(j) / trg_len) * alpha); - } - - static double ComputeZ(unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - double z = 0; - for (int a_j = 1; a_j <= src_len; ++a_j) - z += UnnormalizedProb(a_j, j, src_len, trg_len, alpha); - return z; - } - - const double& GetOrComputeZ(unsigned j, unsigned src_len, unsigned trg_len) const { - if (src_len >= zcache_.size()) - zcache_.resize(src_len + 1); - if (trg_len >= zcache_[src_len].size()) - zcache_[src_len].resize(trg_len + 1); - std::vector& zv = zcache_[src_len][trg_len]; - if (zv.size() == 0) - zv.resize(trg_len); - double& z = zv[j]; - if (!z) - z = ComputeZ(j, src_len, trg_len, alpha_); - return z; - } - - double alpha_; - prob_t pnull_; - prob_t pnotnull_; - mutable std::vector > > zcache_; - typedef std::tr1::unordered_map > ObsCount; - ObsCount obs_; -}; - -#endif diff --git a/gi/pf/reachability.cc b/gi/pf/reachability.cc deleted file mode 100644 index 7d0d04ac..00000000 --- a/gi/pf/reachability.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include "reachability.h" - -#include -#include - -using namespace std; - -struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; -}; - -void Reachability::ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - //cerr << srclen << "," << trglen << ": Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - if (a[srclen][trglen].empty()) { - cerr << "Sequence pair with lengths (" << srclen << ',' << trglen << ") violates reachability constraints\n"; - nodes = 0; - return; - } - - typedef boost::multi_array rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - nodes = 0; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - valid_deltas[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(make_pair(src_delta,j - prevs[k].prev_trg_covered)); - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - assert(max_src_delta[0][0] > 0); - nodes = 0; - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (valid_deltas[i][j].size() > 0) { - node_addresses[i][j] = nodes++; - } else { - node_addresses[i][j] = -1; - } - } - } - cerr << "Sequence pair with lengths (" << srclen << ',' << trglen << ") has " << valid_deltas[0][0].size() << " out edges in its root node, " << nodes << " nodes in total, and outside estimate matrix will require " << sizeof(float)*nodes << " bytes\n"; - } - diff --git a/gi/pf/reachability.h b/gi/pf/reachability.h deleted file mode 100644 index 1e22c76a..00000000 --- a/gi/pf/reachability.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef _REACHABILITY_H_ -#define _REACHABILITY_H_ - -#include "boost/multi_array.hpp" - -// determines minimum and maximum lengths of outgoing edges from all -// coverage positions such that the alignment path respects src and -// trg maximum phrase sizes -// -// runs in O(n^2 * src_max * trg_max) time but should be relatively fast -// -// currently forbids 0 -> n and n -> 0 alignments - -struct Reachability { - unsigned nodes; - boost::multi_array edges; // edges[src_covered][trg_covered][src_delta][trg_delta] is this edge worth exploring? - boost::multi_array max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - boost::multi_array node_addresses; // na[src_covered][trg_covered] -- the index of the node in a one-dimensional array (of size "nodes") - boost::multi_array >, 2> valid_deltas; // valid_deltas[src_covered][trg_covered] list of valid transitions leaving a particular node - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - nodes(), - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]), - node_addresses(boost::extents[srclen][trglen]), - valid_deltas(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len); -}; - -#endif diff --git a/gi/pf/tied_resampler.h b/gi/pf/tied_resampler.h deleted file mode 100644 index a4f4af36..00000000 --- a/gi/pf/tied_resampler.h +++ /dev/null @@ -1,122 +0,0 @@ -#ifndef _TIED_RESAMPLER_H_ -#define _TIED_RESAMPLER_H_ - -#include -#include -#include "sampler.h" -#include "slice_sampler.h" -#include "m.h" - -template -struct TiedResampler { - explicit TiedResampler(double da, double db, double ss, double sr, double d=0.5, double s=1.0) : - d_alpha(da), - d_beta(db), - s_shape(ss), - s_rate(sr), - discount(d), - strength(s) {} - - void Add(CRP* crp) { - crps.insert(crp); - crp->set_discount(discount); - crp->set_strength(strength); - assert(!crp->has_discount_prior()); - assert(!crp->has_strength_prior()); - } - - void Remove(CRP* crp) { - crps.erase(crp); - } - - size_t size() const { - return crps.size(); - } - - double LogLikelihood(double d, double s) const { - if (s <= -d) return -std::numeric_limits::infinity(); - double llh = Md::log_beta_density(d, d_alpha, d_beta) + - Md::log_gamma_density(d + s, s_shape, s_rate); - for (typename std::set::iterator it = crps.begin(); it != crps.end(); ++it) - llh += (*it)->log_crp_prob(d, s); - return llh; - } - - double LogLikelihood() const { - return LogLikelihood(discount, strength); - } - - struct DiscountResampler { - DiscountResampler(const TiedResampler& m) : m_(m) {} - const TiedResampler& m_; - double operator()(const double& proposed_discount) const { - return m_.LogLikelihood(proposed_discount, m_.strength); - } - }; - - struct AlphaResampler { - AlphaResampler(const TiedResampler& m) : m_(m) {} - const TiedResampler& m_; - double operator()(const double& proposed_strength) const { - return m_.LogLikelihood(m_.discount, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng, const unsigned nloop = 5, const unsigned niterations = 10) { - if (size() == 0) { std::cerr << "EMPTY - not resampling\n"; return; } - const DiscountResampler dr(*this); - const AlphaResampler ar(*this); - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -discount + std::numeric_limits::min(), - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits::min(); - if (strength < 0.0) min_discount -= strength; - discount = slice_sampler1d(dr, discount, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } - strength = slice_sampler1d(ar, strength, *rng, -discount + std::numeric_limits::min(), - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "TiedCRPs(d=" << discount << ",s=" - << strength << ") = " << LogLikelihood(discount, strength) << std::endl; - for (typename std::set::iterator it = crps.begin(); it != crps.end(); ++it) - (*it)->set_hyperparameters(discount, strength); - } - private: - std::set crps; - const double d_alpha, d_beta, s_shape, s_rate; - double discount, strength; -}; - -// split according to some criterion -template -struct BinTiedResampler { - explicit BinTiedResampler(unsigned nbins) : - resamplers(nbins, TiedResampler(1,1,1,1)) {} - - void Add(unsigned bin, CRP* crp) { - resamplers[bin].Add(crp); - } - - void Remove(unsigned bin, CRP* crp) { - resamplers[bin].Remove(crp); - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < resamplers.size(); ++i) { - std::cerr << "BIN " << i << " (" << resamplers[i].size() << " CRPs): " << std::flush; - resamplers[i].ResampleHyperparameters(rng); - } - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < resamplers.size(); ++i) - llh += resamplers[i].LogLikelihood(); - return llh; - } - - private: - std::vector > resamplers; -}; - -#endif diff --git a/gi/pf/tpf.cc b/gi/pf/tpf.cc deleted file mode 100644 index 7348d21c..00000000 --- a/gi/pf/tpf.cc +++ /dev/null @@ -1,99 +0,0 @@ -#include -#include -#include - -#include "sampler.h" - -using namespace std; -using namespace tr1; - -shared_ptr prng; - -struct Particle { - Particle() : weight(prob_t::One()) {} - vector states; - prob_t weight; - prob_t gamma_last; -}; - -ostream& operator<<(ostream& os, const Particle& p) { - os << "["; - for (int i = 0; i < p.states.size(); ++i) os << p.states[i] << ' '; - os << "| w=" << log(p.weight) << ']'; - return os; -} - -void Rejuvenate(vector& pps) { - SampleSet ss; - vector nps(pps.size()); - for (int i = 0; i < pps.size(); ++i) { -// cerr << pps[i] << endl; - ss.add(pps[i].weight); - } -// cerr << "REJUVINATING...\n"; - for (int i = 0; i < pps.size(); ++i) { - nps[i] = pps[prng->SelectSample(ss)]; - nps[i].weight = prob_t(1.0 / pps.size()); -// cerr << nps[i] << endl; - } - nps.swap(pps); -// exit(1); -} - -int main(int argc, char** argv) { - const unsigned particles = 100; - prng.reset(new MT19937); - MT19937& rng = *prng; - - // q(a) = 0.8 - // q(b) = 0.8 - // q(c) = 0.4 - SampleSet ssq; - ssq.add(0.4); - ssq.add(0.6); - ssq.add(0); - double qz = 1; - - // p(a) = 0.2 - // p(b) = 0.8 - vector p(3); - p[0] = 0.2; - p[1] = 0.8; - p[2] = 0; - - vector counts(3); - int tot = 0; - - vector pps(particles); - SampleSet ppss; - int LEN = 12; - int PP = 1; - while (pps[0].states.size() < LEN) { - for (int pi = 0; pi < particles; ++pi) { - Particle& prt = pps[pi]; - - bool redo = true; - const Particle savedp = prt; - while (redo) { - redo = false; - for (int i = 0; i < PP; ++i) { - int s = rng.SelectSample(ssq); - double gamma_last = p[s]; - if (!gamma_last) { redo = true; break; } - double q = ssq[s] / qz; - prt.states.push_back(s); - prt.weight *= prob_t(gamma_last / q); - } - if (redo) { prt = savedp; continue; } - } - } - Rejuvenate(pps); - } - ppss.clear(); - for (int i = 0; i < particles; ++i) { ppss.add(pps[i].weight); } - int sp = rng.SelectSample(ppss); - cerr << pps[sp] << endl; - - return 0; -} - diff --git a/gi/pf/transliterations.cc b/gi/pf/transliterations.cc deleted file mode 100644 index b2996f65..00000000 --- a/gi/pf/transliterations.cc +++ /dev/null @@ -1,334 +0,0 @@ -#include "transliterations.h" - -#include -#include - -#include "boost/shared_ptr.hpp" - -#include "backward.h" -#include "filelib.h" -#include "tdict.h" -#include "trule.h" -#include "filelib.h" -#include "ccrp_nt.h" -#include "m.h" -#include "reachability.h" - -using namespace std; -using namespace std::tr1; - -struct TruncatedConditionalLengthModel { - TruncatedConditionalLengthModel(unsigned max_src_size, unsigned max_trg_size, double expected_src_to_trg_ratio) : - plens(max_src_size+1, vector(max_trg_size+1, 0.0)) { - for (unsigned i = 1; i <= max_src_size; ++i) { - prob_t z = prob_t::Zero(); - for (unsigned j = 1; j <= max_trg_size; ++j) - z += (plens[i][j] = prob_t(0.01 + exp(Md::log_poisson(j, i * expected_src_to_trg_ratio)))); - for (unsigned j = 1; j <= max_trg_size; ++j) - plens[i][j] /= z; - //for (unsigned j = 1; j <= max_trg_size; ++j) - // cerr << "P(trg_len=" << j << " | src_len=" << i << ") = " << plens[i][j] << endl; - } - } - - // return p(tlen | slen) for *chunks* not full words - inline const prob_t& operator()(int slen, int tlen) const { - return plens[slen][tlen]; - } - - vector > plens; -}; - -struct CondBaseDist { - CondBaseDist(unsigned max_src_size, unsigned max_trg_size, double expected_src_to_trg_ratio) : - tclm(max_src_size, max_trg_size, expected_src_to_trg_ratio) {} - - prob_t operator()(const vector& src, unsigned sf, unsigned st, - const vector& trg, unsigned tf, unsigned tt) const { - prob_t p = tclm(st - sf, tt - tf); // target len | source length ~ TCLM(source len) - assert(!"not impl"); - return p; - } - inline prob_t operator()(const vector& src, const vector& trg) const { - return (*this)(src, 0, src.size(), trg, 0, trg.size()); - } - TruncatedConditionalLengthModel tclm; -}; - -// represents transliteration phrase probabilities, e.g. -// p( a l - | A l ) , p( o | A w ) , ... -struct TransliterationChunkConditionalModel { - explicit TransliterationChunkConditionalModel(const CondBaseDist& pp0) : - d(0.0), - strength(1.0), - rp0(pp0) { - } - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(\\alpha = " << it->second.alpha() << ") --------------------------" << std::endl; - for (CCRP_NoTable::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second << '\t' << i2->first << std::endl; - } - } - - int DecrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - int count = it->second.decrement(rule); - if (count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return count; - } - - int IncrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - it = r.insert(make_pair(rule.f_, CCRP_NoTable(strength))).first; - } - int count = it->second.increment(rule); - return count; - } - - void IncrementRules(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const std::vector& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p = rp0(rule.f_, rule.e_); - } else { - p = it->second.prob(rule, rp0(rule.f_, rule.e_)); - } - return p; - } - - double LogLikelihood(const double& dd, const double& aa) const { - if (aa <= -dd) return -std::numeric_limits::infinity(); - //double llh = Md::log_beta_density(dd, 10, 3) + Md::log_gamma_density(aa, 1, 1); - double llh = //Md::log_beta_density(dd, 1, 1) + - Md::log_gamma_density(dd + aa, 1, 1); - std::tr1::unordered_map, CCRP_NoTable, boost::hash > >::const_iterator it; - for (it = r.begin(); it != r.end(); ++it) - llh += it->second.log_crp_prob(aa); - return llh; - } - - struct AlphaResampler { - AlphaResampler(const TransliterationChunkConditionalModel& m) : m_(m) {} - const TransliterationChunkConditionalModel& m_; - double operator()(const double& proposed_strength) const { - return m_.LogLikelihood(m_.d, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng) { - std::tr1::unordered_map, CCRP_NoTable, boost::hash > >::iterator it; - //const unsigned nloop = 5; - const unsigned niterations = 10; - //DiscountResampler dr(*this); - AlphaResampler ar(*this); -#if 0 - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -d + std::numeric_limits::min(), - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits::min(); - if (strength < 0.0) min_discount -= strength; - d = slice_sampler1d(dr, d, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } -#endif - strength = slice_sampler1d(ar, strength, *rng, -d, - std::numeric_limits::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "CTMModel(alpha=" << strength << ") = " << LogLikelihood(d, strength) << std::endl; - for (it = r.begin(); it != r.end(); ++it) { -#if 0 - it->second.set_discount(d); -#endif - it->second.set_alpha(strength); - } - } - - prob_t Likelihood() const { - prob_t p; p.logeq(LogLikelihood(d, strength)); - return p; - } - - const CondBaseDist& rp0; - typedef std::tr1::unordered_map, - CCRP_NoTable, - boost::hash > > RuleModelHash; - RuleModelHash r; - double d, strength; -}; - -struct GraphStructure { - GraphStructure() : r() {} - // leak memory - these are basically static - const Reachability* r; - bool IsReachable() const { return r->nodes > 0; } -}; - -struct ProbabilityEstimates { - ProbabilityEstimates() : gs(), backward() {} - explicit ProbabilityEstimates(const GraphStructure& g) : - gs(&g), backward() { - if (g.r->nodes > 0) - backward = new float[g.r->nodes]; - } - // leak memory, these are static - - // returns an estimate of the marginal probability - double MarginalEstimate() const { - if (!backward) return 0; - return backward[0]; - } - - // returns an backward estimate - double Backward(int src_covered, int trg_covered) const { - if (!backward) return 0; - int ind = gs->r->node_addresses[src_covered][trg_covered]; - if (ind < 0) return 0; - return backward[ind]; - } - - prob_t estp; - float* backward; - private: - const GraphStructure* gs; -}; - -struct TransliterationsImpl { - TransliterationsImpl(int max_src, int max_trg, double sr, const BackwardEstimator& b) : - cp0(max_src, max_trg, sr), - tccm(cp0), - be(b), - kMAX_SRC_CHUNK(max_src), - kMAX_TRG_CHUNK(max_trg), - kS2T_RATIO(sr), - tot_pairs(), tot_mem() { - } - const CondBaseDist cp0; - TransliterationChunkConditionalModel tccm; - const BackwardEstimator& be; - - void Initialize(WordID src, const vector& src_lets, WordID trg, const vector& trg_lets) { - const size_t src_len = src_lets.size(); - const size_t trg_len = trg_lets.size(); - - // init graph structure - if (src_len >= graphs.size()) graphs.resize(src_len + 1); - if (trg_len >= graphs[src_len].size()) graphs[src_len].resize(trg_len + 1); - GraphStructure& gs = graphs[src_len][trg_len]; - if (!gs.r) { - double rat = exp(fabs(log(trg_len / (src_len * kS2T_RATIO)))); - if (rat > 1.5 || (rat > 2.4 && src_len < 6)) { - cerr << " ** Forbidding transliterations of size " << src_len << "," << trg_len << ": " << rat << endl; - gs.r = new Reachability(src_len, trg_len, 0, 0); - } else { - gs.r = new Reachability(src_len, trg_len, kMAX_SRC_CHUNK, kMAX_TRG_CHUNK); - } - } - - const Reachability& r = *gs.r; - - // init backward estimates - if (src >= ests.size()) ests.resize(src + 1); - unordered_map::iterator it = ests[src].find(trg); - if (it != ests[src].end()) return; // already initialized - - it = ests[src].insert(make_pair(trg, ProbabilityEstimates(gs))).first; - ProbabilityEstimates& est = it->second; - if (!gs.r->nodes) return; // not derivable subject to length constraints - - be.InitializeGrid(src_lets, trg_lets, r, kS2T_RATIO, est.backward); - cerr << TD::GetString(src_lets) << " ||| " << TD::GetString(trg_lets) << " ||| " << (est.backward[0] / trg_lets.size()) << endl; - tot_pairs++; - tot_mem += sizeof(float) * gs.r->nodes; - } - - void Forbid(WordID src, const vector& src_lets, WordID trg, const vector& trg_lets) { - const size_t src_len = src_lets.size(); - const size_t trg_len = trg_lets.size(); - // TODO - } - - prob_t EstimateProbability(WordID s, const vector& src, WordID t, const vector& trg) const { - assert(src.size() < graphs.size()); - const vector& tv = graphs[src.size()]; - assert(trg.size() < tv.size()); - const GraphStructure& gs = tv[trg.size()]; - if (gs.r->nodes == 0) - return prob_t::Zero(); - const unordered_map::const_iterator it = ests[s].find(t); - assert(it != ests[s].end()); - return it->second.estp; - } - - void GraphSummary() const { - double to = 0; - double tn = 0; - double tt = 0; - for (int i = 0; i < graphs.size(); ++i) { - const vector& vt = graphs[i]; - for (int j = 0; j < vt.size(); ++j) { - const GraphStructure& gs = vt[j]; - if (!gs.r) continue; - tt++; - for (int k = 0; k < i; ++k) { - for (int l = 0; l < j; ++l) { - size_t c = gs.r->valid_deltas[k][l].size(); - if (c) { - tn += 1; - to += c; - } - } - } - } - } - cerr << " Average nodes = " << (tn / tt) << endl; - cerr << "Average out-degree = " << (to / tn) << endl; - cerr << " Unique structures = " << tt << endl; - cerr << " Unique pairs = " << tot_pairs << endl; - cerr << " BEs size = " << (tot_mem / (1024.0*1024.0)) << " MB" << endl; - } - - const int kMAX_SRC_CHUNK; - const int kMAX_TRG_CHUNK; - const double kS2T_RATIO; - unsigned tot_pairs; - size_t tot_mem; - vector > graphs; // graphs[src_len][trg_len] - vector > ests; // ests[src][trg] -}; - -Transliterations::Transliterations(int max_src, int max_trg, double sr, const BackwardEstimator& be) : - pimpl_(new TransliterationsImpl(max_src, max_trg, sr, be)) {} -Transliterations::~Transliterations() { delete pimpl_; } - -void Transliterations::Initialize(WordID src, const vector& src_lets, WordID trg, const vector& trg_lets) { - pimpl_->Initialize(src, src_lets, trg, trg_lets); -} - -prob_t Transliterations::EstimateProbability(WordID s, const vector& src, WordID t, const vector& trg) const { - return pimpl_->EstimateProbability(s, src,t, trg); -} - -void Transliterations::Forbid(WordID src, const vector& src_lets, WordID trg, const vector& trg_lets) { - pimpl_->Forbid(src, src_lets, trg, trg_lets); -} - -void Transliterations::GraphSummary() const { - pimpl_->GraphSummary(); -} - diff --git a/gi/pf/transliterations.h b/gi/pf/transliterations.h deleted file mode 100644 index 49d14684..00000000 --- a/gi/pf/transliterations.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef _TRANSLITERATIONS_H_ -#define _TRANSLITERATIONS_H_ - -#include -#include "wordid.h" -#include "prob.h" - -struct BackwardEstimator; -struct TransliterationsImpl; -struct Transliterations { - // max_src and max_trg indicate how big the transliteration phrases can be - // see reachability.h for information about filter_ratio - explicit Transliterations(int max_src, int max_trg, double s2t_rat, const BackwardEstimator& be); - ~Transliterations(); - void Initialize(WordID src, const std::vector& src_lets, WordID trg, const std::vector& trg_lets); - void Forbid(WordID src, const std::vector& src_lets, WordID trg, const std::vector& trg_lets); - void GraphSummary() const; - prob_t EstimateProbability(WordID s, const std::vector& src, WordID t, const std::vector& trg) const; - private: - TransliterationsImpl* pimpl_; -}; - -#endif - diff --git a/gi/pf/unigrams.cc b/gi/pf/unigrams.cc deleted file mode 100644 index 40829775..00000000 --- a/gi/pf/unigrams.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "unigrams.h" - -#include -#include - -#include "stringlib.h" -#include "filelib.h" - -using namespace std; - -void UnigramModel::LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - const WordID w = TD::Convert(line.substr(pos + 1)); - line[pos] = 0; - float p = atof(&line[0]); - if (w < probs_.size()) probs_[w].logeq(p * log(10)); else cerr << "WARNING: don't know about '" << TD::Convert(w) << "'\n"; - } -} - -void UnigramWordModel::LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - size_t cur = pos + 1; - vector w; - while (cur < line.size()) { - const size_t len = UTF8Len(line[cur]); - w.push_back(TD::Convert(line.substr(cur, len))); - cur += len; - } - line[pos] = 0; - float p = atof(&line[0]); - probs_[w].logeq(p * log(10.0)); - } -} - diff --git a/gi/pf/unigrams.h b/gi/pf/unigrams.h deleted file mode 100644 index 1660d1ed..00000000 --- a/gi/pf/unigrams.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef _UNIGRAMS_H_ -#define _UNIGRAMS_H_ - -#include -#include -#include -#include - -#include "wordid.h" -#include "prob.h" -#include "tdict.h" - -struct UnigramModel { - explicit UnigramModel(const std::string& fname, unsigned vocab_size) : - use_uniform_(fname.size() == 0), - uniform_(1.0 / vocab_size), - probs_() { - if (fname.size() > 0) { - probs_.resize(TD::NumWords() + 1); - LoadUnigrams(fname); - } - } - - const prob_t& operator()(const WordID& w) const { - assert(w); - if (use_uniform_) return uniform_; - return probs_[w]; - } - - private: - void LoadUnigrams(const std::string& fname); - - const bool use_uniform_; - const prob_t uniform_; - std::vector probs_; -}; - - -// reads an ARPA unigram file and converts words like 'cat' into a string 'c a t' -struct UnigramWordModel { - explicit UnigramWordModel(const std::string& fname) : - use_uniform_(false), - uniform_(1.0), - probs_() { - LoadUnigrams(fname); - } - - explicit UnigramWordModel(const unsigned vocab_size) : - use_uniform_(true), - uniform_(1.0 / vocab_size), - probs_() {} - - const prob_t& operator()(const std::vector& s) const { - if (use_uniform_) return uniform_; - const VectorProbHash::const_iterator it = probs_.find(s); - assert(it != probs_.end()); - return it->second; - } - - private: - void LoadUnigrams(const std::string& fname); - - const bool use_uniform_; - const prob_t uniform_; - typedef std::tr1::unordered_map, prob_t, boost::hash > > VectorProbHash; - VectorProbHash probs_; -}; - -#endif diff --git a/gi/pipeline/OLD.clsp.config b/gi/pipeline/OLD.clsp.config deleted file mode 100644 index cd0f9d65..00000000 --- a/gi/pipeline/OLD.clsp.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM xfeats.grammar dev dev-refs test1 testt-eval.sh ... -btec /export/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz xgrammar/grammar.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /export/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /export/ws10smt/data/chinese-english corpus.zh-en.al -aren /export/ws10smt/data/arabic-english corpus.ar-en.al -uren /export/ws10smt/data/urdu-english corpus.ur-en.al -nlfr /export/ws10smt/data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/OLD.evaluation-pipeline.pl b/gi/pipeline/OLD.evaluation-pipeline.pl deleted file mode 100755 index 49c303eb..00000000 --- a/gi/pipeline/OLD.evaluation-pipeline.pl +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use Getopt::Long; -use Cwd; -my $CWD = getcwd; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -my @DEFAULT_FEATS = qw( - LogRuleCount SingletonRule LexE2F LexF2E WordPenalty - LogFCount LanguageModel Glue GlueTop PassThrough SingletonF -); - -my %init_weights = qw( - LogRuleCount 0.2 - LexE2F -0.3 - LexF2E -0.3 - LogFCount 0.1 - WordPenalty -1.5 - LanguageModel 1.2 - Glue -1.0 - GlueTop 0.00001 - PassThrough -10.0 - SingletonRule -0.1 - X_EGivenF -0.3 - X_FGivenE -0.3 - X_LogECount -1 - X_LogFCount -0.1 - X_LogRuleCount 0.3 - X_SingletonE -0.1 - X_SingletonF -0.1 - X_SingletonRule -0.5 -); - -my $CDEC = "$SCRIPT_DIR/../../decoder/cdec"; -my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl"; -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $VEST = "$SCRIPT_DIR/../../vest"; -die "Can't find vest: $VEST" unless -e $VEST && -d $VEST; -my $DISTVEST = "$VEST/dist-vest.pl"; -my $FILTSCORE = "$EXTOOLS/filter_score_grammar"; -my $ADDXFEATS = "$SCRIPT_DIR/scripts/xfeats.pl"; -assert_exec($CDEC, $PARALLELIZE, $FILTSCORE, $DISTVEST, $ADDXFEATS); - -my $config = "$SCRIPT_DIR/OLD.clsp.config"; -print STDERR "CORPORA CONFIGURATION: $config\n"; -open CONF, "<$config" or die "Can't read $config: $!"; -my %paths; -my %corpora; -my %lms; -my %devs; -my %devrefs; -my %tests; -my %testevals; -my %xgrammars; -print STDERR " LANGUAGE PAIRS:"; -while() { - chomp; - next if /^#/; - next if /^\s*$/; - s/^\s+//; - s/\s+$//; - my ($name, $path, $corpus, $lm, $xgrammar, $dev, $devref, @xtests) = split /\s+/; - $paths{$name} = $path; - $corpora{$name} = $corpus; - $lms{$name} = $lm; - $xgrammars{$name} = $xgrammar; - $devs{$name} = $dev; - $devrefs{$name} = $devref; - $tests{$name} = $xtests[0]; - $testevals{$name} = $xtests[1]; - print STDERR " $name"; -} -print STDERR "\n"; - -my %langpairs = map { $_ => 1 } qw( btec zhen fbis aren uren nlfr ); - -my $outdir = "$CWD/exp"; -my $help; -my $XFEATS; -my $EXTRA_FILTER = ''; -my $dataDir = '/export/ws10smt/data'; -if (GetOptions( - "data=s" => \$dataDir, - "xfeats" => \$XFEATS, -) == 0 || @ARGV!=2 || $help) { - print_help(); - exit; -} -my $lp = $ARGV[0]; -my $grammar = $ARGV[1]; -print STDERR " CORPUS REPO: $dataDir\n"; -print STDERR " LANGUAGE PAIR: $lp\n"; -die "I don't know about that language pair\n" unless $paths{$lp}; -my $corpdir = "$dataDir"; -if ($paths{$lp} =~ /^\//) { $corpdir = $paths{$lp}; } else { $corpdir .= '/' . $paths{$lp}; } -die "I can't find the corpora directory: $corpdir" unless -d $corpdir; -print STDERR " GRAMMAR: $grammar\n"; -my $LANG_MODEL = mydircat($corpdir, $lms{$lp}); -print STDERR " LM: $LANG_MODEL\n"; -my $CORPUS = mydircat($corpdir, $corpora{$lp}); -die "Can't find corpus: $CORPUS" unless -f $CORPUS; - -my $dev = mydircat($corpdir, $devs{$lp}); -my $drefs = $devrefs{$lp}; -die "Can't find dev: $dev\n" unless -f $dev; -die "Dev refs not set" unless $drefs; -$drefs = mydircat($corpdir, $drefs); - -my $test = mydircat($corpdir, $tests{$lp}); -my $teval = mydircat($corpdir, $testevals{$lp}); -die "Can't find test: $test\n" unless -f $test; -assert_exec($teval); - -if ($XFEATS) { - my $xgram = mydircat($corpdir, $xgrammars{$lp}); - die "Can't find x-grammar: $xgram" unless -f $xgram; - $EXTRA_FILTER = "$ADDXFEATS $xgram |"; - print STDERR "ADDING X-FEATS FROM $xgram\n"; -} - -# MAKE DEV -print STDERR "\nFILTERING FOR dev...\n"; -print STDERR "DEV: $dev (REFS=$drefs)\n"; -`mkdir -p $outdir`; -my $devgrammar = filter($grammar, $dev, 'dev', $outdir); -my $devini = mydircat($outdir, "cdec-dev.ini"); -write_cdec_ini($devini, $devgrammar); - - -# MAKE TEST -print STDERR "\nFILTERING FOR test...\n"; -print STDERR "TEST: $test (EVAL=$teval)\n"; -`mkdir -p $outdir`; -my $testgrammar = filter($grammar, $test, 'test', $outdir); -my $testini = mydircat($outdir, "cdec-test.ini"); -write_cdec_ini($testini, $testgrammar); - - -# CREATE INIT WEIGHTS -print STDERR "\nCREATING INITIAL WEIGHTS FILE: weights.init\n"; -my $weights = mydircat($outdir, "weights.init"); -write_random_weights_file($weights); - - -# VEST -print STDERR "\nMINIMUM ERROR TRAINING\n"; -my $tuned_weights = mydircat($outdir, 'weights.tuned'); -if (-f $tuned_weights) { - print STDERR "TUNED WEIGHTS $tuned_weights EXISTS: REUSING\n"; -} else { - my $cmd = "$DISTVEST --ref-files=$drefs --source-file=$dev --weights $weights $devini"; - print STDERR "MERT COMMAND: $cmd\n"; - `rm -rf $outdir/vest 2> /dev/null`; - chdir $outdir or die "Can't chdir to $outdir: $!"; - $weights = `$cmd`; - die "MERT reported non-zero exit code" unless $? == 0; - chomp $weights; - safesystem($tuned_weights, "cp $weights $tuned_weights"); - print STDERR "TUNED WEIGHTS: $tuned_weights\n"; - die "$tuned_weights is missing!" unless -f $tuned_weights; -} - -# DECODE -print STDERR "\nDECODE TEST SET\n"; -my $decolog = mydircat($outdir, "test-decode.log"); -my $testtrans = mydircat($outdir, "test.trans"); -my $cmd = "cat $test | $PARALLELIZE -j 20 -e $decolog -- $CDEC -c $testini -w $tuned_weights > $testtrans"; -safesystem($testtrans, $cmd) or die "Failed to decode test set!"; - - -# EVALUATE -print STDERR "\nEVALUATE TEST SET\n"; -print STDERR "TEST: $testtrans\n"; -$cmd = "$teval $testtrans"; -safesystem(undef, $cmd) or die "Failed to evaluate!"; -exit 0; - - -sub write_random_weights_file { - my ($file, @extras) = @_; - open F, ">$file" or die "Can't write $file: $!"; - my @feats = (@DEFAULT_FEATS, @extras); - if ($XFEATS) { - my @xfeats = qw( - X_LogRuleCount X_LogECount X_LogFCount X_EGivenF X_FGivenE X_SingletonRule X_SingletonE X_SingletonF - ); - @feats = (@feats, @xfeats); - } - for my $feat (@feats) { - my $r = rand(1.6); - my $w = $init_weights{$feat} * $r; - if ($w == 0) { $w = 0.0001; print STDERR "WARNING: $feat had no initial weight!\n"; } - print F "$feat $w\n"; - } - close F; -} - -sub filter { - my ($grammar, $set, $name, $outdir) = @_; - my $outgrammar = mydircat($outdir, "$name.scfg.gz"); - if (-f $outgrammar) { print STDERR "$outgrammar exists - REUSING!\n"; } else { - my $cmd = "gunzip -c $grammar | $FILTSCORE -c $CORPUS -t $set | $EXTRA_FILTER gzip > $outgrammar"; - safesystem($outgrammar, $cmd) or die "Can't filter and score grammar!"; - } - return $outgrammar; -} - -sub mydircat { - my ($base, $suffix) = @_; - if ($suffix =~ /^\//) { return $suffix; } - my $res = $base . '/' . $suffix; - $res =~ s/\/\//\//g; - return $res; -} - -sub write_cdec_ini { - my ($filename, $grammar_path) = (@_); - open CDECINI, ">$filename" or die "Can't write $filename: $!"; - print CDECINI <> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - diff --git a/gi/pipeline/backoff-pipe.pl b/gi/pipeline/backoff-pipe.pl deleted file mode 100644 index ac103c8b..00000000 --- a/gi/pipeline/backoff-pipe.pl +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my @grammars; -my $OUTPUTPREFIX = './giwork/bo.hier.grammar'; -safemkdir($OUTPUTPREFIX); -my $backoff_levels = 1; -my $glue_levels = 1; - -usage() unless &GetOptions('grmr=s@' => \ @grammars, - 'outprefix=s' => \ $OUTPUTPREFIX, - 'bo-lvls=i' => \ $backoff_levels, - 'glue-lvls=i' => \ $glue_levels, -); - -my $OUTDIR = $OUTPUTPREFIX . '/hier'; -print STDERR "@grammars\n"; - - -my %grmr = (); -foreach my $grammar (@grammars) { - $grammar =~ m/\/[^\/]*\.t(\d+)\.[^\/]*/; - $grmr{$1} = $grammar; -} - -my @index = sort keys %grmr; -$OUTDIR = $OUTDIR . join('-',@index); -safemkdir($OUTDIR); -my $BACKOFF_GRMR = $OUTDIR . '/backoff.hier.gz'; -safesystem("echo \"\" | gzip > $BACKOFF_GRMR"); -my $GLUE_GRMR = $OUTDIR . '/glue.hier.gz'; -safesystem("echo \"\" | gzip > $GLUE_GRMR"); -my $joinedgrammars = $OUTDIR . '/grammar.hier.gz'; - -join_grammars(); - -for my $i (0..(scalar @index)-2) { - my $freqs = extract_freqs($index[$i], $index[$i+1]); - if ($i < $backoff_levels) { - create_backoff_rules($index[$i],$index[$i+1],$freqs); - } - if ($i < $glue_levels) { - add_glue_rules($index[$i]); - } -} - -output_grammar_info(); - - -sub usage { - print <> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - - -sub join_grammars { - print STDERR "\n!!! JOINING GRAMMARS\n"; - if(-e $joinedgrammars) { - print STDERR "$joinedgrammars exists, reusing...\n"; - return; - } - safesystem("echo \"\" | gzip > $joinedgrammars"); - foreach my $i (@index) { - my $g = $grmr{$i}; - safesystem("zcat $g | sed -r -e 's/X([0-9]+)/X$i\\1/g' - | gzip > $g.2.gz"); - safesystem("zcat $joinedgrammars $g.2.gz | gzip > $joinedgrammars.2.gz"); - safesystem("mv $joinedgrammars.2.gz $joinedgrammars"); - } -} - - -sub extract_freqs { - my($grmr1,$grmr2) = @_; - print STDERR "\n!!!EXTRACTING FREQUENCIES: $grmr1->$grmr2\n"; - my $IN_COARSE = substr($grmr{$grmr1},0,index($grmr{$grmr1},".grammar/")) . "/labeled_spans.txt"; - my $IN_FINE = substr($grmr{$grmr2},0,index($grmr{$grmr2},".grammar/")) . "/labeled_spans.txt"; - my $OUT_SPANS = "$OUTDIR/labeled_spans.hier$grmr1-$grmr2.txt"; - my $FREQS = "$OUTDIR/label_freq.hier$grmr1-$grmr2.txt"; - if(-e $OUT_SPANS && -e $FREQS) { - print STDERR "$OUT_SPANS exists, reusing...\n"; - print STDERR "$FREQS exists, reusing...\n"; - return $FREQS; - } - - safesystem("paste -d ' ' $IN_COARSE $IN_FINE > $OUT_SPANS"); - - my %FREQ_HIER = (); - my %finehier = (); - - open SPANS, $OUT_SPANS or die $!; - while () { - my ($tmp, $coarse, $fine) = split /\|\|\|/; - my @coarse_spans = $coarse =~ /\d+-\d+:X(\d+)/g; - my @fine_spans = $fine =~ /\d+-\d+:X(\d+)/g; - - foreach my $i (0..(scalar @coarse_spans)-1) { - my $coarse_cat = $coarse_spans[$i]; - my $fine_cat = $fine_spans[$i]; - - $FREQ_HIER{$coarse_cat}{$fine_cat}++; - } - } - close SPANS; - foreach (values %FREQ_HIER) { - my $coarse_freq = $_; - my $total = 0; - $total+=$_ for (values %{ $coarse_freq }); - $coarse_freq->{$_}=log($coarse_freq->{$_}/$total) for (keys %{ $coarse_freq }); - } - open FREQS, ">", $FREQS or die $!; - foreach my $coarse_cat (keys %FREQ_HIER) { - print FREQS "$coarse_cat |||"; - foreach my $fine_cat (keys %{$FREQ_HIER{$coarse_cat}}) { - my $freq = $FREQ_HIER{$coarse_cat}{$fine_cat}; - print FREQS " $fine_cat:$freq"; - if(! exists $finehier{$fine_cat} || $finehier{$fine_cat} < $freq) { - $finehier{$fine_cat} = $coarse_cat; - } - } - print FREQS "\n"; - } -# foreach my $fine_cat (keys %finehier) { -# print FREQS "$fine_cat -> $finehier{$fine_cat}\n"; -# } - close FREQS; - return $FREQS; -} - - -sub create_backoff_rules { - print STDERR "\n!!! CREATING BACKOFF RULES\n"; - my ($grmr1, $grmr2, $freq) = @_; - my $OUTFILE = "$OUTDIR/backoff.hier$grmr1-$grmr2.txt"; - if(-e $OUTFILE) { - print STDERR "$OUTFILE exists, reusing...\n"; - return; - } - open FREQS, $freq or die $!; - open TMP, ">", $OUTFILE or die $!; - while () { - my $line = $_; - $line = m/^(\d+) \|\|\| (.+)$/; - my $coarse = $1; - $line = $2; - my @finefreq = $line =~ m/(\d+):(\S+)/g; - for(my $i = 0; $i < scalar @finefreq; $i+=2) { - my $finecat = $finefreq[$i]; - my $finefreq = $finefreq[$i+1]; - print TMP "[X$grmr1$coarse] ||| [X$grmr2$finecat,1]\t[1] ||| BackoffRule=$finefreq A=0-0\n"; - } - } - close TMP; - close FREQS; - safesystem("zcat $BACKOFF_GRMR | cat - $OUTFILE | gzip > $BACKOFF_GRMR.2.gz"); - safesystem("mv $BACKOFF_GRMR.2.gz $BACKOFF_GRMR"); -} - -sub add_glue_rules { - print STDERR "\n!!! CREATING GLUE RULES\n"; - my ($grmr) = @_; - my $OUTFILE = "$OUTDIR/glue.$grmr.gz"; - if (-e $OUTFILE) { - print STDERR "$OUTFILE exists, reusing...\n"; - return; - } - open TMP, ">", $OUTFILE or die $!; - for my $i (0..($grmr-1)) { - print TMP "[S] ||| [S,1] [X$grmr$i,2] ||| [1] [2] ||| Glue=1\n"; - print TMP "[S] ||| [X$grmr$i,1] ||| [1] ||| GlueTop=1\n"; - } - close TMP; - safesystem("zcat $GLUE_GRMR | cat - $OUTFILE | gzip > $GLUE_GRMR.2.gz"); - safesystem("mv $GLUE_GRMR.2.gz $GLUE_GRMR"); -} - -sub output_grammar_info { - print STDERR "\n!!! GRAMMAR INFORMATION\n"; - print STDOUT "GRAMMAR: \t$joinedgrammars\n"; - print STDOUT "GLUE: \t$GLUE_GRMR\n"; - print STDOUT "BACKOFF: \t$BACKOFF_GRMR\n"; -} diff --git a/gi/pipeline/blacklight.config b/gi/pipeline/blacklight.config deleted file mode 100644 index fc59a604..00000000 --- a/gi/pipeline/blacklight.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/usr/users/0/cdyer/ws10smt/data -btec /home/cdyer/ws10smt-data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -zhen /home/cdyer/ws10smt-data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /home/cdyer/ws10smt-data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /usr/users/0/cdyer/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/cdyer/ws10smt-data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/clsp.config b/gi/pipeline/clsp.config deleted file mode 100644 index c23d409f..00000000 --- a/gi/pipeline/clsp.config +++ /dev/null @@ -1,10 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/export/ws10smt/data -btec /export/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /export/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /export/ws10smt/data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /export/ws10smt/data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /export/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /export/ws10smt/data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/evaluation-pipeline.pl b/gi/pipeline/evaluation-pipeline.pl deleted file mode 100755 index 4b4529d9..00000000 --- a/gi/pipeline/evaluation-pipeline.pl +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use Getopt::Long; -use Cwd; -my $CWD = getcwd; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR, "$SCRIPT_DIR/../../environment"; } -use LocalConfig; - -my $JOBS = 15; -my $PMEM = "9G"; -my $NUM_TRANSLATIONS = 50; -my $GOAL = "S"; - -# featurize_grammar may add multiple features from a single feature extractor -# the key in this map is the extractor name, the value is a list of the extracted features -my $feat_map = { - "LogRuleCount" => [ "LogRuleCount", "SingletonRule" ] , -# "XFeatures" => [ "XFE","XEF" ] , - "XFeatures" => [ "XFE","XEF","LabelledEF","LabelledFE"], # ,"XE_Singleton","XF_Singleton"] , - "LabelledRuleConditionals" => [ "LabelledFE","LabelledEF" ] , - "LexProb" => [ "LexE2F", "LexF2E" ] , - "BackoffRule" => [ "BackoffRule" ] , - "RulePenalty" => [ "RulePenalty" ] , - "LHSProb" => [ "LHSProb" ] , - "LabellingShape" => [ "LabellingShape" ] , - "GenerativeProb" => [ "GenerativeProb" ] , -}; - -my %init_weights = qw( - EGivenF -0.735245 - FGivenE -0.219391 - Glue -0.306709 - GlueTop 0.0473331 - LanguageModel 2.40403 - LexE2F -0.266989 - LexF2E -0.550373 - LogECount -0.129853 - LogFCount -0.194037 - LogRuleCount 0.256706 - BackoffRule 0.5 - XFE -0.256706 - XEF -0.256706 - XF_Singleton -0.05 - XE_Singleton -0.8 - LabelledFE -0.256706 - LabelledEF -0.256706 - PassThrough -0.9304905 - SingletonE -3.04161 - SingletonF 0.0714027 - SingletonRule -0.889377 - WordPenalty -1.99495 - RulePenalty -0.1 - LabellingShape -0.1 - LHSProb -0.1 - GenerativeProb -0.1 -); - - -# these features are included by default -my @DEFAULT_FEATS = qw( PassThrough Glue GlueTop LanguageModel WordPenalty ); - - - -my $FILTERBYF = "$SCRIPT_DIR/scripts/filter-by-f.pl"; -my $CDEC = "$SCRIPT_DIR/../../decoder/cdec"; -my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl"; -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $VEST = "$SCRIPT_DIR/../../vest"; -die "Can't find vest: $VEST" unless -e $VEST && -d $VEST; -my $DISTVEST = "$VEST/dist-vest.pl"; -my $FILTER = "$EXTOOLS/filter_grammar"; -my $FEATURIZE = "$EXTOOLS/featurize_grammar"; -assert_exec($CDEC, $PARALLELIZE, $FILTER, $FEATURIZE, $DISTVEST, $FILTERBYF); - -my $numtopics = 25; - -my $config = "$SCRIPT_DIR/" . (lc environment_name()) . '.config'; -print STDERR "CORPORA CONFIGURATION: $config\n"; -open CONF, "<$config" or die "Can't read $config: $!"; -my %paths; -my %corpora; -my %lms; -my %devs; -my %devrefs; -my %tests; -my %testevals; -my $datadir; -print STDERR " LANGUAGE PAIRS:"; -while() { - chomp; - next if /^#/; - next if /^\s*$/; - s/^\s+//; - s/\s+$//; - if (! defined $datadir) { $datadir = $_; next; } - my ($name, $path, $corpus, $lm, $dev, $devref, @xtests) = split /\s+/; - $paths{$name} = $path; - $corpora{$name} = $corpus; - $lms{$name} = $lm; - $devs{$name} = $dev; - $devrefs{$name} = $devref; - $tests{$name} = $xtests[0]; - $testevals{$name} = $xtests[1]; - print STDERR " $name"; -} -print STDERR "\n"; - -my %langpairs = map { $_ => 1 } qw( btec zhen fbis aren uren nlfr ); - -my $outdir = "$CWD/exp"; -my $help; -my $FEATURIZER_OPTS = ''; -my $dataDir = '/export/ws10smt/data'; -my @features; -my $bkoffgram; -my $gluegram; -my $oovgram; -my $usefork; -my $lmorder = 3; -my $density; -if (GetOptions( - "backoff-grammar=s" => \$bkoffgram, - "density-prune=f" => \$density, - "glue-grammar=s" => \$gluegram, - "oov-grammar=s" => \$oovgram, - "data=s" => \$dataDir, - "pmem=s" => \$PMEM, - "n=i" => \$NUM_TRANSLATIONS, - "features=s@" => \@features, - "use-fork" => \$usefork, - "jobs=i" => \$JOBS, - "out-dir=s" => \$outdir, - "lmorder=i" => \$lmorder, - "goal=s" => \$GOAL, -) == 0 || @ARGV!=2 || $help) { - print_help(); - exit; -} -my $DENSITY_PRUNE = ''; -if ($density) { - $DENSITY_PRUNE = "--density-prune $density"; -} -if ($usefork) { $usefork="--use-fork"; } else { $usefork = ''; } -my @fkeys = keys %$feat_map; -die "You must specify one or more features with -f. Known features: @fkeys\n" unless scalar @features > 0; -my @xfeats; -for my $feat (@features) { - my $rs = $feat_map->{$feat}; - if (!defined $rs) { die "DON'T KNOW ABOUT FEATURE $feat\n"; } - my @xfs = @$rs; - @xfeats = (@xfeats, @xfs); - $FEATURIZER_OPTS .= " -f $feat" unless $feat eq "BackoffRule"; -} -print STDERR "X-FEATS: @xfeats\n"; - -my $lp = $ARGV[0]; -my $grammar = $ARGV[1]; -print STDERR " CORPUS REPO: $dataDir\n"; -print STDERR " LANGUAGE PAIR: $lp\n"; -die "I don't know about that language pair\n" unless $paths{$lp}; -my $corpdir = "$dataDir"; -if ($paths{$lp} =~ /^\//) { $corpdir = $paths{$lp}; } else { $corpdir .= '/' . $paths{$lp}; } -die "I can't find the corpora directory: $corpdir" unless -d $corpdir; -print STDERR " GRAMMAR: $grammar\n"; -my $LANG_MODEL = mydircat($corpdir, $lms{$lp}); -print STDERR " LM: $LANG_MODEL\n"; -my $CORPUS = mydircat($corpdir, $corpora{$lp}); -die "Can't find corpus: $CORPUS" unless -f $CORPUS; - -my $dev = mydircat($corpdir, $devs{$lp}); -my $drefs = $devrefs{$lp}; -die "Can't find dev: $dev\n" unless -f $dev; -die "Dev refs not set" unless $drefs; -$drefs = mydircat($corpdir, $drefs); - -my $test = mydircat($corpdir, $tests{$lp}); -my $teval = mydircat($corpdir, $testevals{$lp}); -#die "Can't find test: $test\n" unless -f $test; -#assert_exec($teval); - -`mkdir -p $outdir`; - -# CREATE INIT WEIGHTS -print STDERR "\nCREATING INITIAL WEIGHTS FILE: weights.init\n"; -my $weights = mydircat($outdir, "weights.init"); -write_random_weights_file($weights, @xfeats); - -my $bkoff_grmr; -my $glue_grmr; -if($bkoffgram) { - print STDERR "Placing backoff grammar…\n"; - $bkoff_grmr = mydircat($outdir, "backoff.scfg.gz"); - print STDERR "cp $bkoffgram $bkoff_grmr\n"; - safesystem(undef,"cp $bkoffgram $bkoff_grmr"); -} -if($gluegram) { - print STDERR "Placing glue grammar…\n"; - $glue_grmr = mydircat($outdir, "glue.bo.scfg.gz"); - print STDERR "cp $gluegram $glue_grmr\n"; - safesystem(undef,"cp $gluegram $glue_grmr"); -} - -# MAKE DEV -print STDERR "\nFILTERING FOR dev...\n"; -print STDERR "DEV: $dev (REFS=$drefs)\n"; -my $devgrammar = filter($grammar, $dev, 'dev', $outdir); -my $devini = mydircat($outdir, "cdec-dev.ini"); -write_cdec_ini($devini, $devgrammar); - - -# MAKE TEST -print STDERR "\nFILTERING FOR test...\n"; -print STDERR "TEST: $test (EVAL=$teval)\n"; -`mkdir -p $outdir`; -my $testgrammar = filter($grammar, $test, 'test', $outdir); -my $testini = mydircat($outdir, "cdec-test.ini"); -write_cdec_ini($testini, $testgrammar); - - -# VEST -print STDERR "\nMINIMUM ERROR TRAINING\n"; -my $tuned_weights = mydircat($outdir, 'weights.tuned'); -if (-f $tuned_weights) { - print STDERR "TUNED WEIGHTS $tuned_weights EXISTS: REUSING\n"; -} else { - my $cmd = "$DISTVEST $usefork $DENSITY_PRUNE --decode-nodes $JOBS --pmem=$PMEM --ref-files=$drefs --source-file=$dev --weights $weights $devini"; - print STDERR "MERT COMMAND: $cmd\n"; - `rm -rf $outdir/vest 2> /dev/null`; - chdir $outdir or die "Can't chdir to $outdir: $!"; - $weights = `$cmd`; - die "MERT reported non-zero exit code" unless $? == 0; - chomp $weights; - safesystem($tuned_weights, "cp $weights $tuned_weights"); - print STDERR "TUNED WEIGHTS: $tuned_weights\n"; - die "$tuned_weights is missing!" unless -f $tuned_weights; -} - -# DECODE -print STDERR "\nDECODE TEST SET\n"; -my $decolog = mydircat($outdir, "test-decode.log"); -my $testtrans = mydircat($outdir, "test.trans"); -my $cmd = "cat $test | $PARALLELIZE $usefork -j $JOBS -e $decolog -- $CDEC -c $testini -w $tuned_weights > $testtrans"; -safesystem($testtrans, $cmd) or die "Failed to decode test set!"; - - -# EVALUATE -print STDERR "\nEVALUATE TEST SET\n"; -print STDERR "TEST: $testtrans\n"; -$cmd = "$teval $testtrans"; -safesystem(undef, $cmd) or die "Failed to evaluate!"; -exit 0; - - -sub write_random_weights_file { - my ($file, @extras) = @_; - if (-f $file) { - print STDERR "$file exists - REUSING!\n"; - return; - } - open F, ">$file" or die "Can't write $file: $!"; - my @feats = (@DEFAULT_FEATS, @extras); - for my $feat (@feats) { - my $r = rand(0.4) + 0.8; - my $w = $init_weights{$feat} * $r; - if ($w == 0) { $w = 0.0001; print STDERR "WARNING: $feat had no initial weight!\n"; } - print F "$feat $w\n"; - } - close F; -} - -sub filter { - my ($grammar, $set, $name, $outdir) = @_; - my $out1 = mydircat($outdir, "$name.filt.gz"); - my $out2 = mydircat($outdir, "$name.f_feat.gz"); - my $outgrammar = mydircat($outdir, "$name.scfg.gz"); - if (-f $outgrammar) { print STDERR "$outgrammar exists - REUSING!\n"; } else { - my $cmd = "gunzip -c $grammar | $FILTER -t $set | gzip > $out1"; - safesystem($out1, $cmd) or die "Filtering failed."; - $cmd = "gunzip -c $out1 | $FEATURIZE $FEATURIZER_OPTS -g $out1 -c $CORPUS | gzip > $out2"; - safesystem($out2, $cmd) or die "Featurizing failed"; - $cmd = "$FILTERBYF $NUM_TRANSLATIONS $out2 $outgrammar"; - safesystem($outgrammar, $cmd) or die "Secondary filtering failed"; - } - return $outgrammar; -} - -sub mydircat { - my ($base, $suffix) = @_; - if ($suffix =~ /^\//) { return $suffix; } - my $res = $base . '/' . $suffix; - $res =~ s/\/\//\//g; - return $res; -} - -sub write_cdec_ini { - my ($filename, $grammar_path) = (@_); - open CDECINI, ">$filename" or die "Can't write $filename: $!"; - my $glue = ($gluegram ? "$glue_grmr" : "$datadir/glue/glue.scfg.gz"); - my $oov = ($oovgram ? "$oovgram" : "$datadir/oov.scfg.gz"); - print CDECINI <> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - diff --git a/gi/pipeline/local-gi-pipeline.pl b/gi/pipeline/local-gi-pipeline.pl deleted file mode 100755 index e31167a2..00000000 --- a/gi/pipeline/local-gi-pipeline.pl +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use File::Copy; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -use Getopt::Long "GetOptions"; - -my $GZIP = 'gzip'; -my $ZCAT = 'gunzip -c'; -my $SED = 'sed -e'; -my $BASE_PHRASE_MAX_SIZE = 10; -my $COMPLETE_CACHE = 1; -my $ITEMS_IN_MEMORY = 10000000; # cache size in extractors -my $NUM_TOPICS = 50; -my $NUM_TOPICS_COARSE; -my $NUM_TOPICS_FINE = $NUM_TOPICS; -my $NUM_SAMPLES = 1000; -my $CONTEXT_SIZE = 1; -my $BIDIR = 0; -my $TOPICS_CONFIG = "pyp-topics.conf"; -my $LANGUAGE = "target"; -my $LABEL_THRESHOLD = "0"; -my $PRESERVE_PHRASES; - -my $MODEL = "pyp"; -my $NUM_ITERS = 100; -my $PR_SCALE_P = 0; -my $PR_SCALE_C = 0; -my $PR_FLAGS = ""; -my $MORFMARK = ""; - -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $PYPTOOLS = "$SCRIPT_DIR/../pyp-topics/src"; -die "Can't find pyp-topics: $PYPTOOLS" unless -e $PYPTOOLS && -d $PYPTOOLS; -my $PYPSCRIPTS = "$SCRIPT_DIR/../pyp-topics/scripts"; -die "Can't find pyp-topics: $PYPSCRIPTS" unless -e $PYPSCRIPTS && -d $PYPSCRIPTS; -my $PRTOOLS = "$SCRIPT_DIR/../posterior-regularisation"; -die "Can't find posterior-regularisation: $PRTOOLS" unless -e $PRTOOLS && -d $PRTOOLS; -my $REDUCER = "$EXTOOLS/mr_stripe_rule_reduce"; -my $C2D = "$PYPSCRIPTS/contexts2documents.py"; -my $S2L = "$PYPSCRIPTS/spans2labels.py"; -my $SPLIT = "$SCRIPT_DIR/../posterior-regularisation/split-languages.py"; - -my $PREM_TRAIN="$PRTOOLS/prjava/train-PR-cluster.sh"; - -my $SORT_KEYS = "$SCRIPT_DIR/scripts/sort-by-key.sh"; -my $PATCH_CORPUS = "$SCRIPT_DIR/scripts/patch-corpus.pl"; -my $REMOVE_TAGS_CORPUS = "$SCRIPT_DIR/scripts/remove-tags-from-corpus.pl"; -my $REMOVE_TAGS_CONTEXT = "$SCRIPT_DIR/scripts/remove-tags-from-contexts.pl"; -my $EXTRACTOR = "$EXTOOLS/extractor"; -my $TOPIC_TRAIN = "$PYPTOOLS/pyp-contexts-train"; -my $MORF_DOC_FILTER = "$SCRIPT_DIR/../morf-segmentation/filter_docs.pl"; - -assert_exec($PATCH_CORPUS, $SORT_KEYS, $REDUCER, $EXTRACTOR, - $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS, $MORF_DOC_FILTER); - -my $BACKOFF_GRAMMAR; -my $DEFAULT_CAT; -my $HIER_CAT; -my %FREQ_HIER = (); -my $TAGGED_CORPUS; - -my $NAME_SHORTCUT; - -my $OUTPUT = './giwork'; -usage() unless &GetOptions('base_phrase_max_size=i' => \$BASE_PHRASE_MAX_SIZE, - 'backoff_grammar' => \$BACKOFF_GRAMMAR, - 'output=s' => \$OUTPUT, - 'model=s' => \$MODEL, - 'topics=i' => \$NUM_TOPICS_FINE, - 'coarse_topics=i' => \$NUM_TOPICS_COARSE, - 'trg_context=i' => \$CONTEXT_SIZE, - 'samples=i' => \$NUM_SAMPLES, - 'label_threshold=f' => \$LABEL_THRESHOLD, - 'use_default_cat' => \$DEFAULT_CAT, - 'topics-config=s' => \$TOPICS_CONFIG, - 'iterations=i' => \$NUM_ITERS, - 'pr-scale-phrase=f' => \$PR_SCALE_P, - 'pr-scale-context=f' => \$PR_SCALE_C, - 'pr-flags=s' => \$PR_FLAGS, - 'tagged_corpus=s' => \$TAGGED_CORPUS, - 'language=s' => \$LANGUAGE, - 'get_name_only' => \$NAME_SHORTCUT, - 'preserve_phrases' => \$PRESERVE_PHRASES, - 'morf=s' => \$MORFMARK, - ); -if ($NAME_SHORTCUT) { - $NUM_TOPICS = $NUM_TOPICS_FINE; - print STDERR labeled_dir(); - exit 0; -} -usage() unless scalar @ARGV == 1; -my $CORPUS = $ARGV[0]; -open F, "<$CORPUS" or die "Can't read $CORPUS: $!"; close F; - -$NUM_TOPICS = $NUM_TOPICS_FINE; - -$HIER_CAT = ( $NUM_TOPICS_COARSE ? 1 : 0 ); - -print STDERR " Output: $OUTPUT\n"; -my $DATA_DIR = $OUTPUT . '/corpora'; -my $LEX_NAME = "corpus.f_e_a.$LANGUAGE.lex"; -my $CORPUS_LEX = $DATA_DIR . '/' . $LEX_NAME; # corpus used to extract rules -my $CORPUS_CLUSTER = $DATA_DIR . "/corpus.f_e_a.$LANGUAGE.cluster"; # corpus used for clustering (often identical) - -my $CONTEXT_DIR = $OUTPUT . '/' . context_dir(); -my $CLUSTER_DIR = $OUTPUT . '/' . cluster_dir(); -my $LABELED_DIR = $OUTPUT . '/' . labeled_dir(); -my $CLUSTER_DIR_C; -my $CLUSTER_DIR_F; -my $LABELED_DIR_C; -my $LABELED_DIR_F; -if($HIER_CAT) { - $CLUSTER_DIR_F = $CLUSTER_DIR; - $LABELED_DIR_F = $LABELED_DIR; - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR_C = $OUTPUT . '/' . cluster_dir(); - $LABELED_DIR_C = $OUTPUT . '/' . labeled_dir(); - $NUM_TOPICS = $NUM_TOPICS_FINE; -} -my $GRAMMAR_DIR = $OUTPUT . '/' . grammar_dir(); -print STDERR " Context: $CONTEXT_DIR\n Cluster: $CLUSTER_DIR\n Labeled: $LABELED_DIR\n Grammar: $GRAMMAR_DIR\n"; -safemkdir($OUTPUT) or die "Couldn't create output directory $OUTPUT: $!"; -safemkdir($DATA_DIR) or die "Couldn't create output directory $DATA_DIR: $!"; -safemkdir($CONTEXT_DIR) or die "Couldn't create output directory $CONTEXT_DIR: $!"; -safemkdir($CLUSTER_DIR) or die "Couldn't create output directory $CLUSTER_DIR: $!"; -if($HIER_CAT) { - safemkdir($CLUSTER_DIR_C) or die "Couldn't create output directory $CLUSTER_DIR_C: $!"; - safemkdir($LABELED_DIR_C) or die "Couldn't create output directory $LABELED_DIR_C: $!"; -} -safemkdir($LABELED_DIR) or die "Couldn't create output directory $LABELED_DIR: $!"; -safemkdir($GRAMMAR_DIR) or die "Couldn't create output directory $GRAMMAR_DIR: $!"; -if(-e $TOPICS_CONFIG) { - copy($TOPICS_CONFIG, $CLUSTER_DIR) or die "Copy failed: $!"; -} - -setup_data(); - -if (lc($MODEL) eq "blagree") { - extract_bilingual_context(); -} else { - extract_context(); -} - -if (lc($MODEL) eq "pyp") { - if($HIER_CAT) { - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR = $CLUSTER_DIR_C; - topic_train(); - $NUM_TOPICS = $NUM_TOPICS_FINE; - $CLUSTER_DIR = $CLUSTER_DIR_F; - topic_train(); - } else { - topic_train(); - } -} elsif (lc($MODEL) =~ /pr|em|agree/) { - prem_train(); -} else { die "Unsupported model type: $MODEL. Must be one of PYP or PREM.\n"; } -if($HIER_CAT) { - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR = $CLUSTER_DIR_C; - $LABELED_DIR = $LABELED_DIR_C; - label_spans_with_topics(); - $NUM_TOPICS = $NUM_TOPICS_FINE; - $CLUSTER_DIR = $CLUSTER_DIR_F; - $LABELED_DIR = $LABELED_DIR_F; - label_spans_with_topics(); - extract_freqs(); -} else { - label_spans_with_topics(); -} -my $res; -if ($BIDIR) { - $res = grammar_extract_bidir(); -} else { - $res = grammar_extract(); -} -print STDERR "\n!!!COMPLETE!!!\n"; -print STDERR "GRAMMAR: $res\nYou should probably run: $SCRIPT_DIR/evaluation-pipeline.pl LANGPAIR giwork/ct1s0.L10.PYP.t4.s20.grammar/grammar.gz -f FEAT1 -f FEAT2\n\n"; -exit 0; - -sub setup_data { - print STDERR "\n!!!PREPARE CORPORA!!!\n"; - if (-f $CORPUS_LEX && $CORPUS_CLUSTER) { - print STDERR "$CORPUS_LEX and $CORPUS_CLUSTER exist, reusing...\n"; - return; - } - copy($CORPUS, $CORPUS_LEX); - if ($TAGGED_CORPUS) { - die "Can't find $TAGGED_CORPUS" unless -f $TAGGED_CORPUS; - my $opt=""; - $opt = "-s" if ($LANGUAGE eq "source"); - $opt = $opt . " -a" if ($PRESERVE_PHRASES); - my $cmd="$PATCH_CORPUS $opt $TAGGED_CORPUS $CORPUS_LEX > $CORPUS_CLUSTER"; - safesystem($cmd) or die "Failed to extract contexts."; - } else { - symlink($LEX_NAME, $CORPUS_CLUSTER); - } -} - -sub context_dir { - return "ct${CONTEXT_SIZE}s0.L$BASE_PHRASE_MAX_SIZE.l$LANGUAGE"; -} - -sub cluster_dir { - if (lc($MODEL) eq "pyp") { - return context_dir() . ".PYP.t$NUM_TOPICS.s$NUM_SAMPLES"; - } elsif (lc($MODEL) eq "em") { - return context_dir() . ".EM.t$NUM_TOPICS.i$NUM_ITERS"; - } elsif (lc($MODEL) eq "pr") { - return context_dir() . ".PR.t$NUM_TOPICS.i$NUM_ITERS.sp$PR_SCALE_P.sc$PR_SCALE_C"; - } elsif (lc($MODEL) eq "agree") { - return context_dir() . ".AGREE.t$NUM_TOPICS.i$NUM_ITERS"; - } elsif (lc($MODEL) eq "blagree") { - return context_dir() . ".BLAGREE.t$NUM_TOPICS.i$NUM_ITERS"; - } -} - -sub labeled_dir { - if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD ne "0") { - return cluster_dir() . "_lt$LABEL_THRESHOLD"; - } else { - return cluster_dir(); - } -} - -sub grammar_dir { - # TODO add grammar config options -- adjacent NTs, etc - if($HIER_CAT) { - return cluster_dir() . ".hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.grammar"; - } else { - return labeled_dir() . ".grammar"; - } -} - - - -sub safemkdir { - my $dir = shift; - if (-d $dir) { return 1; } - return mkdir($dir); -} - -sub usage { - print < $CLUSTER_DIR/clusters.txt") or die "Failed to unzip"; - safesystem("$EXTRACTOR --base_phrase_spans -i $CORPUS_CLUSTER -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -S $CONTEXT_SIZE | $S2L $CLUSTER_DIR/clusters.txt $CONTEXT_SIZE $LABEL_THRESHOLD $extra > $OUT_SPANS") or die "Failed to label spans"; - unlink("$CLUSTER_DIR/clusters.txt") or warn "Failed to remove $CLUSTER_DIR/clusters.txt"; - safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS | sed 's/ *||| *\$//' > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste"; - } -} - -sub extract_freqs { - print STDERR "\n!!!EXTRACTING FREQUENCIES\n"; - my $IN_COARSE = "$LABELED_DIR_C/labeled_spans.txt"; - my $IN_FINE = "$LABELED_DIR_F/labeled_spans.txt"; - my $OUT_SPANS = "$LABELED_DIR_F/labeled_spans.hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.txt"; - my $FREQS = "$LABELED_DIR_F/label_freq.hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.txt"; - my $COARSE_EXPR = "\'s/\\(X[0-9][0-9]*\\)/\\1c/g\'"; #' - my $FINE_EXPR = "\'s/\\(X[0-9][0-9]*\\)/\\1f/g\'"; #' - my %finehier = (); - if (-e $OUT_SPANS) { - print STDERR "$OUT_SPANS exists, reusing...\n"; - } else { - safesystem("paste -d ' ' $IN_COARSE $IN_FINE > $OUT_SPANS"); - } - open SPANS, $OUT_SPANS or die $!; - while () { - my ($tmp, $coarse, $fine) = split /\|\|\|/; - my @coarse_spans = $coarse =~ /\d+-\d+:X(\d+)/g; - my @fine_spans = $fine =~ /\d+-\d+:X(\d+)/g; - - foreach my $i (0..(scalar @coarse_spans)-1) { - my $coarse_cat = $coarse_spans[$i]; - my $fine_cat = $fine_spans[$i]; - - $FREQ_HIER{$coarse_cat}{$fine_cat}++; - } - } - close SPANS; - foreach (values %FREQ_HIER) { - my $coarse_freq = $_; - my $total = 0; - $total+=$_ for (values %{ $coarse_freq }); - $coarse_freq->{$_}=log($coarse_freq->{$_}/$total) for (keys %{ $coarse_freq }); - } - open FREQS, ">", $FREQS or die $!; - foreach my $coarse_cat (keys %FREQ_HIER) { - print FREQS "$coarse_cat |||"; - foreach my $fine_cat (keys %{$FREQ_HIER{$coarse_cat}}) { - my $res = $FREQ_HIER{$coarse_cat}{$fine_cat}; - print FREQS " $fine_cat:$res"; - if(! exists $finehier{$fine_cat} || $finehier{$fine_cat} < $res) { - $finehier{$fine_cat} = $coarse_cat; - } - } - print FREQS "\n"; - } -# foreach my $fine_cat (keys %finehier) { -# print FREQS "$fine_cat -> $finehier{$fine_cat}\n"; -# } - close FREQS; - $CLUSTER_DIR = $CLUSTER_DIR_F; -} - -sub grammar_extract { - my $LABELED = "$LABELED_DIR/corpus.src_trg_al_label"; - print STDERR "\n!!!EXTRACTING GRAMMAR\n"; - my $OUTGRAMMAR = "$GRAMMAR_DIR/grammar.gz"; - if (-e $OUTGRAMMAR) { - print STDERR "$OUTGRAMMAR exists, reusing...\n"; - } else { - my $BACKOFF_ARG = ($BACKOFF_GRAMMAR ? "-g" : ""); - my $DEFAULT_CAT_ARG = ($DEFAULT_CAT ? "-d X" : ""); - safesystem("$EXTRACTOR -i $LABELED -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -t $NUM_TOPICS $BACKOFF_ARG $DEFAULT_CAT_ARG | $SORT_KEYS | $REDUCER -p | $GZIP > $OUTGRAMMAR") or die "Couldn't extract grammar"; - } - return $OUTGRAMMAR; -} - -sub grammar_extract_bidir { -#gzcat ex.output.gz | ./mr_stripe_rule_reduce -p -b | sort -t $'\t' -k 1 | ./mr_stripe_rule_reduce | gzip > phrase-table.gz - my $LABELED = "$LABELED_DIR/corpus.src_trg_al_label"; - print STDERR "\n!!!EXTRACTING GRAMMAR\n"; - my $OUTGRAMMAR = "$GRAMMAR_DIR/grammar.bidir.gz"; - if (-e $OUTGRAMMAR) { - print STDERR "$OUTGRAMMAR exists, reusing...\n"; - } else { - my $BACKOFF_ARG = ($BACKOFF_GRAMMAR ? "-g" : ""); - safesystem("$EXTRACTOR -i $LABELED -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -b -t $NUM_TOPICS $BACKOFF_ARG | $SORT_KEYS | $REDUCER -p -b | $SORT_KEYS | $REDUCER | $GZIP > $OUTGRAMMAR") or die "Couldn't extract grammar"; - } - return $OUTGRAMMAR; -} - -sub safesystem { - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - exit(1); - } - else { - my $exitcode = $? >> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - diff --git a/gi/pipeline/lticluster.config b/gi/pipeline/lticluster.config deleted file mode 100644 index 3e23c8cb..00000000 --- a/gi/pipeline/lticluster.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/home/cdyer/ws10smt-data -btec /home/cdyer/ws10smt-data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -zhen /home/cdyer/ws10smt-data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /home/cdyer/ws10smt-data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /home/cdyer/ws10smt-data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/cdyer/ws10smt-data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/scripts/filter-by-f.pl b/gi/pipeline/scripts/filter-by-f.pl deleted file mode 100755 index 0cef0606..00000000 --- a/gi/pipeline/scripts/filter-by-f.pl +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -my $REKEY="$SCRIPT_DIR/rekey.pl"; -my $REFILTER="$SCRIPT_DIR/refilter.pl"; -my $SORT="$SCRIPT_DIR/sort-by-key.sh"; -assert_exec($REKEY, $REFILTER, $SORT); - - -die "Usage: $0 NUM-TRANSLATIONS ingrammar.gz outgrammar.gz\n" unless scalar @ARGV == 3; -my $translations = shift @ARGV; -die "Need number: $translations" unless $translations > 0; -die unless $ARGV[0] =~ /\.gz$/; -die unless $ARGV[1] =~ /\.gz$/; -die if $ARGV[0] eq $ARGV[1]; -die "Can't find $ARGV[0]" unless -f $ARGV[0]; - -my $cmd = "gunzip -c $ARGV[0] | $REKEY | $SORT | $REFILTER $translations | gzip > $ARGV[1]"; -safesystem($ARGV[1], $cmd) or die "Filtering failed"; -exit 0; - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - -sub safesystem { - my $output = shift @_; - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - else { - my $exitcode = $? >> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - diff --git a/gi/pipeline/scripts/patch-corpus.pl b/gi/pipeline/scripts/patch-corpus.pl deleted file mode 100755 index c0eec43e..00000000 --- a/gi/pipeline/scripts/patch-corpus.pl +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $PATCH = shift @ARGV; -my $TGT = 1; -my $APPEND; -while ($PATCH eq "-s" || $PATCH eq "-a") { - if ($PATCH eq "-s") { - undef $TGT; - } else { - $APPEND = 1; - } - $PATCH = shift @ARGV; -} - -die "Usage: $0 [-s] [-a] tagged.en[_fr] < lexical.en_fr_al[_...]\n" unless $PATCH; - -open P, "<$PATCH" or die "Can't read tagged corpus $PATCH: $!"; -my $first=

; close P; -my @fields = split / \|\|\| /, $first; -die "Bad format!" if (scalar @fields > 2); - -if (scalar @fields != 1) { - # TODO support this - die "Patching source and target not supported yet!"; -} - -my $line = 0; -open P, "<$PATCH" or die "Can't read tagged corpus $PATCH: $!"; -while(my $pline =

) { - chomp $pline; - $line++; - my $line = <>; - die "Too few lines in lexical corpus!" unless $line; - chomp $line; - @fields = split / \|\|\| /, $line; - my @pwords = split /\s+/, $pline; - if ($TGT) { - my @lwords = split /\s+/, $fields[1]; - die "Length mismatch in line $line!\n" unless (scalar @pwords == scalar @lwords); - if ($APPEND) { - foreach my $i (0..(scalar @pwords-1)) { - $lwords[$i] = $lwords[$i] . '_' . $pwords[$i]; - } - $fields[1] = join ' ', @lwords; - } else { - $fields[1] = $pline; - } - } else { # source side - my @lwords = split /\s+/, $fields[0]; - die "Length mismatch in line $line!\n" unless (scalar @pwords == scalar @lwords); - if ($APPEND) { - foreach my $i (0..(scalar @pwords-1)) { - $lwords[$i] = $lwords[$i] . '_' . $pwords[$i]; - } - $fields[0] = join ' ', @lwords; - } else { - $fields[0] = $pline; - } - } - print join ' ||| ', @fields; - print "\n"; -} - - diff --git a/gi/pipeline/scripts/refilter.pl b/gi/pipeline/scripts/refilter.pl deleted file mode 100755 index a783eb4e..00000000 --- a/gi/pipeline/scripts/refilter.pl +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $NUM_TRANSLATIONS = shift @ARGV; -unless ($NUM_TRANSLATIONS) { $NUM_TRANSLATIONS=30; } -print STDERR "KEEPING $NUM_TRANSLATIONS TRANSLATIONS FOR SOURCE\n"; - -my $pk = ''; -my %dict; -while(<>) { - s/^(.+)\t//; - my $key = $1; - if ($key ne $pk) { - if ($pk) { - emit_dict(); - } - %dict = (); - $pk = $key; - } - my ($lhs, $f, $e, $s) = split / \|\|\| /; - my $score = 0; - if ($s =~ /XEF=([^ ]+)/) { - $score += $1; - } else { die; } - if ($s =~ /GenerativeProb=([^ ]+)/) { - $score += ($1 / 10); - } else { die; } - $dict{"$lhs ||| $f ||| $e ||| $s"} = $score; -} -emit_dict(); - -sub emit_dict { - my $cc = 0; - for my $k (sort { $dict{$a} <=> $dict{$b} } keys %dict) { - print "$k"; - $cc++; - if ($cc >= $NUM_TRANSLATIONS) { last; } - } -} - diff --git a/gi/pipeline/scripts/rekey.pl b/gi/pipeline/scripts/rekey.pl deleted file mode 100755 index 31eb86b8..00000000 --- a/gi/pipeline/scripts/rekey.pl +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/perl - -while(<>) { - my ($lhs, $f, $e, $s) = split / \|\|\| /; - $f =~ s/\[X[0-9]+\]/\[X\]/g; - print "$f\t$_"; -} - diff --git a/gi/pipeline/scripts/remove-tags-from-contexts.pl b/gi/pipeline/scripts/remove-tags-from-contexts.pl deleted file mode 100755 index 20698816..00000000 --- a/gi/pipeline/scripts/remove-tags-from-contexts.pl +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my $PHRASE = 'tok'; -my $CONTEXT = 'tag'; - -die "Usage: $0 [--phrase=tok|tag] [--context=tok|tag] < corpus" - unless &GetOptions('phrase=s' => \$PHRASE, 'context=s' => \$CONTEXT); - -my $lno = 0; -while(my $line = <>) { - $lno++; - chomp $line; - my @top = split /\t/, $line; - die unless (scalar @top == 2); - - my @pwords = split /\s+/, $top[0]; - foreach my $token (@pwords) { - #print $token . "\n"; - my @parts = split /_(?!.*_)/, $token; - die unless (scalar @parts == 2); - if ($PHRASE eq "tok") { - $token = $parts[0] - } elsif ($PHRASE eq "tag") { - $token = $parts[1] - } - } - - my @fields = split / \|\|\| /, $top[1]; - foreach my $i (0..((scalar @fields) / 2 - 1)) { - #print $i . ": " . $fields[2*$i] . " of " . (scalar @fields) . "\n"; - my @cwords = split /\s+/, $fields[2*$i]; - foreach my $token (@cwords) { - #print $i . ": " . $token . "\n"; - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - if ($CONTEXT eq "tok") { - $token = $parts[0] - } elsif ($CONTEXT eq "tag") { - $token = $parts[1] - } - } - } - $fields[2*$i] = join ' ', @cwords; - } - - print join ' ', @pwords; - print "\t"; - print join ' ||| ', @fields; - print "\n"; -} diff --git a/gi/pipeline/scripts/remove-tags-from-corpus.pl b/gi/pipeline/scripts/remove-tags-from-corpus.pl deleted file mode 100755 index be3e97c0..00000000 --- a/gi/pipeline/scripts/remove-tags-from-corpus.pl +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my $LANGUAGE = shift @ARGV; -$LANGUAGE = 'target' unless ($LANGUAGE); - -my $lno = 0; -while(my $line = <>) { - $lno++; - chomp $line; - - my @fields = split / \|\|\| /, $line; - - if ($LANGUAGE eq "source" or $LANGUAGE eq "both") { - my @cwords = split /\s+/, $fields[0]; - foreach my $token (@cwords) { - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - $token = $parts[0] - } else { - print STDERR "WARNING: invalid tagged token $token\n"; - } - } - $fields[0] = join ' ', @cwords; - } - - if ($LANGUAGE eq "target" or $LANGUAGE eq "both") { - my @cwords = split /\s+/, $fields[1]; - foreach my $token (@cwords) { - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - $token = $parts[1] - } else { - print STDERR "WARNING: invalid tagged token $token\n"; - } - } - $fields[0] = join ' ', @cwords; - } - - print join ' ||| ', @fields; - print "\n"; -} diff --git a/gi/pipeline/scripts/sort-by-key.sh b/gi/pipeline/scripts/sort-by-key.sh deleted file mode 100755 index 7ae33e03..00000000 --- a/gi/pipeline/scripts/sort-by-key.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export LANG=C -sort -t $'\t' -k 1 -T /tmp -S 6000000000 - diff --git a/gi/pipeline/scripts/xfeats.pl b/gi/pipeline/scripts/xfeats.pl deleted file mode 100755 index dc578513..00000000 --- a/gi/pipeline/scripts/xfeats.pl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -die "Usage: $0 x-grammar.scfg[.gz] < cat-grammar.scfg\n" unless scalar @ARGV > 0; - -my $xgrammar = shift @ARGV; -die "Can't find $xgrammar" unless -f $xgrammar; -my $fh; -if ($xgrammar =~ /\.gz$/) { - open $fh, "gunzip -c $xgrammar|" or die "Can't fork: $!"; -} else { - open $fh, "<$xgrammar" or die "Can't read $xgrammar: $!"; -} -print STDERR "Reading X-feats from $xgrammar...\n"; -my %dict; -while(<$fh>) { - chomp; - my ($lhs, $f, $e, $feats) = split / \|\|\| /; - my $xfeats; - my $cc = 0; - my @xfeats = (); - while ($feats =~ /(EGivenF|FGivenE|LogRuleCount|LogECount|LogFCount|SingletonRule|SingletonE|SingletonF)=([^ ]+)( |$)/og) { - push @xfeats, "X_$1=$2"; - } - #print "$lhs ||| $f ||| $e ||| @xfeats\n"; - $dict{"$lhs ||| $f ||| $e"} = "@xfeats"; -} -close $fh; - -print STDERR "Add features...\n"; -while(<>) { - chomp; - my ($lhs, $f, $e) = split / \|\|\| /; - $f=~ s/\[[^]]+,([12])\]/\[X,$1\]/g; - my $xfeats = $dict{"[X] ||| $f ||| $e"}; - die "Can't find x features for: $_\n" unless $xfeats; - print "$_ $xfeats\n"; -} - diff --git a/gi/pipeline/valhalla.config b/gi/pipeline/valhalla.config deleted file mode 100644 index e00a8485..00000000 --- a/gi/pipeline/valhalla.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/home/chris/ws10smt/data -btec /home/chris/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /home/chris/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /home/chris/ws10smt/data/chinese-english corpus.zh-en.al -aren /home/chris/ws10smt/data/arabic-english corpus.ar-en.al -uren /home/chris/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/chris/ws10smt/data/dutch-french corpus.nl-fr.al diff --git a/gi/posterior-regularisation/Corpus.java b/gi/posterior-regularisation/Corpus.java deleted file mode 100644 index 07b27387..00000000 --- a/gi/posterior-regularisation/Corpus.java +++ /dev/null @@ -1,167 +0,0 @@ -import gnu.trove.TIntArrayList; - -import java.io.*; -import java.util.*; -import java.util.regex.Pattern; - -public class Corpus -{ - private Lexicon tokenLexicon = new Lexicon(); - private Lexicon phraseLexicon = new Lexicon(); - private Lexicon contextLexicon = new Lexicon(); - private List edges = new ArrayList(); - private List> phraseToContext = new ArrayList>(); - private List> contextToPhrase = new ArrayList>(); - - public class Edge - { - Edge(int phraseId, int contextId, int count) - { - this.phraseId = phraseId; - this.contextId = contextId; - this.count = count; - } - public int getPhraseId() - { - return phraseId; - } - public TIntArrayList getPhrase() - { - return phraseLexicon.lookup(phraseId); - } - public String getPhraseString() - { - StringBuffer b = new StringBuffer(); - for (int tid: getPhrase().toNativeArray()) - { - if (b.length() > 0) - b.append(" "); - b.append(tokenLexicon.lookup(tid)); - } - return b.toString(); - } - public int getContextId() - { - return contextId; - } - public TIntArrayList getContext() - { - return contextLexicon.lookup(contextId); - } - public String getContextString() - { - StringBuffer b = new StringBuffer(); - for (int tid: getContext().toNativeArray()) - { - if (b.length() > 0) - b.append(" "); - b.append(tokenLexicon.lookup(tid)); - } - return b.toString(); - } - public int getCount() - { - return count; - } - private int phraseId; - private int contextId; - private int count; - } - - List getEdges() - { - return edges; - } - - int getNumEdges() - { - return edges.size(); - } - - int getNumPhrases() - { - return phraseLexicon.size(); - } - - List getEdgesForPhrase(int phraseId) - { - return phraseToContext.get(phraseId); - } - - int getNumContexts() - { - return contextLexicon.size(); - } - - List getEdgesForContext(int contextId) - { - return contextToPhrase.get(contextId); - } - - int getNumTokens() - { - return tokenLexicon.size(); - } - - static Corpus readFromFile(Reader in) throws IOException - { - Corpus c = new Corpus(); - - // read in line-by-line - BufferedReader bin = new BufferedReader(in); - String line; - Pattern separator = Pattern.compile(" \\|\\|\\| "); - - while ((line = bin.readLine()) != null) - { - // split into phrase and contexts - StringTokenizer st = new StringTokenizer(line, "\t"); - assert (st.hasMoreTokens()); - String phraseToks = st.nextToken(); - assert (st.hasMoreTokens()); - String rest = st.nextToken(); - assert (!st.hasMoreTokens()); - - // process phrase - st = new StringTokenizer(phraseToks, " "); - TIntArrayList ptoks = new TIntArrayList(); - while (st.hasMoreTokens()) - ptoks.add(c.tokenLexicon.insert(st.nextToken())); - int phraseId = c.phraseLexicon.insert(ptoks); - if (phraseId == c.phraseToContext.size()) - c.phraseToContext.add(new ArrayList()); - - // process contexts - String[] parts = separator.split(rest); - assert (parts.length % 2 == 0); - for (int i = 0; i < parts.length; i += 2) - { - // process pairs of strings - context and count - TIntArrayList ctx = new TIntArrayList(); - String ctxString = parts[i]; - String countString = parts[i + 1]; - StringTokenizer ctxStrtok = new StringTokenizer(ctxString, " "); - while (ctxStrtok.hasMoreTokens()) - { - String token = ctxStrtok.nextToken(); - if (!token.equals("")) - ctx.add(c.tokenLexicon.insert(token)); - } - int contextId = c.contextLexicon.insert(ctx); - if (contextId == c.contextToPhrase.size()) - c.contextToPhrase.add(new ArrayList()); - - assert (countString.startsWith("C=")); - Edge e = c.new Edge(phraseId, contextId, - Integer.parseInt(countString.substring(2).trim())); - c.edges.add(e); - - // index the edge for fast phrase, context lookup - c.phraseToContext.get(phraseId).add(e); - c.contextToPhrase.get(contextId).add(e); - } - } - - return c; - } -} diff --git a/gi/posterior-regularisation/Lexicon.java b/gi/posterior-regularisation/Lexicon.java deleted file mode 100644 index 9f0245ee..00000000 --- a/gi/posterior-regularisation/Lexicon.java +++ /dev/null @@ -1,32 +0,0 @@ -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class Lexicon -{ - public int insert(T word) - { - Integer i = wordToIndex.get(word); - if (i == null) - { - i = indexToWord.size(); - wordToIndex.put(word, i); - indexToWord.add(word); - } - return i; - } - - public T lookup(int index) - { - return indexToWord.get(index); - } - - public int size() - { - return indexToWord.size(); - } - - private Map wordToIndex = new HashMap(); - private List indexToWord = new ArrayList(); -} \ No newline at end of file diff --git a/gi/posterior-regularisation/PhraseContextModel.java b/gi/posterior-regularisation/PhraseContextModel.java deleted file mode 100644 index 85bcfb89..00000000 --- a/gi/posterior-regularisation/PhraseContextModel.java +++ /dev/null @@ -1,466 +0,0 @@ -// Input of the form: -// " the phantom of the opera " tickets for tonight ? ||| C=1 ||| seats for ? ||| C=1 ||| i see ? ||| C=1 -// phrase TAB [context]+ -// where context = phrase ||| C=... which are separated by ||| - -// Model parameterised as follows: -// - each phrase, p, is allocated a latent state, t -// - this is used to generate the contexts, c -// - each context is generated using 4 independent multinomials, one for each position LL, L, R, RR - -// Training with EM: -// - e-step is estimating q(t) = P(t|p,c) for all x,c -// - m-step is estimating model parameters P(c,t|p) = P(t) P(c|t) -// - PR uses alternate e-step, which first optimizes lambda -// min_q KL(q||p) + delta sum_pt max_c E_q[phi_ptc] -// where -// q(t|p,c) propto p(t,c|p) exp( -phi_ptc ) -// Then q is used to obtain expectations for vanilla M-step. - -// Sexing it up: -// - learn p-specific conditionals P(t|p) -// - or generate phrase internals, e.g., generate edge words from -// different distribution to central words -// - agreement between phrase->context model and context->phrase model - -import java.io.*; -import optimization.gradientBasedMethods.*; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.gradientBasedMethods.stats.ProjectedOptimizerStats; -import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc; -import optimization.linesearch.GenericPickFirstStep; -import optimization.linesearch.InterpolationPickFirstStep; -import optimization.linesearch.LineSearchMethod; -import optimization.linesearch.WolfRuleLineSearch; -import optimization.projections.SimplexProjection; -import optimization.stopCriteria.CompositeStopingCriteria; -import optimization.stopCriteria.NormalizedProjectedGradientL2Norm; -import optimization.stopCriteria.NormalizedValueDifference; -import optimization.stopCriteria.ProjectedGradientL2Norm; -import optimization.stopCriteria.StopingCriteria; -import optimization.stopCriteria.ValueDifference; -import optimization.util.MathUtils; -import java.util.*; -import java.util.regex.*; -import gnu.trove.TDoubleArrayList; -import gnu.trove.TIntArrayList; -import static java.lang.Math.*; - -class PhraseContextModel -{ - // model/optimisation configuration parameters - int numTags; - boolean posteriorRegularisation = true; - double constraintScale = 3; // FIXME: make configurable - - // copied from L1LMax in depparsing code - final double c1= 0.0001, c2=0.9, stoppingPrecision = 1e-5, maxStep = 10; - final int maxZoomEvals = 10, maxExtrapolationIters = 200; - int maxProjectionIterations = 200; - int minOccurrencesForProjection = 0; - - // book keeping - int numPositions; - Random rng = new Random(); - - // training set - Corpus training; - - // model parameters (learnt) - double emissions[][][]; // position in 0 .. 3 x tag x word Pr(word | tag, position) - double prior[][]; // phrase x tag Pr(tag | phrase) - double lambda[]; // edge = (phrase, context) x tag flattened lagrange multipliers - - PhraseContextModel(Corpus training, int tags) - { - this.training = training; - this.numTags = tags; - assert (!training.getEdges().isEmpty()); - assert (numTags > 1); - - // now initialise emissions - numPositions = training.getEdges().get(0).getContext().size(); - assert (numPositions > 0); - - emissions = new double[numPositions][numTags][training.getNumTokens()]; - prior = new double[training.getNumEdges()][numTags]; - if (posteriorRegularisation) - lambda = new double[training.getNumEdges() * numTags]; - - for (double[][] emissionTW : emissions) - { - for (double[] emissionW : emissionTW) - { - randomise(emissionW); -// for (int i = 0; i < emissionW.length; ++i) -// emissionW[i] = i+1; -// normalise(emissionW); - } - } - - for (double[] priorTag : prior) - { - randomise(priorTag); -// for (int i = 0; i < priorTag.length; ++i) -// priorTag[i] = i+1; -// normalise(priorTag); - } - } - - void expectationMaximisation(int numIterations) - { - double lastLlh = Double.NEGATIVE_INFINITY; - - for (int iteration = 0; iteration < numIterations; ++iteration) - { - double emissionsCounts[][][] = new double[numPositions][numTags][training.getNumTokens()]; - double priorCounts[][] = new double[training.getNumPhrases()][numTags]; - - // E-step - double llh = 0; - if (posteriorRegularisation) - { - EStepDualObjective objective = new EStepDualObjective(); - - // copied from x2y2withconstraints -// LineSearchMethod ls = new ArmijoLineSearchMinimizationAlongProjectionArc(new InterpolationPickFirstStep(1)); -// OptimizerStats stats = new OptimizerStats(); -// ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls); -// CompositeStopingCriteria compositeStop = new CompositeStopingCriteria(); -// compositeStop.add(new ProjectedGradientL2Norm(0.001)); -// compositeStop.add(new ValueDifference(0.001)); -// optimizer.setMaxIterations(50); -// boolean succeed = optimizer.optimize(objective,stats,compositeStop); - - // copied from depparser l1lmaxobjective - ProjectedOptimizerStats stats = new ProjectedOptimizerStats(); - GenericPickFirstStep pickFirstStep = new GenericPickFirstStep(1); - LineSearchMethod linesearch = new WolfRuleLineSearch(pickFirstStep, c1, c2); - ProjectedGradientDescent optimizer = new ProjectedGradientDescent(linesearch); - optimizer.setMaxIterations(maxProjectionIterations); - CompositeStopingCriteria stop = new CompositeStopingCriteria(); - stop.add(new NormalizedProjectedGradientL2Norm(stoppingPrecision)); - stop.add(new NormalizedValueDifference(stoppingPrecision)); - boolean succeed = optimizer.optimize(objective, stats, stop); - - System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1)); - //System.out.println("Solution: " + objective.parameters); - if (!succeed) - System.out.println("Failed to optimize"); - //System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - - //lambda = objective.getParameters(); - llh = objective.primal(); - - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - for (int t = 0; t < numTags; t++) - { - double p = objective.q.get(i).get(j).get(t); - priorCounts[i][t] += e.getCount() * p; - TIntArrayList tokens = e.getContext(); - for (int k = 0; k < tokens.size(); ++k) - emissionsCounts[k][t][tokens.get(k)] += e.getCount() * p; - } - } - } - } - else - { - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - double probs[] = posterior(i, e); - double z = normalise(probs); - llh += log(z) * e.getCount(); - - TIntArrayList tokens = e.getContext(); - for (int t = 0; t < numTags; ++t) - { - priorCounts[i][t] += e.getCount() * probs[t]; - for (int k = 0; k < tokens.size(); ++k) - emissionsCounts[j][t][tokens.get(k)] += e.getCount() * probs[t]; - } - } - } - } - - // M-step: normalise - for (double[][] emissionTW : emissionsCounts) - for (double[] emissionW : emissionTW) - normalise(emissionW); - - for (double[] priorTag : priorCounts) - normalise(priorTag); - - emissions = emissionsCounts; - prior = priorCounts; - - System.out.println("Iteration " + iteration + " llh " + llh); - -// if (llh - lastLlh < 1e-4) -// break; -// else -// lastLlh = llh; - } - } - - static double normalise(double probs[]) - { - double z = 0; - for (double p : probs) - z += p; - for (int i = 0; i < probs.length; ++i) - probs[i] /= z; - return z; - } - - void randomise(double probs[]) - { - double z = 0; - for (int i = 0; i < probs.length; ++i) - { - probs[i] = 10 + rng.nextDouble(); - z += probs[i]; - } - - for (int i = 0; i < probs.length; ++i) - probs[i] /= z; - } - - static int argmax(double probs[]) - { - double m = Double.NEGATIVE_INFINITY; - int mi = -1; - for (int i = 0; i < probs.length; ++i) - { - if (probs[i] > m) - { - m = probs[i]; - mi = i; - } - } - return mi; - } - - double[] posterior(int phraseId, Corpus.Edge e) // unnormalised - { - double probs[] = new double[numTags]; - TIntArrayList tokens = e.getContext(); - for (int t = 0; t < numTags; ++t) - { - probs[t] = prior[phraseId][t]; - for (int k = 0; k < tokens.size(); ++k) - probs[t] *= emissions[k][t][tokens.get(k)]; - } - return probs; - } - - void displayPosterior() - { - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - for (Corpus.Edge e: edges) - { - double probs[] = posterior(i, e); - normalise(probs); - - // emit phrase - System.out.print(e.getPhraseString()); - System.out.print("\t"); - System.out.print(e.getContextString()); - System.out.print("||| C=" + e.getCount() + " |||"); - - int t = argmax(probs); - System.out.print(" " + t + " ||| " + probs[t]); - // for (int t = 0; t < numTags; ++t) - // System.out.print(" " + probs[t]); - System.out.println(); - } - } - } - - public static void main(String[] args) - { - assert (args.length >= 2); - try - { - Corpus corpus = Corpus.readFromFile(new FileReader(new File(args[0]))); - PhraseContextModel model = new PhraseContextModel(corpus, Integer.parseInt(args[1])); - model.expectationMaximisation(Integer.parseInt(args[2])); - model.displayPosterior(); - } - catch (IOException e) - { - System.out.println("Failed to read input file: " + args[0]); - e.printStackTrace(); - } - } - - class EStepDualObjective extends ProjectedObjective - { - List> conditionals; // phrase id x context # x tag - precomputed - List> q; // ditto, but including exp(-lambda) terms - double objective = 0; // log(z) - // Objective.gradient = d log(z) / d lambda = E_q[phi] - double llh = 0; - - public EStepDualObjective() - { - super(); - // compute conditionals p(context, tag | phrase) for all training instances - conditionals = new ArrayList>(training.getNumPhrases()); - q = new ArrayList>(training.getNumPhrases()); - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - - conditionals.add(new ArrayList(edges.size())); - q.add(new ArrayList(edges.size())); - - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - double probs[] = posterior(i, e); - double z = normalise(probs); - llh += log(z) * e.getCount(); - conditionals.get(i).add(new TDoubleArrayList(probs)); - q.get(i).add(new TDoubleArrayList(probs)); - } - } - - gradient = new double[training.getNumEdges()*numTags]; - setInitialParameters(lambda); - computeObjectiveAndGradient(); - } - - @Override - public double[] projectPoint(double[] point) - { - SimplexProjection p = new SimplexProjection(constraintScale); - - double[] newPoint = point.clone(); - int edgeIndex = 0; - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - - for (int t = 0; t < numTags; t++) - { - double[] subPoint = new double[edges.size()]; - for (int j = 0; j < edges.size(); ++j) - subPoint[j] = point[edgeIndex+j*numTags+t]; - - p.project(subPoint); - for (int j = 0; j < edges.size(); ++j) - newPoint[edgeIndex+j*numTags+t] = subPoint[j]; - } - - edgeIndex += edges.size() * numTags; - } -// System.out.println("Proj from: " + Arrays.toString(point)); -// System.out.println("Proj to: " + Arrays.toString(newPoint)); - return newPoint; - } - - @Override - public void setParameters(double[] params) - { - super.setParameters(params); - computeObjectiveAndGradient(); - } - - @Override - public double[] getGradient() - { - gradientCalls += 1; - return gradient; - } - - @Override - public double getValue() - { - functionCalls += 1; - return objective; - } - - public void computeObjectiveAndGradient() - { - int edgeIndex = 0; - objective = 0; - Arrays.fill(gradient, 0); - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - - double z = 0; - for (int t = 0; t < numTags; t++) - { - double v = conditionals.get(i).get(j).get(t) * exp(-parameters[edgeIndex+t]); - q.get(i).get(j).set(t, v); - z += v; - } - objective += log(z) * e.getCount(); - - for (int t = 0; t < numTags; t++) - { - double v = q.get(i).get(j).get(t) / z; - q.get(i).get(j).set(t, v); - gradient[edgeIndex+t] -= e.getCount() * v; - } - - edgeIndex += numTags; - } - } -// System.out.println("computeObjectiveAndGradient logz=" + objective); -// System.out.println("lambda= " + Arrays.toString(parameters)); -// System.out.println("gradient=" + Arrays.toString(gradient)); - } - - public String toString() - { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getCanonicalName()).append(" with "); - sb.append(parameters.length).append(" parameters and "); - sb.append(training.getNumPhrases() * numTags).append(" constraints"); - return sb.toString(); - } - - double primal() - { - // primal = llh + KL(q||p) + scale * sum_pt max_c E_q[phi_pct] - // kl = sum_Y q(Y) log q(Y) / p(Y|X) - // = sum_Y q(Y) { -lambda . phi(Y) - log Z } - // = -log Z - lambda . E_q[phi] - // = -objective + lambda . gradient - - double kl = -objective + MathUtils.dotProduct(parameters, gradient); - double l1lmax = 0; - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List edges = training.getEdgesForPhrase(i); - for (int t = 0; t < numTags; t++) - { - double lmax = Double.NEGATIVE_INFINITY; - for (int j = 0; j < edges.size(); ++j) - lmax = max(lmax, q.get(i).get(j).get(t)); - l1lmax += lmax; - } - } - - return llh + kl + constraintScale * l1lmax; - } - } -} diff --git a/gi/posterior-regularisation/README b/gi/posterior-regularisation/README deleted file mode 100644 index a3d54ffc..00000000 --- a/gi/posterior-regularisation/README +++ /dev/null @@ -1,3 +0,0 @@ - 557 ./cdec_extools/extractor -i btec/split.zh-en.al -c 500000 -L 12 -C | sort -t $'\t' -k 1 | ./cdec_extools/mr_stripe_rule_reduce > btec.concordance - 559 wc -l btec.concordance - 588 cat btec.concordance | sed 's/.* //' | awk '{ for (i=1; i < NF; i++) { x=substr($i, 1, 2); if (x == "C=") printf "\n"; else if (x != "||") printf "%s ", $i; }; printf "\n"; }' | sort | uniq | wc -l diff --git a/gi/posterior-regularisation/alphabet.hh b/gi/posterior-regularisation/alphabet.hh deleted file mode 100644 index 1db928da..00000000 --- a/gi/posterior-regularisation/alphabet.hh +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef _alphabet_hh -#define _alphabet_hh - -#include -#include -#include -#include -#include - -// Alphabet: indexes a set of types -template -class Alphabet: protected std::map -{ -public: - Alphabet() {}; - - bool empty() const { return std::map::empty(); } - int size() const { return std::map::size(); } - - int operator[](const T &k) const - { - typename std::map::const_iterator cit = find(k); - if (cit != std::map::end()) - return cit->second; - else - return -1; - } - - int lookup(const T &k) const { return (*this)[k]; } - - int insert(const T &k) - { - int sz = size(); - assert((unsigned) sz == _items.size()); - - std::pair::iterator, bool> - ins = std::map::insert(make_pair(k, sz)); - - if (ins.second) - _items.push_back(k); - - return ins.first->second; - } - - const T &type(int i) const - { - assert(i >= 0); - assert(i < size()); - return _items[i]; - } - - std::ostream &display(std::ostream &out, int i) const - { - return out << type(i); - } - -private: - std::vector _items; -}; - -#endif diff --git a/gi/posterior-regularisation/canned.concordance b/gi/posterior-regularisation/canned.concordance deleted file mode 100644 index 710973ff..00000000 --- a/gi/posterior-regularisation/canned.concordance +++ /dev/null @@ -1,4 +0,0 @@ -a 0 0 0 0 ||| C=1 ||| 1 1 1 1 ||| C=1 ||| 2 2 2 2 ||| C=1 -b 0 0 0 0 ||| C=1 ||| 1 1 1 1 ||| C=1 -c 2 2 2 2 ||| C=1 ||| 4 4 4 4 ||| C=1 ||| 5 5 5 5 ||| C=1 -d 4 4 4 4 ||| C=1 ||| 5 5 5 5 ||| C=1 diff --git a/gi/posterior-regularisation/em.cc b/gi/posterior-regularisation/em.cc deleted file mode 100644 index f6c9fd68..00000000 --- a/gi/posterior-regularisation/em.cc +++ /dev/null @@ -1,830 +0,0 @@ -// Input of the form: -// " the phantom of the opera " tickets for tonight ? ||| C=1 ||| seats for ? ||| C=1 ||| i see ? ||| C=1 -// phrase TAB [context]+ -// where context = phrase ||| C=... which are separated by ||| - -// Model parameterised as follows: -// - each phrase, p, is allocated a latent state, t -// - this is used to generate the contexts, c -// - each context is generated using 4 independent multinomials, one for each position LL, L, R, RR - -// Training with EM: -// - e-step is estimating P(t|p,c) for all x,c -// - m-step is estimating model parameters P(p,c,t) = P(t) P(p|t) P(c|t) - -// Sexing it up: -// - constrain the posteriors P(t|c) and P(t|p) to have few high-magnitude entries -// - improve the generation of phrase internals, e.g., generate edge words from -// different distribution to central words - -#include "alphabet.hh" -#include "log_add.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; -using namespace std::tr1; - -const int numTags = 5; -const int numIterations = 100; -const bool posterior_regularisation = true; -const double PHRASE_VIOLATION_WEIGHT = 10; -const double CONTEXT_VIOLATION_WEIGHT = 0; -const bool includePhraseProb = false; - -// Data structures: -Alphabet lexicon; -typedef vector Phrase; -typedef tuple Context; -Alphabet phrases; -Alphabet contexts; - -typedef map ContextCounts; -typedef map PhraseCounts; -typedef map PhraseToContextCounts; -typedef map ContextToPhraseCounts; - -PhraseToContextCounts concordancePhraseToContexts; -ContextToPhraseCounts concordanceContextToPhrases; - -typedef vector Dist; -typedef vector ConditionalDist; -Dist prior; // class -> P(class) -vector probCtx; // word -> class -> P(word | class), for each position of context word -ConditionalDist probPhrase; // class -> P(word | class) -Dist probPhraseLength; // class -> P(length | class) expressed as geometric distribution parameter - -mt19937 randomGenerator((size_t) time(NULL)); -uniform_real uniDist(0.0, 1e-1); -variate_generator< mt19937, uniform_real > rng(randomGenerator, uniDist); - -void addRandomNoise(Dist &d); -void normalise(Dist &d); -void addTo(Dist &d, const Dist &e); -int argmax(const Dist &d); - -map > lambda_indices; - -Dist conditional_probs(const Phrase &phrase, const Context &context, double *normalisation = 0); -template -Dist -penalised_conditionals(const Phrase &phrase, const Context &context, - const T &lambda, double *normalisation); -//Dist penalised_conditionals(const Phrase &phrase, const Context &context, const double *lambda, double *normalisation = 0); -double penalised_log_likelihood(int n, const double *lambda, double *gradient, void *data); -void optimise_lambda(double delta, double gamma, vector &lambda); -double expected_violation_phrases(const double *lambda); -double expected_violation_contexts(const double *lambda); -double primal_kl_divergence(const double *lambda); -double dual(const double *lambda); -void print_primal_dual(const double *lambda, double delta, double gamma); - -ostream &operator<<(ostream &, const Phrase &); -ostream &operator<<(ostream &, const Context &); -ostream &operator<<(ostream &, const Dist &); -ostream &operator<<(ostream &, const ConditionalDist &); - -int -main(int argc, char *argv[]) -{ - randomGenerator.seed(time(NULL)); - - int edges = 0; - istream &input = cin; - while (input.good()) - { - // read the phrase - string phraseString; - Phrase phrase; - getline(input, phraseString, '\t'); - istringstream pinput(phraseString); - string token; - while (pinput >> token) - phrase.push_back(lexicon.insert(token)); - int phraseId = phrases.insert(phrase); - - // read the rest, storing each context - string remainder; - getline(input, remainder, '\n'); - istringstream rinput(remainder); - Context context(-1, -1, -1, -1); - int index = 0; - while (rinput >> token) - { - if (token != "|||" && token != "") - { - if (index < 4) - { - // eugh! damn templates - switch (index) - { - case 0: get<0>(context) = lexicon.insert(token); break; - case 1: get<1>(context) = lexicon.insert(token); break; - case 2: get<2>(context) = lexicon.insert(token); break; - case 3: get<3>(context) = lexicon.insert(token); break; - default: assert(false); - } - index += 1; - } - else if (token.find("C=") == 0) - { - int contextId = contexts.insert(context); - int count = atoi(token.substr(strlen("C=")).c_str()); - concordancePhraseToContexts[phraseId][contextId] += count; - concordanceContextToPhrases[contextId][phraseId] += count; - index = 0; - context = Context(-1, -1, -1, -1); - edges += 1; - } - } - } - - // trigger EOF - input >> ws; - } - - cout << "Read in " << phrases.size() << " phrases" - << " and " << contexts.size() << " contexts" - << " and " << edges << " edges" - << " and " << lexicon.size() << " word types\n"; - - // FIXME: filter out low count phrases and low count contexts (based on individual words?) - // now populate model parameters with uniform + random noise - prior.resize(numTags, 1.0); - addRandomNoise(prior); - normalise(prior); - - probCtx.resize(4, ConditionalDist(numTags, Dist(lexicon.size(), 1.0))); - if (includePhraseProb) - probPhrase.resize(numTags, Dist(lexicon.size(), 1.0)); - for (int t = 0; t < numTags; ++t) - { - for (int j = 0; j < 4; ++j) - { - addRandomNoise(probCtx[j][t]); - normalise(probCtx[j][t]); - } - if (includePhraseProb) - { - addRandomNoise(probPhrase[t]); - normalise(probPhrase[t]); - } - } - if (includePhraseProb) - { - probPhraseLength.resize(numTags, 0.5); // geometric distribution p=0.5 - addRandomNoise(probPhraseLength); - } - - cout << "\tprior: " << prior << "\n"; - //cout << "\tcontext: " << probCtx << "\n"; - //cout << "\tphrase: " << probPhrase << "\n"; - //cout << "\tphraseLen: " << probPhraseLength << endl; - - vector lambda; - - // now do EM training - for (int iteration = 0; iteration < numIterations; ++iteration) - { - cout << "EM iteration " << iteration << endl; - - if (posterior_regularisation) - optimise_lambda(PHRASE_VIOLATION_WEIGHT, CONTEXT_VIOLATION_WEIGHT, lambda); - //cout << "\tlambda " << lambda << endl; - - Dist countsPrior(numTags, 0.0); - vector countsCtx(4, ConditionalDist(numTags, Dist(lexicon.size(), 1e-10))); - ConditionalDist countsPhrase(numTags, Dist(lexicon.size(), 1e-10)); - Dist countsPhraseLength(numTags, 0.0); - Dist nPhrases(numTags, 0.0); - - double llh = 0; - for (PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.begin(); - pcit != concordancePhraseToContexts.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - - // e-step: estimate latent class probs; compile (class,word) stats for m-step - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - - double z = 0; - Dist tagCounts; - if (!posterior_regularisation) - tagCounts = conditional_probs(phrase, context, &z); - else - tagCounts = penalised_conditionals(phrase, context, lambda, &z); - - llh += log(z) * ccit->second; - addTo(countsPrior, tagCounts); // FIXME: times ccit->secon - - for (int t = 0; t < numTags; ++t) - { - for (int j = 0; j < 4; ++j) - countsCtx[j][t][get<0>(context)] += tagCounts[t] * ccit->second; - - if (includePhraseProb) - { - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - countsPhrase[t][*pit] += tagCounts[t] * ccit->second; - countsPhraseLength[t] += phrase.size() * tagCounts[t] * ccit->second; - nPhrases[t] += tagCounts[t] * ccit->second; - } - } - } - } - - cout << "M-step\n"; - - // m-step: normalise prior and (class,word) stats and assign to model parameters - normalise(countsPrior); - prior = countsPrior; - for (int t = 0; t < numTags; ++t) - { - //cout << "\t\tt " << t << " prior " << countsPrior[t] << "\n"; - for (int j = 0; j < 4; ++j) - normalise(countsCtx[j][t]); - if (includePhraseProb) - { - normalise(countsPhrase[t]); - countsPhraseLength[t] = nPhrases[t] / countsPhraseLength[t]; - } - } - probCtx = countsCtx; - if (includePhraseProb) - { - probPhrase = countsPhrase; - probPhraseLength = countsPhraseLength; - } - - double *larray = new double[lambda.size()]; - copy(lambda.begin(), lambda.end(), larray); - print_primal_dual(larray, PHRASE_VIOLATION_WEIGHT, CONTEXT_VIOLATION_WEIGHT); - delete [] larray; - - //cout << "\tllh " << llh << endl; - //cout << "\tprior: " << prior << "\n"; - //cout << "\tcontext: " << probCtx << "\n"; - //cout << "\tphrase: " << probPhrase << "\n"; - //cout << "\tphraseLen: " << probPhraseLength << "\n"; - } - - // output class membership - for (PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.begin(); - pcit != concordancePhraseToContexts.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - Dist tagCounts = conditional_probs(phrase, context, 0); - cout << phrase << " ||| " << context << " ||| " << argmax(tagCounts) << "\n"; - } - } - - return 0; -} - -void addRandomNoise(Dist &d) -{ - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - *dit += rng(); -} - -void normalise(Dist &d) -{ - double z = 0; - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - z += *dit; - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - *dit /= z; -} - -void addTo(Dist &d, const Dist &e) -{ - assert(d.size() == e.size()); - for (int i = 0; i < (int) d.size(); ++i) - d[i] += e[i]; -} - -int argmax(const Dist &d) -{ - double best = d[0]; - int index = 0; - for (int i = 1; i < (int) d.size(); ++i) - { - if (d[i] > best) - { - best = d[i]; - index = i; - } - } - return index; -} - -ostream &operator<<(ostream &out, const Phrase &phrase) -{ - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - lexicon.display(((pit == phrase.begin()) ? out : out << " "), *pit); - return out; -} - -ostream &operator<<(ostream &out, const Context &context) -{ - lexicon.display(out, get<0>(context)); - lexicon.display(out << " ", get<1>(context)); - lexicon.display(out << " ", get<2>(context)); - lexicon.display(out << " ", get<3>(context)); - return out; -} - -ostream &operator<<(ostream &out, const Dist &dist) -{ - for (Dist::const_iterator dit = dist.begin(); dit != dist.end(); ++dit) - out << ((dit == dist.begin()) ? "" : " ") << *dit; - return out; -} - -ostream &operator<<(ostream &out, const ConditionalDist &dist) -{ - for (ConditionalDist::const_iterator dit = dist.begin(); dit != dist.end(); ++dit) - out << ((dit == dist.begin()) ? "" : "; ") << *dit; - return out; -} - -// FIXME: slow - just use the phrase index, context index to do the mapping -// (n.b. it's a sparse setup, not just equal to 3d array index) -int -lambda_index(const Phrase &phrase, const Context &context, int tag) -{ - return lambda_indices[phrase][context] + tag; -} - -template -Dist -penalised_conditionals(const Phrase &phrase, const Context &context, - const T &lambda, double *normalisation) -{ - Dist d = conditional_probs(phrase, context, 0); - - double z = 0; - for (int t = 0; t < numTags; ++t) - { - d[t] *= exp(-lambda[lambda_index(phrase, context, t)]); - z += d[t]; - } - - if (normalisation) - *normalisation = z; - - for (int t = 0; t < numTags; ++t) - d[t] /= z; - - return d; -} - -Dist -conditional_probs(const Phrase &phrase, const Context &context, double *normalisation) -{ - Dist tagCounts(numTags, 0.0); - double z = 0; - for (int t = 0; t < numTags; ++t) - { - double prob = prior[t]; - prob *= (probCtx[0][t][get<0>(context)] * probCtx[1][t][get<1>(context)] * - probCtx[2][t][get<2>(context)] * probCtx[3][t][get<3>(context)]); - - if (includePhraseProb) - { - prob *= pow(1 - probPhraseLength[t], phrase.size() - 1) * probPhraseLength[t]; - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - prob *= probPhrase[t][*pit]; - } - - tagCounts[t] = prob; - z += prob; - } - if (normalisation) - *normalisation = z; - - for (int t = 0; t < numTags; ++t) - tagCounts[t] /= z; - - return tagCounts; -} - -double -penalised_log_likelihood(int n, const double *lambda, double *grad, void *) -{ - // return log Z(lambda, theta) over the corpus - // where theta are the global parameters (prior, probCtx*, probPhrase*) - // and lambda are lagrange multipliers for the posterior sparsity constraints - // - // this is formulated as: - // f = log Z(lambda) = sum_i log ( sum_i p_theta(t_i|p_i,c_i) exp [-lambda_{t_i,p_i,c_i}] ) - // where i indexes the training examples - specifying the (p, c) pair (which may occur with count > 1) - // - // with derivative: - // f'_{tpc} = frac { - count(t,p,c) p_theta(t|p,c) exp (-lambda_{t,p,c}) } - // { sum_t' p_theta(t'|p,c) exp (-lambda_{t',p,c}) } - - //cout << "penalised_log_likelihood with lambda "; - //copy(lambda, lambda+n, ostream_iterator(cout, " ")); - //cout << "\n"; - - double f = 0; - if (grad) - { - for (int i = 0; i < n; ++i) - grad[i] = 0.0; - } - - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double z = 0; - Dist scores = penalised_conditionals(phrase, context, lambda, &z); - - f += ccit->second * log(z); - //cout << "\tphrase: " << phrase << " context: " << context << " count: " << ccit->second << " z " << z << endl; - //cout << "\t\tscores: " << scores << "\n"; - - if (grad) - { - for (int t = 0; t < numTags; ++t) - { - int i = lambda_index(phrase, context, t); // FIXME: redundant lookups - assert(grad[i] == 0.0); - grad[i] = - ccit->second * scores[t]; - } - } - } - } - - //cout << "penalised_log_likelihood returning " << f; - //if (grad) - //{ - //cout << "\ngradient: "; - //copy(grad, grad+n, ostream_iterator(cout, " ")); - //} - //cout << "\n"; - - return f; -} - -typedef struct -{ - // one of p or c should be set to -1, in which case it will be marginalised out - // i.e. sum_p' lambda_{p'ct} <= threshold - // or sum_c' lambda_{pc't} <= threshold - int p, c, t, threshold; -} constraint_data; - -double -constraint_and_gradient(int n, const double *lambda, double *grad, void *data) -{ - constraint_data *d = (constraint_data *) data; - assert(d->t >= 0); - assert(d->threshold >= 0); - - //cout << "constraint_and_gradient: t " << d->t << " p " << d->p << " c " << d->c << " tau " << d->threshold << endl; - //cout << "\tlambda "; - //copy(lambda, lambda+n, ostream_iterator(cout, " ")); - //cout << "\n"; - - // FIXME: it's crazy to use a dense gradient here => will only have a handful of non-zero entries - if (grad) - { - for (int i = 0; i < n; ++i) - grad[i] = 0.0; - } - - //cout << "constraint_and_gradient: " << d->p << "; " << d->c << "; " << d->t << "; " << d->threshold << endl; - - if (d->p >= 0) - { - assert(d->c < 0); - // sum_c lambda_pct <= delta [a.k.a. threshold] - // => sum_c lambda_pct - delta <= 0 - // derivative_pct = { 1, if p and t match; 0, otherwise } - - double val = -d->threshold; - - const Phrase &phrase = phrases.type(d->p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(d->p); - assert(pcit != concordancePhraseToContexts.end()); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - int i = lambda_index(phrase, context, d->t); - val += lambda[i]; - if (grad) grad[i] = 1; - } - //cout << "\treturning " << val << endl; - - return val; - } - else - { - assert(d->c >= 0); - assert(d->p < 0); - // sum_p lambda_pct <= gamma [a.k.a. threshold] - // => sum_p lambda_pct - gamma <= 0 - // derivative_pct = { 1, if c and t match; 0, otherwise } - - double val = -d->threshold; - - const Context &context = contexts.type(d->c); - ContextToPhraseCounts::iterator cpit = concordanceContextToPhrases.find(d->c); - assert(cpit != concordanceContextToPhrases.end()); - for (PhraseCounts::iterator pcit = cpit->second.begin(); - pcit != cpit->second.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - int i = lambda_index(phrase, context, d->t); - val += lambda[i]; - if (grad) grad[i] = 1; - } - //cout << "\treturning " << val << endl; - - return val; - } -} - -void -optimise_lambda(double delta, double gamma, vector &lambdav) -{ - int num_lambdas = lambdav.size(); - if (lambda_indices.empty() || lambdav.empty()) - { - lambda_indices.clear(); - lambdav.clear(); - - int i = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - lambda_indices[phrase][context] = i; - i += numTags; - } - } - num_lambdas = i; - lambdav.resize(num_lambdas); - } - //cout << "optimise_lambda: #langrange multipliers " << num_lambdas << endl; - - // FIXME: better to work with an implicit representation to save memory usage - int num_constraints = (((delta > 0) ? phrases.size() : 0) + ((gamma > 0) ? contexts.size() : 0)) * numTags; - //cout << "optimise_lambda: #constraints " << num_constraints << endl; - constraint_data *data = new constraint_data[num_constraints]; - int i = 0; - if (delta > 0) - { - for (int p = 0; p < phrases.size(); ++p) - { - for (int t = 0; t < numTags; ++t, ++i) - { - constraint_data &d = data[i]; - d.p = p; - d.c = -1; - d.t = t; - d.threshold = delta; - } - } - } - - if (gamma > 0) - { - for (int c = 0; c < contexts.size(); ++c) - { - for (int t = 0; t < numTags; ++t, ++i) - { - constraint_data &d = data[i]; - d.p = -1; - d.c = c; - d.t = t; - d.threshold = gamma; - } - } - } - assert(i == num_constraints); - - double lambda[num_lambdas]; - double lb[num_lambdas], ub[num_lambdas]; - for (i = 0; i < num_lambdas; ++i) - { - lambda[i] = lambdav[i]; // starting value - lb[i] = 0; // lower bound - if (delta <= 0) // upper bound - ub[i] = gamma; - else if (gamma <= 0) - ub[i] = delta; - else - assert(false); - } - - //print_primal_dual(lambda, delta, gamma); - - double minf; - int error_code = nlopt_minimize_constrained(NLOPT_LN_COBYLA, num_lambdas, penalised_log_likelihood, NULL, - num_constraints, constraint_and_gradient, data, sizeof(constraint_data), - lb, ub, lambda, &minf, -HUGE_VAL, 0.0, 0.0, 1e-4, NULL, 0, 0.0); - //cout << "optimise error code " << error_code << endl; - - //print_primal_dual(lambda, delta, gamma); - - delete [] data; - - if (error_code < 0) - cout << "WARNING: optimisation failed with error code: " << error_code << endl; - //else - //{ - //cout << "success; minf " << minf << endl; - //print_primal_dual(lambda, delta, gamma); - //} - - lambdav = vector(&lambda[0], &lambda[0] + num_lambdas); -} - -// FIXME: inefficient - cache the scores -double -expected_violation_phrases(const double *lambda) -{ - // sum_pt max_c E_q[phi_pct] - double violation = 0; - - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - - for (int t = 0; t < numTags; ++t) - { - double best = 0; - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - Dist scores = penalised_conditionals(phrase, context, lambda, 0); - best = max(best, scores[t]); - } - violation += best; - } - } - - return violation; -} - -// FIXME: inefficient - cache the scores -double -expected_violation_contexts(const double *lambda) -{ - // sum_ct max_p E_q[phi_pct] - double violation = 0; - - for (int c = 0; c < contexts.size(); ++c) - { - const Context &context = contexts.type(c); - ContextToPhraseCounts::iterator cpit = concordanceContextToPhrases.find(c); - - for (int t = 0; t < numTags; ++t) - { - double best = 0; - for (PhraseCounts::iterator pit = cpit->second.begin(); - pit != cpit->second.end(); ++pit) - { - const Phrase &phrase = phrases.type(pit->first); - Dist scores = penalised_conditionals(phrase, context, lambda, 0); - best = max(best, scores[t]); - } - violation += best; - } - } - - return violation; -} - -// FIXME: possibly inefficient -double -primal_likelihood() // FIXME: primal evaluation needs to use lambda and calculate l1linf terms -{ - double llh = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double z = 0; - Dist scores = conditional_probs(phrase, context, &z); - llh += ccit->second * log(z); - } - } - return llh; -} - -// FIXME: inefficient - cache the scores -double -primal_kl_divergence(const double *lambda) -{ - // return KL(q || p) = sum_y q(y) { log q(y) - log p(y | x) } - // = sum_y q(y) { log p(y | x) - lambda . phi(x, y) - log Z - log p(y | x) } - // = sum_y q(y) { - lambda . phi(x, y) } - log Z - // and q(y) factors with each edge, ditto for Z - - double feature_sum = 0, log_z = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - - double local_z = 0; - double local_f = 0; - Dist d = conditional_probs(phrase, context, 0); - for (int t = 0; t < numTags; ++t) - { - int i = lambda_index(phrase, context, t); - double s = d[t] * exp(-lambda[i]); - local_f += lambda[i] * s; - local_z += s; - } - - log_z += ccit->second * log(local_z); - feature_sum += ccit->second * (local_f / local_z); - } - } - - return -feature_sum - log_z; -} - -// FIXME: inefficient - cache the scores -double -dual(const double *lambda) -{ - // return log(Z) = - log { sum_y p(y | x) exp( - lambda . phi(x, y) } - // n.b. have flipped the sign as we're minimising - - double z = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double lz = 0; - Dist scores = penalised_conditionals(phrase, context, lambda, &z); - z += lz * ccit->second; - } - } - return log(z); -} - -void -print_primal_dual(const double *lambda, double delta, double gamma) -{ - double likelihood = primal_likelihood(); - double kl = primal_kl_divergence(lambda); - double sum_pt = expected_violation_phrases(lambda); - double sum_ct = expected_violation_contexts(lambda); - //double d = dual(lambda); - - cout << "\tllh=" << likelihood - << " kl=" << kl - << " violations phrases=" << sum_pt - << " contexts=" << sum_ct - //<< " primal=" << (kl + delta * sum_pt + gamma * sum_ct) - //<< " dual=" << d - << " objective=" << (likelihood - kl + delta * sum_pt + gamma * sum_ct) - << endl; -} diff --git a/gi/posterior-regularisation/invert.hh b/gi/posterior-regularisation/invert.hh deleted file mode 100644 index d06356e9..00000000 --- a/gi/posterior-regularisation/invert.hh +++ /dev/null @@ -1,45 +0,0 @@ -// The following code inverts the matrix input using LU-decomposition with -// backsubstitution of unit vectors. Reference: Numerical Recipies in C, 2nd -// ed., by Press, Teukolsky, Vetterling & Flannery. -// Code written by Fredrik Orderud. -// http://www.crystalclearsoftware.com/cgi-bin/boost_wiki/wiki.pl?LU_Matrix_Inversion - -#ifndef INVERT_MATRIX_HPP -#define INVERT_MATRIX_HPP - -// REMEMBER to update "lu.hpp" header includes from boost-CVS -#include -#include -#include -#include -#include -#include - -namespace ublas = boost::numeric::ublas; - -/* Matrix inversion routine. - Uses lu_factorize and lu_substitute in uBLAS to invert a matrix */ -template -bool invert_matrix(const ublas::matrix& input, ublas::matrix& inverse) -{ - using namespace boost::numeric::ublas; - typedef permutation_matrix pmatrix; - // create a working copy of the input - matrix A(input); - // create a permutation matrix for the LU-factorization - pmatrix pm(A.size1()); - - // perform LU-factorization - int res = lu_factorize(A,pm); - if( res != 0 ) return false; - - // create identity matrix of "inverse" - inverse.assign(ublas::identity_matrix(A.size1())); - - // backsubstitute to get the inverse - lu_substitute(A, pm, inverse); - - return true; -} - -#endif //INVERT_MATRIX_HPP diff --git a/gi/posterior-regularisation/linesearch.py b/gi/posterior-regularisation/linesearch.py deleted file mode 100644 index 5a3f2e9c..00000000 --- a/gi/posterior-regularisation/linesearch.py +++ /dev/null @@ -1,58 +0,0 @@ -## Automatically adapted for scipy Oct 07, 2005 by convertcode.py - -from scipy.optimize import minpack2 -import numpy - -import __builtin__ -pymin = __builtin__.min - -def line_search(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, - args=(), c1=1e-4, c2=0.9, amax=50): - - fc = 0 - gc = 0 - phi0 = old_fval - derphi0 = numpy.dot(gfk,pk) - alpha1 = pymin(1.0,1.01*2*(phi0-old_old_fval)/derphi0) - # trevor: added this test - alpha1 = pymin(alpha1,amax) - - if isinstance(myfprime,type(())): - eps = myfprime[1] - fprime = myfprime[0] - newargs = (f,eps) + args - gradient = False - else: - fprime = myfprime - newargs = args - gradient = True - - xtol = 1e-14 - amin = 1e-8 - isave = numpy.zeros((2,), numpy.intc) - dsave = numpy.zeros((13,), float) - task = 'START' - fval = old_fval - gval = gfk - - while 1: - stp,fval,derphi,task = minpack2.dcsrch(alpha1, phi0, derphi0, c1, c2, - xtol, task, amin, amax,isave,dsave) - #print 'minpack2.dcsrch', alpha1, phi0, derphi0, c1, c2, xtol, task, amin, amax,isave,dsave - #print 'returns', stp,fval,derphi,task - - if task[:2] == 'FG': - alpha1 = stp - fval = f(xk+stp*pk,*args) - fc += 1 - gval = fprime(xk+stp*pk,*newargs) - if gradient: gc += 1 - else: fc += len(xk) + 1 - phi0 = fval - derphi0 = numpy.dot(gval,pk) - else: - break - - if task[:5] == 'ERROR' or task[1:4] == 'WARN': - stp = None # failed - return stp, fc, gc, fval, old_fval, gval diff --git a/gi/posterior-regularisation/log_add.hh b/gi/posterior-regularisation/log_add.hh deleted file mode 100644 index e0620c5a..00000000 --- a/gi/posterior-regularisation/log_add.hh +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef log_add_hh -#define log_add_hh - -#include -#include -#include -#include - -template -struct Log -{ - static T zero() { return -std::numeric_limits::infinity(); } - - static T add(T l1, T l2) - { - if (l1 == zero()) return l2; - if (l1 > l2) - return l1 + std::log(1 + exp(l2 - l1)); - else - return l2 + std::log(1 + exp(l1 - l2)); - } - - static T subtract(T l1, T l2) - { - //std::assert(l1 >= l2); - return l1 + log(1 - exp(l2 - l1)); - } -}; - -#endif diff --git a/gi/posterior-regularisation/prjava.jar b/gi/posterior-regularisation/prjava.jar deleted file mode 120000 index da8bf761..00000000 --- a/gi/posterior-regularisation/prjava.jar +++ /dev/null @@ -1 +0,0 @@ -prjava/prjava-20100708.jar \ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/Makefile b/gi/posterior-regularisation/prjava/Makefile deleted file mode 100755 index bd3bfca0..00000000 --- a/gi/posterior-regularisation/prjava/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -all: - ant dist - -check: - echo no tests - -clean: - ant clean diff --git a/gi/posterior-regularisation/prjava/build.xml b/gi/posterior-regularisation/prjava/build.xml deleted file mode 100644 index 7222b3c8..00000000 --- a/gi/posterior-regularisation/prjava/build.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar b/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar deleted file mode 100644 index 43b4b369..00000000 Binary files a/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar and /dev/null differ diff --git a/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar b/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar deleted file mode 100644 index 56373621..00000000 Binary files a/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar and /dev/null differ diff --git a/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar b/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar deleted file mode 100644 index 3e59fbf3..00000000 Binary files a/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar and /dev/null differ diff --git a/gi/posterior-regularisation/prjava/src/arr/F.java b/gi/posterior-regularisation/prjava/src/arr/F.java deleted file mode 100644 index be0a6ed6..00000000 --- a/gi/posterior-regularisation/prjava/src/arr/F.java +++ /dev/null @@ -1,99 +0,0 @@ -package arr; - -import java.util.Arrays; -import java.util.Random; - -public class F { - public static Random rng = new Random(); - - public static void randomise(double probs[]) - { - randomise(probs, true); - } - - public static void randomise(double probs[], boolean normalise) - { - double z = 0; - for (int i = 0; i < probs.length; ++i) - { - probs[i] = 10 + rng.nextDouble(); - if (normalise) - z += probs[i]; - } - - if (normalise) - for (int i = 0; i < probs.length; ++i) - probs[i] /= z; - } - - public static void uniform(double probs[]) - { - for (int i = 0; i < probs.length; ++i) - probs[i] = 1.0 / probs.length; - } - - public static void l1normalize(double [] a){ - double sum=0; - for(int i=0;i m) - { - m = probs[i]; - mi = i; - } - } - return mi; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/data/Corpus.java b/gi/posterior-regularisation/prjava/src/data/Corpus.java deleted file mode 100644 index 425ede11..00000000 --- a/gi/posterior-regularisation/prjava/src/data/Corpus.java +++ /dev/null @@ -1,233 +0,0 @@ -package data; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Scanner; - -public class Corpus { - - public static final String alphaFilename="../posdata/corpus.alphabet"; - public static final String tagalphaFilename="../posdata/corpus.tag.alphabet"; - -// public static final String START_SYM=""; - public static final String END_SYM=""; - public static final String NUM_TOK=""; - - public static final String UNK_TOK=""; - - private ArrayListsent; - private ArrayListdata; - - public ArrayListtag; - public ArrayListtagData; - - public static boolean convertNumTok=true; - - private HashMapfreq; - public HashMapvocab; - - public HashMaptagVocab; - private int tagV; - - private int V; - - public static void main(String[] args) { - Corpus c=new Corpus("../posdata/en_test.conll"); - System.out.println( - Arrays.toString(c.get(0)) - ); - System.out.println( - Arrays.toString(c.getInt(0)) - ); - - System.out.println( - Arrays.toString(c.get(1)) - ); - System.out.println( - Arrays.toString(c.getInt(1)) - ); - } - - public Corpus(String filename,HashMapdict){ - V=0; - tagV=0; - freq=new HashMap(); - tagVocab=new HashMap(); - vocab=dict; - - sent=new ArrayList(); - tag=new ArrayList(); - - Scanner sc=io.FileUtil.openInFile(filename); - ArrayLists=new ArrayList(); - // s.add(START_SYM); - while(sc.hasNextLine()){ - String line=sc.nextLine(); - String toks[]=line.split("\t"); - if(toks.length<2){ - s.add(END_SYM); - sent.add(s.toArray(new String[0])); - s=new ArrayList(); - // s.add(START_SYM); - continue; - } - String tok=toks[1].toLowerCase(); - s.add(tok); - } - sc.close(); - - buildData(); - } - - public Corpus(String filename){ - V=0; - freq=new HashMap(); - vocab=new HashMap(); - tagVocab=new HashMap(); - - sent=new ArrayList(); - tag=new ArrayList(); - - System.out.println("Reading:"+filename); - - Scanner sc=io.FileUtil.openInFile(filename); - ArrayLists=new ArrayList(); - ArrayListtags=new ArrayList(); - //s.add(START_SYM); - while(sc.hasNextLine()){ - String line=sc.nextLine(); - String toks[]=line.split("\t"); - if(toks.length<2){ - s.add(END_SYM); - tags.add(END_SYM); - if(s.size()>2){ - sent.add(s.toArray(new String[0])); - tag.add(tags.toArray(new String [0])); - } - s=new ArrayList(); - tags=new ArrayList(); - // s.add(START_SYM); - continue; - } - - String tok=toks[1].toLowerCase(); - if(convertNumTok && tok.matches(".*\\d.*")){ - tok=NUM_TOK; - } - s.add(tok); - - if(toks.length>3){ - tok=toks[3].toLowerCase(); - }else{ - tok="_"; - } - tags.add(tok); - - } - sc.close(); - - for(int i=0;i(); - for(int i=0;i(); - for(int i=0;i2){ - vocab.put(key, V); - V++; - } - } - io.SerializedObjects.writeSerializedObject(vocab, alphaFilename); - io.SerializedObjects.writeSerializedObject(tagVocab,tagalphaFilename); - } - - private void addTag(String tag){ - Integer i=tagVocab.get(tag); - if(i==null){ - tagVocab.put(tag, tagV); - tagV++; - } - } - -} diff --git a/gi/posterior-regularisation/prjava/src/hmm/HMM.java b/gi/posterior-regularisation/prjava/src/hmm/HMM.java deleted file mode 100644 index 17a4679f..00000000 --- a/gi/posterior-regularisation/prjava/src/hmm/HMM.java +++ /dev/null @@ -1,579 +0,0 @@ -package hmm; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Scanner; - -public class HMM { - - - //trans[i][j]=prob of going FROM i to j - double [][]trans; - double [][]emit; - double []pi; - int [][]data; - int [][]tagdata; - - double logtrans[][]; - - public HMMObjective o; - - public static void main(String[] args) { - - } - - public HMM(int n_state,int n_emit,int [][]data){ - trans=new double [n_state][n_state]; - emit=new double[n_state][n_emit]; - pi=new double [n_state]; - System.out.println(" random initial parameters"); - fillRand(trans); - fillRand(emit); - fillRand(pi); - - this.data=data; - - } - - private void fillRand(double [][] a){ - for(int i=0;i=0;n--){ - for(int i=0;imaxprob){ - maxprob=p[seq.length-1][i]; - maxIdx=i; - } - } - int ans[]=new int [seq.length]; - ans[seq.length-1]=maxIdx; - for(int i=seq.length-2;i>=0;i--){ - ans[i]=backp[i+1][ans[i+1]]; - } - return ans; - } - - public double l1norm(double a[]){ - double norm=0; - for(int i=0;i s=new ArrayList(); - int state=sample(pi); - int sym=sample(emit[state]); - while(sym!=terminalSym){ - s.add(sym); - state=sample(trans[state]); - sym=sample(emit[state]); - } - - int ans[]=new int [s.size()]; - for(int i=0;i=r){ - return i; - } - } - return p.length-1; - } - - public void train(int tagdata[][]){ - double trans_exp_cnt[][]=new double [trans.length][trans.length]; - double emit_exp_cnt[][]=new double[trans.length][emit[0].length]; - double start_exp_cnt[]=new double[trans.length]; - - for(int i=0;imaxwt[i][d[sentNum][n]]){ - maxwt[i][d[sentNum][n]]=py; - } - - } - } - - //the last state - int len=post.length; - for(int i=0;imaxwt[i][d[sentNum][len-1]]){ - maxwt[i][d[sentNum][len-1]]=py; - } - - } - - } - - } - -}//end of class diff --git a/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java b/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java deleted file mode 100644 index 70b6c966..00000000 --- a/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java +++ /dev/null @@ -1,351 +0,0 @@ -package hmm; - -import gnu.trove.TIntArrayList; -import optimization.gradientBasedMethods.ProjectedGradientDescent; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc; -import optimization.linesearch.InterpolationPickFirstStep; -import optimization.linesearch.LineSearchMethod; -import optimization.projections.SimplexProjection; -import optimization.stopCriteria.CompositeStopingCriteria; -import optimization.stopCriteria.ProjectedGradientL2Norm; -import optimization.stopCriteria.StopingCriteria; -import optimization.stopCriteria.ValueDifference; - -public class HMMObjective extends ProjectedObjective{ - - - private static final double GRAD_DIFF = 3; - public static double INIT_STEP_SIZE=10; - public static double VAL_DIFF=1000; - - private HMM hmm; - double[] newPoint ; - - //posterior[sent num][tok num][tag]=index into lambda - private int posteriorMap[][][]; - //projection[word][tag].get(occurence)=index into lambda - private TIntArrayList projectionMap[][]; - - //Size of the simplex - public double scale=10; - private SimplexProjection projection; - - private int wordFreq[]; - private static int MIN_FREQ=10; - private int numWordsToProject=0; - - private int n_param; - - public double loglikelihood; - - public HMMObjective(HMM h){ - hmm=h; - - countWords(); - buildMap(); - - gradient=new double [n_param]; - projection = new SimplexProjection(scale); - newPoint = new double[n_param]; - setInitialParameters(new double[n_param]); - - } - - /**@brief counts word frequency in the corpus - * - */ - private void countWords(){ - wordFreq=new int [hmm.emit[0].length]; - for(int i=0;i

- %(title)s - - Headers - - - Classes - - - Index - -
-''' % self.args ) - self.section = { - 'headers' : self._getChild('library-reference',id='%(id)s.headers' % self.args), - 'classes' : self._getChild('index',id='%(id)s.classes' % self.args), - 'index' : self._getChild('index',id='%(id)s.index' % self.args) - } - #~ Remove the index sections if we aren't generating it. - if not self.args['index']: - self.section['classes'].parentNode.removeChild(self.section['classes']) - self.section['classes'].unlink() - del self.section['classes'] - self.section['index'].parentNode.removeChild(self.section['index']) - self.section['index'].unlink() - del self.section['index'] - #~ The symbols, per Doxygen notion, that we translated. - self.symbols = {} - #~ Map of Doxygen IDs and BoostBook IDs, so we can translate as needed. - self.idmap = {} - #~ Marks generation, to prevent redoing it. - self.generated = False - - #~ Add an Doxygen generated XML document to the content we are translating. - def addDox( self, document ): - self._translateNode(document.documentElement) - - #~ Turns the internal XML tree into an output UTF-8 string. - def tostring( self ): - self._generate() - #~ return self.boostbook.toprettyxml(' ') - return self.boostbook.toxml('utf-8') - - #~ Does post-processing on the partial generated content to generate additional info - #~ now that we have the complete source documents. - def _generate( self ): - if not self.generated: - self.generated = True - symbols = self.symbols.keys() - symbols.sort() - #~ Populate the header section. - for symbol in symbols: - if self.symbols[symbol]['kind'] in ('header'): - self.section['headers'].appendChild(self.symbols[symbol]['dom']) - for symbol in symbols: - if self.symbols[symbol]['kind'] not in ('namespace', 'header'): - container = self._resolveContainer(self.symbols[symbol], - self.symbols[self.symbols[symbol]['header']]['dom']) - if container.nodeName != 'namespace': - ## The current BoostBook to Docbook translation doesn't - ## respect, nor assign, IDs to inner types of any kind. - ## So nuke the ID entry so as not create bogus links. - del self.idmap[self.symbols[symbol]['id']] - container.appendChild(self.symbols[symbol]['dom']) - self._rewriteIDs(self.boostbook.documentElement) - - #~ Rewrite the various IDs from Doxygen references to the newly created - #~ BoostBook references. - def _rewriteIDs( self, node ): - if node.nodeName in ('link'): - if (self.idmap.has_key(node.getAttribute('linkend'))): - #~ A link, and we have someplace to repoint it at. - node.setAttribute('linkend',self.idmap[node.getAttribute('linkend')]) - else: - #~ A link, but we don't have a generated target for it. - node.removeAttribute('linkend') - elif hasattr(node,'hasAttribute') and node.hasAttribute('id') and self.idmap.has_key(node.getAttribute('id')): - #~ Simple ID, and we have a translation. - node.setAttribute('id',self.idmap[node.getAttribute('id')]) - #~ Recurse, and iterate, depth-first traversal which turns out to be - #~ left-to-right and top-to-bottom for the document. - if node.firstChild: - self._rewriteIDs(node.firstChild) - if node.nextSibling: - self._rewriteIDs(node.nextSibling) - - def _resolveContainer( self, cpp, root ): - container = root - for ns in cpp['namespace']: - node = self._getChild('namespace',name=ns,root=container) - if not node: - node = container.appendChild( - self._createNode('namespace',name=ns)) - container = node - for inner in cpp['name'].split('::'): - node = self._getChild(name=inner,root=container) - if not node: - break - container = node - return container - - def _setID( self, id, name ): - self.idmap[id] = name.replace('::','.').replace('/','.') - #~ print '--| setID:',id,'::',self.idmap[id] - - #~ Translate a given node within a given context. - #~ The translation dispatches to a local method of the form - #~ "_translate[_context0,...,_contextN]", and the keyword args are - #~ passed along. If there is no translation handling method we - #~ return None. - def _translateNode( self, *context, **kwargs ): - node = None - names = [ ] - for c in context: - if c: - if not isinstance(c,xml.dom.Node): - suffix = '_'+c.replace('-','_') - else: - suffix = '_'+c.nodeName.replace('-','_') - node = c - names.append('_translate') - names = map(lambda x: x+suffix,names) - if node: - for name in names: - if hasattr(self,name): - return getattr(self,name)(node,**kwargs) - return None - - #~ Translates the children of the given parent node, appending the results - #~ to the indicated target. For nodes not translated by the translation method - #~ it copies the child over and recurses on that child to translate any - #~ possible interior nodes. Hence this will translate the entire subtree. - def _translateChildren( self, parent, **kwargs ): - target = kwargs['target'] - for n in parent.childNodes: - child = self._translateNode(n,target=target) - if child: - target.appendChild(child) - else: - child = n.cloneNode(False) - if hasattr(child,'data'): - child.data = re.sub(r'\s+',' ',child.data) - target.appendChild(child) - self._translateChildren(n,target=child) - - #~ Translate the given node as a description, into the description subnode - #~ of the target. If no description subnode is present in the target it - #~ is created. - def _translateDescription( self, node, target=None, tag='description', **kwargs ): - description = self._getChild(tag,root=target) - if not description: - description = target.appendChild(self._createNode(tag)) - self._translateChildren(node,target=description) - return description - - #~ Top level translation of: ..., - #~ translates the children. - def _translate_doxygen( self, node ): - #~ print '_translate_doxygen:', node.nodeName - result = [] - for n in node.childNodes: - newNode = self._translateNode(n) - if newNode: - result.append(newNode) - return result - - #~ Top level translation of: - #~ - #~ - #~ - #~ ... - #~ - #~ ... - #~ - #~ ... - #~ - #~ builds the class and symbol sections, if requested. - def _translate_doxygenindex( self, node ): - #~ print '_translate_doxygenindex:', node.nodeName - if self.args['index']: - entries = [] - classes = [] - #~ Accumulate all the index entries we care about. - for n in node.childNodes: - if n.nodeName == 'compound': - if n.getAttribute('kind') not in ('file','dir','define'): - cpp = self._cppName(self._getChildData('name',root=n)) - entry = { - 'name' : cpp['name'], - 'compoundname' : cpp['compoundname'], - 'id' : n.getAttribute('refid') - } - if n.getAttribute('kind') in ('class','struct'): - classes.append(entry) - entries.append(entry) - for m in n.childNodes: - if m.nodeName == 'member': - cpp = self._cppName(self._getChildData('name',root=m)) - entry = { - 'name' : cpp['name'], - 'compoundname' : cpp['compoundname'], - 'id' : n.getAttribute('refid') - } - if hasattr(m,'getAttribute') and m.getAttribute('kind') in ('class','struct'): - classes.append(entry) - entries.append(entry) - #~ Put them in a sensible order. - entries.sort(lambda x,y: cmp(x['name'].lower(),y['name'].lower())) - classes.sort(lambda x,y: cmp(x['name'].lower(),y['name'].lower())) - #~ And generate the BoostBook for them. - self._translate_index_(entries,target=self.section['index']) - self._translate_index_(classes,target=self.section['classes']) - return None - - #~ Translate a set of index entries in the BoostBook output. The output - #~ is grouped into groups of the first letter of the entry names. - def _translate_index_(self, entries, target=None, **kwargs ): - i = 0 - targetID = target.getAttribute('id') - while i < len(entries): - dividerKey = entries[i]['name'][0].upper() - divider = target.appendChild(self._createNode('indexdiv',id=targetID+'.'+dividerKey)) - divider.appendChild(self._createText('title',dividerKey)) - while i < len(entries) and dividerKey == entries[i]['name'][0].upper(): - iename = entries[i]['name'] - ie = divider.appendChild(self._createNode('indexentry')) - ie = ie.appendChild(self._createText('primaryie',iename)) - while i < len(entries) and entries[i]['name'] == iename: - ie.appendChild(self.boostbook.createTextNode(' (')) - ie.appendChild(self._createText( - 'link',entries[i]['compoundname'],linkend=entries[i]['id'])) - ie.appendChild(self.boostbook.createTextNode(')')) - i += 1 - - #~ Translate a ..., - #~ by retranslating with the "kind" of compounddef. - def _translate_compounddef( self, node, target=None, **kwargs ): - return self._translateNode(node,node.getAttribute('kind')) - - #~ Translate a .... For - #~ namespaces we just collect the information for later use as there is no - #~ currently namespaces are not included in the BoostBook format. In the future - #~ it might be good to generate a namespace index. - def _translate_compounddef_namespace( self, node, target=None, **kwargs ): - namespace = { - 'id' : node.getAttribute('id'), - 'kind' : 'namespace', - 'name' : self._getChildData('compoundname',root=node), - 'brief' : self._getChildData('briefdescription',root=node), - 'detailed' : self._getChildData('detaileddescription',root=node), - 'parsed' : False - } - if self.symbols.has_key(namespace['name']): - if not self.symbols[namespace['name']]['parsed']: - self.symbols[namespace['name']]['parsed'] = True - #~ for n in node.childNodes: - #~ if hasattr(n,'getAttribute'): - #~ self._translateNode(n,n.getAttribute('kind'),target=target,**kwargs) - else: - self.symbols[namespace['name']] = namespace - #~ self._setID(namespace['id'],namespace['name']) - return None - - #~ Translate a ..., which - #~ forwards to the kind=struct as they are the same. - def _translate_compounddef_class( self, node, target=None, **kwargs ): - return self._translate_compounddef_struct(node,tag='class',target=target,**kwargs) - - #~ Translate a ... into: - #~
- #~ - #~ ... - #~ - #~
- def _translate_compounddef_struct( self, node, tag='struct', target=None, **kwargs ): - result = None - includes = self._getChild('includes',root=node) - if includes: - ## Add the header into the output table. - self._translate_compounddef_includes_(includes,includes,**kwargs) - ## Compounds are the declared symbols, classes, types, etc. - ## We add them to the symbol table, along with the partial DOM for them - ## so that they can be organized into the output later. - compoundname = self._getChildData('compoundname',root=node) - compoundname = self._cppName(compoundname) - self._setID(node.getAttribute('id'),compoundname['compoundname']) - struct = self._createNode(tag,name=compoundname['name'].split('::')[-1]) - self.symbols[compoundname['compoundname']] = { - 'header' : includes.firstChild.data, - 'namespace' : compoundname['namespace'], - 'id' : node.getAttribute('id'), - 'kind' : tag, - 'name' : compoundname['name'], - 'dom' : struct - } - ## Add the children which will be the members of the struct. - for n in node.childNodes: - self._translateNode(n,target=struct,scope=compoundname['compoundname']) - result = struct - return result - - #~ Translate a ..., - def _translate_compounddef_includes_( self, node, target=None, **kwargs ): - name = node.firstChild.data - if not self.symbols.has_key(name): - self._setID(node.getAttribute('refid'),name) - self.symbols[name] = { - 'kind' : 'header', - 'id' : node.getAttribute('refid'), - 'dom' : self._createNode('header', - id=node.getAttribute('refid'), - name=name) - } - return None - - #~ Translate a ... into: - #~ - #~ ... - #~ - def _translate_basecompoundref( self, ref, target=None, **kwargs ): - inherit = target.appendChild(self._createNode('inherit', - access=ref.getAttribute('prot'))) - self._translateChildren(ref,target=inherit) - return - - #~ Translate: - #~ - #~ - #~ ... - #~ ... - #~ ... - #~ ... - #~ - #~ ... - #~ - #~ Into: - #~ - def _translate_templateparamlist( self, templateparamlist, target=None, **kwargs ): - template = target.appendChild(self._createNode('template')) - for param in templateparamlist.childNodes: - if param.nodeName == 'param': - type = self._getChildData('type',root=param) - defval = self._getChild('defval',root=param) - paramKind = None - if type in ('class','typename'): - paramKind = 'template-type-parameter' - else: - paramKind = 'template-nontype-parameter' - templateParam = template.appendChild( - self._createNode(paramKind, - name=self._getChildData('declname',root=param))) - if paramKind == 'template-nontype-parameter': - template_type = templateParam.appendChild(self._createNode('type')) - self._translate_type( - self._getChild('type',root=param),target=template_type) - if defval: - value = self._getChildData('ref',root=defval.firstChild) - if not value: - value = self._getData(defval) - templateParam.appendChild(self._createText('default',value)) - return template - - #~ Translate: - #~ ... - #~ Into: - #~ ... - def _translate_briefdescription( self, brief, target=None, **kwargs ): - self._translateDescription(brief,target=target,**kwargs) - return self._translateDescription(brief,target=target,tag='purpose',**kwargs) - - #~ Translate: - #~ ... - #~ Into: - #~ ... - def _translate_detaileddescription( self, detailed, target=None, **kwargs ): - return self._translateDescription(detailed,target=target,**kwargs) - - #~ Translate: - #~ ... - #~ With kind specific translation. - def _translate_sectiondef( self, sectiondef, target=None, **kwargs ): - self._translateNode(sectiondef,sectiondef.getAttribute('kind'),target=target,**kwargs) - - #~ Translate non-function sections. - def _translate_sectiondef_x_( self, sectiondef, target=None, **kwargs ): - for n in sectiondef.childNodes: - if hasattr(n,'getAttribute'): - self._translateNode(n,n.getAttribute('kind'),target=target,**kwargs) - return None - - #~ Translate: - #~ ... - def _translate_sectiondef_public_type( self, sectiondef, target=None, **kwargs ): - return self._translate_sectiondef_x_(sectiondef,target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_sectiondef_public_attrib( self, sectiondef, target=None, **kwargs): - return self._translate_sectiondef_x_(sectiondef,target=target,**kwargs) - - #~ Translate: - #~ ... - #~ All the various function group translations end up here for which - #~ they are translated into: - #~ - #~ ... - #~ - def _translate_sectiondef_func_( self, sectiondef, name='functions', target=None, **kwargs ): - members = target.appendChild(self._createNode('method-group',name=name)) - for n in sectiondef.childNodes: - if hasattr(n,'getAttribute'): - self._translateNode(n,n.getAttribute('kind'),target=members,**kwargs) - return members - - #~ Translate: - #~ ... - def _translate_sectiondef_public_func( self, sectiondef, target=None, **kwargs ): - return self._translate_sectiondef_func_(sectiondef, - name='public member functions',target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_sectiondef_public_static_func( self, sectiondef, target=None, **kwargs): - return self._translate_sectiondef_func_(sectiondef, - name='public static functions',target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_sectiondef_protected_func( self, sectiondef, target=None, **kwargs ): - return self._translate_sectiondef_func_(sectiondef, - name='protected member functions',target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_sectiondef_private_static_func( self, sectiondef, target=None, **kwargs): - return self._translate_sectiondef_func_(sectiondef, - name='private static functions',target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_sectiondef_private_func( self, sectiondef, target=None, **kwargs ): - return self._translate_sectiondef_func_(sectiondef, - name='private member functions',target=target,**kwargs) - - #~ Translate: - #~
...
... - def _translate_sectiondef_user_defined( self, sectiondef, target=None, **kwargs ): - return self._translate_sectiondef_func_(sectiondef, - name=self._getChildData('header', root=sectiondef),target=target,**kwargs) - - #~ Translate: - #~ - #~ ... - #~ - #~ To: - #~ - #~ ... - #~ - def _translate_memberdef_typedef( self, memberdef, target=None, scope=None, **kwargs ): - self._setID(memberdef.getAttribute('id'), - scope+'::'+self._getChildData('name',root=memberdef)) - typedef = target.appendChild(self._createNode('typedef', - id=memberdef.getAttribute('id'), - name=self._getChildData('name',root=memberdef))) - typedef_type = typedef.appendChild(self._createNode('type')) - self._translate_type(self._getChild('type',root=memberdef),target=typedef_type) - return typedef - - #~ Translate: - #~ - #~ ... - #~ - #~ To: - #~ - #~ ... - #~ - def _translate_memberdef_function( self, memberdef, target=None, scope=None, **kwargs ): - name = self._getChildData('name',root=memberdef) - self._setID(memberdef.getAttribute('id'),scope+'::'+name) - ## Check if we have some specific kind of method. - if name == scope.split('::')[-1]: - kind = 'constructor' - target = target.parentNode - elif name == '~'+scope.split('::')[-1]: - kind = 'destructor' - target = target.parentNode - elif name == 'operator=': - kind = 'copy-assignment' - target = target.parentNode - else: - kind = 'method' - method = target.appendChild(self._createNode(kind, - # id=memberdef.getAttribute('id'), - name=name, - cv=' '.join([ - if_attribute(memberdef,'const','const','').strip() - ]), - specifiers=' '.join([ - if_attribute(memberdef,'static','static',''), - if_attribute(memberdef,'explicit','explicit',''), - if_attribute(memberdef,'inline','inline','') - ]).strip() - )) - ## We iterate the children to translate each part of the function. - for n in memberdef.childNodes: - self._translateNode(memberdef,'function',n,target=method) - return method - - #~ Translate: - #~ ... - def _translate_memberdef_function_templateparamlist( - self, templateparamlist, target=None, **kwargs ): - return self._translate_templateparamlist(templateparamlist,target=target,**kwargs) - - #~ Translate: - #~ ... - #~ To: - #~ ...? - def _translate_memberdef_function_type( self, resultType, target=None, **kwargs ): - methodType = self._createNode('type') - self._translate_type(resultType,target=methodType) - if methodType.hasChildNodes(): - target.appendChild(methodType) - return methodType - - #~ Translate: - #~ ... - def _translate_memberdef_function_briefdescription( self, description, target=None, **kwargs ): - result = self._translateDescription(description,target=target,**kwargs) - ## For functions if we translate the brief docs to the purpose they end up - ## right above the regular description. And since we just added the brief to that - ## on the previous line, don't bother with the repetition. - # result = self._translateDescription(description,target=target,tag='purpose',**kwargs) - return result - - #~ Translate: - #~ ... - def _translate_memberdef_function_detaileddescription( self, description, target=None, **kwargs ): - return self._translateDescription(description,target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_memberdef_function_inbodydescription( self, description, target=None, **kwargs ): - return self._translateDescription(description,target=target,**kwargs) - - #~ Translate: - #~ ... - def _translate_memberdef_function_param( self, param, target=None, **kwargs ): - return self._translate_param(param,target=target,**kwargs) - - #~ Translate: - #~ - #~ ... - #~ ... - #~ - #~ To: - #~ - #~ ... - #~ - def _translate_memberdef_variable( self, memberdef, target=None, scope=None, **kwargs ): - self._setID(memberdef.getAttribute('id'), - scope+'::'+self._getChildData('name',root=memberdef)) - data_member = target.appendChild(self._createNode('data-member', - id=memberdef.getAttribute('id'), - name=self._getChildData('name',root=memberdef))) - data_member_type = data_member.appendChild(self._createNode('type')) - self._translate_type(self._getChild('type',root=memberdef),target=data_member_type) - - #~ Translate: - #~ - #~ ... - #~ ... - #~ - #~ To: - #~ - #~ ... - #~ - def _translate_memberdef_enum( self, memberdef, target=None, scope=None, **kwargs ): - self._setID(memberdef.getAttribute('id'), - scope+'::'+self._getChildData('name',root=memberdef)) - enum = target.appendChild(self._createNode('enum', - id=memberdef.getAttribute('id'), - name=self._getChildData('name',root=memberdef))) - for n in memberdef.childNodes: - self._translateNode(memberdef,'enum',n,target=enum,scope=scope,**kwargs) - return enum - - #~ Translate: - #~ - #~ - #~ ... - #~ ... - #~ - #~ - #~ To: - #~ - #~ ... - #~ - def _translate_memberdef_enum_enumvalue( self, enumvalue, target=None, scope=None, **kwargs ): - self._setID(enumvalue.getAttribute('id'), - scope+'::'+self._getChildData('name',root=enumvalue)) - value = target.appendChild(self._createNode('enumvalue', - id=enumvalue.getAttribute('id'), - name=self._getChildData('name',root=enumvalue))) - initializer = self._getChild('initializer',root=enumvalue) - if initializer: - self._translateChildren(initializer, - target=target.appendChild(self._createNode('default'))) - return value - - #~ Translate: - #~ - #~ ... - #~ ... - #~ ... - #~ - #~ To: - #~ - #~ ... - #~ ... - #~ - def _translate_param( self, param, target=None, **kwargs): - parameter = target.appendChild(self._createNode('parameter', - name=self._getChildData('declname',root=param))) - paramtype = parameter.appendChild(self._createNode('paramtype')) - self._translate_type(self._getChild('type',root=param),target=paramtype) - defval = self._getChild('defval',root=param) - if defval: - self._translateChildren(self._getChild('defval',root=param),target=parameter) - return parameter - - #~ Translate: - #~ ... - def _translate_ref( self, ref, **kwargs ): - return self._translateNode(ref,ref.getAttribute('kindref')) - - #~ Translate: - #~ ... - #~ To: - #~ ... - def _translate_ref_compound( self, ref, **kwargs ): - result = self._createNode('link',linkend=ref.getAttribute('refid')) - classname = result.appendChild(self._createNode('classname')) - self._translateChildren(ref,target=classname) - return result - - #~ Translate: - #~ ... - #~ To: - #~ ... - def _translate_ref_member( self, ref, **kwargs ): - result = self._createNode('link',linkend=ref.getAttribute('refid')) - self._translateChildren(ref,target=result) - return result - - #~ Translate: - #~ ... - def _translate_type( self, type, target=None, **kwargs ): - result = self._translateChildren(type,target=target,**kwargs) - #~ Filter types to clean up various readability problems, most notably - #~ with really long types. - xml = target.toxml('utf-8'); - if ( - xml.startswith('boost::mpl::') or - xml.startswith('BOOST_PP_') or - re.match('boost::(lazy_)?(enable|disable)_if',xml) - ): - while target.firstChild: - target.removeChild(target.firstChild) - target.appendChild(self._createText('emphasis','unspecified')) - return result - - def _getChild( self, tag = None, id = None, name = None, root = None ): - if not root: - root = self.boostbook.documentElement - for n in root.childNodes: - found = True - if tag and found: - found = found and tag == n.nodeName - if id and found: - if n.hasAttribute('id'): - found = found and n.getAttribute('id') == id - else: - found = found and n.hasAttribute('id') and n.getAttribute('id') == id - if name and found: - found = found and n.hasAttribute('name') and n.getAttribute('name') == name - if found: - #~ print '--|', n - return n - return None - - def _getChildData( self, tag, **kwargs ): - return self._getData(self._getChild(tag,**kwargs),**kwargs) - - def _getData( self, node, **kwargs ): - if node: - text = self._getChild('#text',root=node) - if text: - return text.data.strip() - return '' - - def _cppName( self, type ): - parts = re.search('^([^<]+)[<]?(.*)[>]?$',type.strip().strip(':')) - result = { - 'compoundname' : parts.group(1), - 'namespace' : parts.group(1).split('::')[0:-1], - 'name' : parts.group(1).split('::')[-1], - 'specialization' : parts.group(2) - } - if result['namespace'] and len(result['namespace']) > 0: - namespace = '::'.join(result['namespace']) - while ( - len(result['namespace']) > 0 and ( - not self.symbols.has_key(namespace) or - self.symbols[namespace]['kind'] != 'namespace') - ): - result['name'] = result['namespace'].pop()+'::'+result['name'] - namespace = '::'.join(result['namespace']) - return result - - def _createNode( self, tag, **kwargs ): - result = self.boostbook.createElement(tag) - for k in kwargs.keys(): - if kwargs[k] != '': - if k == 'id': - result.setAttribute('id',kwargs[k]) - else: - result.setAttribute(k,kwargs[k]) - return result - - def _createText( self, tag, data, **kwargs ): - result = self._createNode(tag,**kwargs) - data = data.strip() - if len(data) > 0: - result.appendChild(self.boostbook.createTextNode(data)) - return result - - -def main( xmldir=None, output=None, id=None, title=None, index=False ): - #~ print '--- main: xmldir = %s, output = %s' % (xmldir,output) - - input = glob.glob( os.path.abspath( os.path.join( xmldir, "*.xml" ) ) ) - input.sort - translator = Doxygen2BoostBook(id=id, title=title, index=index) - #~ Feed in the namespaces first to build up the set of namespaces - #~ and definitions so that lookup is unambiguous when reading in the definitions. - namespace_files = filter( - lambda x: - os.path.basename(x).startswith('namespace'), - input) - decl_files = filter( - lambda x: - not os.path.basename(x).startswith('namespace') and not os.path.basename(x).startswith('_'), - input) - for dox in namespace_files: - #~ print '--|',os.path.basename(dox) - translator.addDox(xml.dom.minidom.parse(dox)) - for dox in decl_files: - #~ print '--|',os.path.basename(dox) - translator.addDox(xml.dom.minidom.parse(dox)) - - if output: - output = open(output,'w') - else: - output = sys.stdout - if output: - output.write(translator.tostring()) - - -main( **get_args() ) diff --git a/jam-files/boost-build/tools/doxygen-config.jam b/jam-files/boost-build/tools/doxygen-config.jam deleted file mode 100644 index 2cd2ccae..00000000 --- a/jam-files/boost-build/tools/doxygen-config.jam +++ /dev/null @@ -1,11 +0,0 @@ -#~ Copyright 2005, 2006 Rene Rivera. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Automatic configuration for Doxygen tools. To use, just import this module. - -import toolset : using ; - -ECHO "warning: doxygen-config.jam is deprecated. Use 'using doxygen ;' instead." ; - -using doxygen ; diff --git a/jam-files/boost-build/tools/doxygen.jam b/jam-files/boost-build/tools/doxygen.jam deleted file mode 100644 index 8394848d..00000000 --- a/jam-files/boost-build/tools/doxygen.jam +++ /dev/null @@ -1,776 +0,0 @@ -# Copyright 2003, 2004 Douglas Gregor -# Copyright 2003, 2004, 2005 Vladimir Prus -# Copyright 2006 Rene Rivera -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module defines rules to handle generation of various outputs from source -# files documented with doxygen comments. The supported transformations are: -# -# * Source -> Doxygen XML -> BoostBook XML -# * Source -> Doxygen HTML -# -# The type of transformation is selected based on the target requested. For -# BoostBook XML, the default, specifying a target with an ".xml" suffix, or an -# empty suffix, will produce a .xml and .boostbook. For Doxygen -# HTML specifying a target with an ".html" suffix will produce a directory -# with the Doxygen html files, and a .html file redirecting to -# that directory. - -import "class" : new ; -import targets ; -import feature ; -import property ; -import generators ; -import boostbook ; -import type ; -import path ; -import print ; -import regex ; -import stage ; -import project ; -import xsltproc ; -import make ; -import os ; -import toolset : flags ; -import alias ; -import common ; -import modules ; -import project ; -import utility ; -import errors ; - - -# Use to specify extra configuration paramters. These get translated -# into a doxyfile which configures the building of the docs. -feature.feature doxygen:param : : free ; - -# Specify the "boost.doxygen.header.prefix" XSLT option. -feature.feature prefix : : free ; - -# Specify the "boost.doxygen.reftitle" XSLT option. -feature.feature reftitle : : free ; - -# Which processor to use for various translations from Doxygen. -feature.feature doxygen.processor : xsltproc doxproc : propagated implicit ; - -# To generate, or not, index sections. -feature.feature doxygen.doxproc.index : no yes : propagated incidental ; - -# The ID for the resulting BoostBook reference section. -feature.feature doxygen.doxproc.id : : free ; - -# The title for the resulting BoostBook reference section. -feature.feature doxygen.doxproc.title : : free ; - -# Location for images when generating XML -feature.feature doxygen:xml-imagedir : : free ; - -# Indicates whether the entire directory should be deleted -feature.feature doxygen.rmdir : off on : optional incidental ; - -# Doxygen configuration input file. -type.register DOXYFILE : doxyfile ; - -# Doxygen XML multi-file output. -type.register DOXYGEN_XML_MULTIFILE : xml-dir : XML ; - -# Doxygen XML coallesed output. -type.register DOXYGEN_XML : doxygen : XML ; - -# Doxygen HTML multifile directory. -type.register DOXYGEN_HTML_MULTIFILE : html-dir : HTML ; - -# Redirection HTML file to HTML multifile directory. -type.register DOXYGEN_HTML : : HTML ; - -type.register DOXYGEN_XML_IMAGES : doxygen-xml-images ; - -# Initialize the Doxygen module. Parameters are: -# name: the name of the 'doxygen' executable. If not specified, the name -# 'doxygen' will be used -# -rule init ( name ? ) -{ - if ! $(.initialized) - { - .initialized = true ; - - .doxproc = [ modules.binding $(__name__) ] ; - .doxproc = $(.doxproc:D)/doxproc.py ; - - generators.register-composing doxygen.headers-to-doxyfile - : H HPP CPP : DOXYFILE ; - generators.register-standard doxygen.run - : DOXYFILE : DOXYGEN_XML_MULTIFILE ; - generators.register-standard doxygen.xml-dir-to-boostbook - : DOXYGEN_XML_MULTIFILE : BOOSTBOOK : doxproc ; - generators.register-standard doxygen.xml-to-boostbook - : DOXYGEN_XML : BOOSTBOOK : xsltproc ; - generators.register-standard doxygen.collect - : DOXYGEN_XML_MULTIFILE : DOXYGEN_XML ; - generators.register-standard doxygen.run - : DOXYFILE : DOXYGEN_HTML_MULTIFILE ; - generators.register-standard doxygen.html-redirect - : DOXYGEN_HTML_MULTIFILE : DOXYGEN_HTML ; - generators.register-standard doxygen.copy-latex-pngs - : DOXYGEN_HTML : DOXYGEN_XML_IMAGES ; - - IMPORT $(__name__) : doxygen : : doxygen ; - } - - if $(name) - { - modify-config ; - .doxygen = $(name) ; - check-doxygen ; - } - - if ! $(.doxygen) - { - check-doxygen ; - } -} - -rule freeze-config ( ) -{ - if ! $(.initialized) - { - errors.user-error "doxygen must be initialized before it can be used." ; - } - if ! $(.config-frozen) - { - .config-frozen = true ; - - if [ .is-cygwin ] - { - .is-cygwin = true ; - } - } -} - -rule modify-config ( ) -{ - if $(.config-frozen) - { - errors.user-error "Cannot change doxygen after it has been used." ; - } -} - -rule check-doxygen ( ) -{ - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using doxygen ":" $(.doxygen) ; - } - local extra-paths ; - if [ os.name ] = NT - { - local ProgramFiles = [ modules.peek : ProgramFiles ] ; - if $(ProgramFiles) - { - extra-paths = "$(ProgramFiles:J= )" ; - } - else - { - extra-paths = "C:\\Program Files" ; - } - } - .doxygen = [ common.get-invocation-command doxygen : - doxygen : $(.doxygen) : $(extra-paths) ] ; -} - -rule name ( ) -{ - freeze-config ; - return $(.doxygen) ; -} - -rule .is-cygwin ( ) -{ - if [ os.on-windows ] - { - local file = [ path.make [ modules.binding $(__name__) ] ] ; - local dir = [ path.native - [ path.join [ path.parent $(file) ] doxygen ] ] ; - local command = - "cd \"$(dir)\" && \"$(.doxygen)\" windows-paths-check.doxyfile 2>&1" ; - result = [ SHELL $(command) ] ; - if [ MATCH "(Parsing file /)" : $(result) ] - { - return true ; - } - } -} - -# Runs Doxygen on the given Doxygen configuration file (the source) to generate -# the Doxygen files. The output is dumped according to the settings in the -# Doxygen configuration file, not according to the target! Because of this, we -# essentially "touch" the target file, in effect making it look like we have -# really written something useful to it. Anyone that uses this action must deal -# with this behavior. -# -actions doxygen-action -{ - $(RM) "$(*.XML)" & "$(NAME:E=doxygen)" "$(>)" && echo "Stamped" > "$(<)" -} - - -# Runs the Python doxproc XML processor. -# -actions doxproc -{ - python "$(DOXPROC)" "--xmldir=$(>)" "--output=$(<)" "$(OPTIONS)" "--id=$(ID)" "--title=$(TITLE)" -} - - -rule translate-path ( path ) -{ - freeze-config ; - if [ os.on-windows ] - { - if [ os.name ] = CYGWIN - { - if $(.is-cygwin) - { - return $(path) ; - } - else - { - return $(path:W) ; - } - } - else - { - if $(.is-cygwin) - { - match = [ MATCH ^(.):(.*) : $(path) ] ; - if $(match) - { - return /cygdrive/$(match[1])$(match[2]:T) ; - } - else - { - return $(path:T) ; - } - } - else - { - return $(path) ; - } - } - } - else - { - return $(path) ; - } -} - - -# Generates a doxygen configuration file (doxyfile) given a set of C++ sources -# and a property list that may contain features. -# -rule headers-to-doxyfile ( target : sources * : properties * ) -{ - local text "# Generated by Boost.Build version 2" ; - - local output-dir ; - - # Translate into command line flags. - for local param in [ feature.get-values : $(properties) ] - { - local namevalue = [ regex.match ([^=]*)=(.*) : $(param) ] ; - if $(namevalue[1]) = OUTPUT_DIRECTORY - { - output-dir = [ translate-path - [ utility.unquote $(namevalue[2]) ] ] ; - text += "OUTPUT_DIRECTORY = \"$(output-dir)\"" ; - } - else - { - text += "$(namevalue[1]) = $(namevalue[2])" ; - } - } - - if ! $(output-dir) - { - output-dir = [ translate-path [ on $(target) return $(LOCATE) ] ] ; - text += "OUTPUT_DIRECTORY = \"$(output-dir)\"" ; - } - - local headers = ; - for local header in $(sources:G=) - { - header = [ translate-path $(header) ] ; - headers += \"$(header)\" ; - } - - # Doxygen generates LaTex by default. So disable it unconditionally, or at - # least until someone needs, and hence writes support for, LaTex output. - text += "GENERATE_LATEX = NO" ; - text += "INPUT = $(headers:J= )" ; - print.output $(target) plain ; - print.text $(text) : true ; -} - - -# Run Doxygen. See doxygen-action for a description of the strange properties of -# this rule. -# -rule run ( target : source : properties * ) -{ - freeze-config ; - if on in $(properties) - { - local output-dir = - [ path.make - [ MATCH OUTPUT_DIRECTORY=\"?([^\"]*) : - $(properties) ] ] ; - local html-dir = - [ path.make - [ MATCH HTML_OUTPUT=(.*) : - $(properties) ] ] ; - if $(output-dir) && $(html-dir) && - [ path.glob $(output-dir) : $(html-dir) ] - { - HTMLDIR on $(target) = - [ path.native [ path.join $(output-dir) $(html-dir) ] ] ; - rm-htmldir $(target) ; - } - } - doxygen-action $(target) : $(source) ; - NAME on $(target) = $(.doxygen) ; - RM on $(target) = [ modules.peek common : RM ] ; - *.XML on $(target) = - [ path.native - [ path.join - [ path.make [ on $(target) return $(LOCATE) ] ] - $(target:B:S=) - *.xml ] ] ; -} - -if [ os.name ] = NT -{ - RMDIR = rmdir /s /q ; -} -else -{ - RMDIR = rm -rf ; -} - -actions quietly rm-htmldir -{ - $(RMDIR) $(HTMLDIR) -} - -# The rules below require Boost.Book stylesheets, so we need some code to check -# that the boostbook module has actualy been initialized. -# -rule check-boostbook ( ) -{ - if ! [ modules.peek boostbook : .initialized ] - { - ECHO "error: the boostbook module is not initialized" ; - ECHO "error: you've attempted to use the 'doxygen' toolset, " ; - ECHO "error: which requires Boost.Book," ; - ECHO "error: but never initialized Boost.Book." ; - EXIT "error: Hint: add 'using boostbook ;' to your user-config.jam" ; - } -} - - -# Collect the set of Doxygen XML files into a single XML source file that can be -# handled by an XSLT processor. The source is completely ignored (see -# doxygen-action), because this action picks up the Doxygen XML index file -# xml/index.xml. This is because we can not teach Doxygen to act like a NORMAL -# program and take a "-o output.xml" argument (grrrr). The target of the -# collection will be a single Doxygen XML file. -# -rule collect ( target : source : properties * ) -{ - check-boostbook ; - local collect-xsl-dir - = [ path.native [ path.join [ boostbook.xsl-dir ] doxygen collect ] ] ; - local source-path - = [ path.make [ on $(source) return $(LOCATE) ] ] ; - local collect-path - = [ path.root [ path.join $(source-path) $(source:B) ] [ path.pwd ] ] ; - local native-path - = [ path.native $(collect-path) ] ; - local real-source - = [ path.native [ path.join $(collect-path) index.xml ] ] ; - xsltproc.xslt $(target) : $(real-source) $(collect-xsl-dir:S=.xsl) - : doxygen.xml.path=$(native-path) ; -} - - -# Translate Doxygen XML into BoostBook. -# -rule xml-to-boostbook ( target : source : properties * ) -{ - check-boostbook ; - local xsl-dir = [ boostbook.xsl-dir ] ; - local d2b-xsl = [ path.native [ path.join [ boostbook.xsl-dir ] doxygen - doxygen2boostbook.xsl ] ] ; - - local xslt-properties = $(properties) ; - for local prefix in [ feature.get-values : $(properties) ] - { - xslt-properties += "boost.doxygen.header.prefix=$(prefix)" ; - } - for local title in [ feature.get-values : $(properties) ] - { - xslt-properties += "boost.doxygen.reftitle=$(title)" ; - } - - xsltproc.xslt $(target) : $(source) $(d2b-xsl) : $(xslt-properties) ; -} - - -flags doxygen.xml-dir-to-boostbook OPTIONS yes : --enable-index ; -flags doxygen.xml-dir-to-boostbook ID ; -flags doxygen.xml-dir-to-boostbook TITLE ; - - -rule xml-dir-to-boostbook ( target : source : properties * ) -{ - DOXPROC on $(target) = $(.doxproc) ; - - LOCATE on $(source:S=) = [ on $(source) return $(LOCATE) ] ; - - doxygen.doxproc $(target) : $(source:S=) ; -} - - -# Generate the HTML redirect to HTML dir index.html file. -# -rule html-redirect ( target : source : properties * ) -{ - local uri = "$(target:B)/index.html" ; - print.output $(target) plain ; - print.text -" - - - - - - - - - Automatic redirection failed, please go to $(uri). - - -" - : true ; -} - -rule copy-latex-pngs ( target : source : requirements * ) -{ - local directory = [ path.native - [ feature.get-values : - $(requirements) ] ] ; - - local location = [ on $(target) return $(LOCATE) ] ; - - local pdf-location = - [ path.native - [ path.join - [ path.make $(location) ] - [ path.make $(directory) ] ] ] ; - local html-location = - [ path.native - [ path.join - . - html - [ path.make $(directory) ] ] ] ; - - common.MkDir $(pdf-location) ; - common.MkDir $(html-location) ; - - DEPENDS $(target) : $(pdf-location) $(html-location) ; - - if [ os.name ] = NT - { - CP on $(target) = copy /y ; - FROM on $(target) = \\*.png ; - TOHTML on $(target) = .\\html\\$(directory) ; - TOPDF on $(target) = \\$(directory) ; - } - else - { - CP on $(target) = cp ; - FROM on $(target) = /*.png ; - TOHTML on $(target) = ./html/$(directory) ; - TOPDF on $(target) = $(target:D)/$(directory) ; - } -} - -actions copy-latex-pngs -{ - $(CP) $(>:S=)$(FROM) $(TOHTML) - $(CP) $(>:S=)$(FROM) $(<:D)$(TOPDF) - echo "Stamped" > "$(<)" -} - -# building latex images for doxygen XML depends -# on latex, dvips, and ps being in your PATH. -# This is true for most Unix installs, but -# not on Win32, where you will need to install -# MkTex and Ghostscript and add these tools -# to your path. - -actions check-latex -{ - latex -version >$(<) -} - -actions check-dvips -{ - dvips -version >$(<) -} - -if [ os.name ] = "NT" -{ - actions check-gs - { - gswin32c -version >$(<) - } -} -else -{ - actions check-gs - { - gs -version >$(<) - } -} - -rule check-tools ( ) -{ - if ! $(.check-tools-targets) - { - # Find the root project. - local root-project = [ project.current ] ; - root-project = [ $(root-project).project-module ] ; - while - [ project.attribute $(root-project) parent-module ] && - [ project.attribute $(root-project) parent-module ] != user-config - { - root-project = - [ project.attribute $(root-project) parent-module ] ; - } - - .latex.check = [ new file-target latex.check - : - : [ project.target $(root-project) ] - : [ new action : doxygen.check-latex ] - : - ] ; - .dvips.check = [ new file-target dvips.check - : - : [ project.target $(root-project) ] - : [ new action : doxygen.check-dvips ] - : - ] ; - .gs.check = [ new file-target gs.check - : - : [ project.target $(root-project) ] - : [ new action : doxygen.check-gs ] - : - ] ; - .check-tools-targets = $(.latex.check) $(.dvips.check) $(.gs.check) ; - } - return $(.check-tools-targets) ; -} - -project.initialize $(__name__) ; -project doxygen ; - -class doxygen-check-tools-target-class : basic-target -{ - import doxygen ; - rule construct ( name : sources * : property-set ) - { - return [ property-set.empty ] [ doxygen.check-tools ] ; - } -} - -local project = [ project.current ] ; - -targets.main-target-alternative - [ new doxygen-check-tools-target-class check-tools : $(project) - : [ targets.main-target-sources : check-tools : no-renaming ] - : [ targets.main-target-requirements : $(project) ] - : [ targets.main-target-default-build : $(project) ] - : [ targets.main-target-usage-requirements : $(project) ] - ] ; - -# User-level rule to generate BoostBook XML from a set of headers via Doxygen. -# -rule doxygen ( target : sources * : requirements * : default-build * : usage-requirements * ) -{ - freeze-config ; - local project = [ project.current ] ; - - if $(target:S) = .html - { - # Build an HTML directory from the sources. - local html-location = [ feature.get-values : $(requirements) ] ; - local output-dir ; - if [ $(project).get build-dir ] - { - # Explicitly specified build dir. Add html at the end. - output-dir = [ path.join [ $(project).build-dir ] $(html-location:E=html) ] ; - } - else - { - # Trim 'bin' from implicit build dir, for no other reason that backward - # compatibility. - output-dir = [ path.join [ path.parent [ $(project).build-dir ] ] - $(html-location:E=html) ] ; - } - output-dir = [ path.root $(output-dir) [ path.pwd ] ] ; - local output-dir-native = [ path.native $(output-dir) ] ; - requirements = [ property.change $(requirements) : ] ; - - ## The doxygen configuration file. - targets.main-target-alternative - [ new typed-target $(target:S=.tag) : $(project) : DOXYFILE - : [ targets.main-target-sources $(sources) : $(target:S=.tag) ] - : [ targets.main-target-requirements $(requirements) - GENERATE_HTML=YES - GENERATE_XML=NO - "OUTPUT_DIRECTORY=\"$(output-dir-native)\"" - HTML_OUTPUT=$(target:B) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target:S=.tag) ; - - ## The html directory to generate by running doxygen. - targets.main-target-alternative - [ new typed-target $(target:S=.dir) : $(project) : DOXYGEN_HTML_MULTIFILE - : $(target:S=.tag) - : [ targets.main-target-requirements $(requirements) - "OUTPUT_DIRECTORY=\"$(output-dir-native)\"" - HTML_OUTPUT=$(target:B) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target:S=.dir) ; - - ## The redirect html file into the generated html. - targets.main-target-alternative - [ new typed-target $(target) : $(project) : DOXYGEN_HTML - : $(target:S=.dir) - : [ targets.main-target-requirements $(requirements) - $(output-dir) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - } - else - { - # Build a BoostBook XML file from the sources. - local location-xml = [ feature.get-values : $(requirements) ] ; - requirements = [ property.change $(requirements) : ] ; - local target-xml = $(target:B=$(target:B)-xml) ; - - # Check whether we need to build images - local images-location = - [ feature.get-values : $(requirements) ] ; - if $(images-location) - { - doxygen $(target).doxygen-xml-images.html : $(sources) - : $(requirements) - on - QUIET=YES - WARNINGS=NO - WARN_IF_UNDOCUMENTED=NO - /doxygen//check-tools ; - $(project).mark-target-as-explicit - $(target).doxygen-xml-images.html ; - - targets.main-target-alternative - [ new typed-target $(target).doxygen-xml-images - : $(project) : DOXYGEN_XML_IMAGES - : $(target).doxygen-xml-images.html - : [ targets.main-target-requirements $(requirements) - : $(project) ] - : [ targets.main-target-default-build $(default-build) - : $(project) ] - ] ; - - $(project).mark-target-as-explicit - $(target).doxygen-xml-images ; - - if ! [ regex.match "^(.*/)$" : $(images-location) ] - { - images-location = $(images-location)/ ; - } - - requirements += - $(target).doxygen-xml-images - boost.doxygen.formuladir=$(images-location) ; - } - - ## The doxygen configuration file. - targets.main-target-alternative - [ new typed-target $(target-xml:S=.tag) : $(project) : DOXYFILE - : [ targets.main-target-sources $(sources) : $(target-xml:S=.tag) ] - : [ targets.main-target-requirements $(requirements) - GENERATE_HTML=NO - GENERATE_XML=YES - XML_OUTPUT=$(target-xml) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target-xml:S=.tag) ; - - ## The Doxygen XML directory of the processed source files. - targets.main-target-alternative - [ new typed-target $(target-xml:S=.dir) : $(project) : DOXYGEN_XML_MULTIFILE - : $(target-xml:S=.tag) - : [ targets.main-target-requirements $(requirements) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target-xml:S=.dir) ; - - ## The resulting BoostBook file is generated by the processor tool. The - ## tool can be either the xsltproc plus accompanying XSL scripts. Or it - ## can be the python doxproc.py script. - targets.main-target-alternative - [ new typed-target $(target-xml) : $(project) : BOOSTBOOK - : $(target-xml:S=.dir) - : [ targets.main-target-requirements $(requirements) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target-xml) ; - - targets.main-target-alternative - [ new install-target-class $(target:S=.xml) : $(project) - : $(target-xml) - : [ targets.main-target-requirements $(requirements) - $(location-xml:E=.) - $(target:S=.xml) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(target:S=.xml) ; - - targets.main-target-alternative - [ new alias-target-class $(target) : $(project) - : - : [ targets.main-target-requirements $(requirements) - : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - : [ targets.main-target-usage-requirements $(usage-requirements) - $(target:S=.xml) - : $(project) ] - ] ; - } -} diff --git a/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile b/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile deleted file mode 100644 index 9b969df9..00000000 --- a/jam-files/boost-build/tools/doxygen/windows-paths-check.doxyfile +++ /dev/null @@ -1,3 +0,0 @@ -INPUT = windows-paths-check.hpp -GENERATE_HTML = NO -GENERATE_LATEX = NO diff --git a/jam-files/boost-build/tools/doxygen/windows-paths-check.hpp b/jam-files/boost-build/tools/doxygen/windows-paths-check.hpp deleted file mode 100644 index e69de29b..00000000 diff --git a/jam-files/boost-build/tools/fop.jam b/jam-files/boost-build/tools/fop.jam deleted file mode 100644 index c24b8725..00000000 --- a/jam-files/boost-build/tools/fop.jam +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2003-2004 Doug Gregor and Dave Abrahams. Distributed -# under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) -# -# This module defines rules to handle generation of PDF and -# PostScript files from XSL Formatting Objects via Apache FOP - -import generators ; -import common ; -import boostbook ; - -generators.register-standard fop.render.pdf : FO : PDF ; -generators.register-standard fop.render.ps : FO : PS ; - -# Initializes the fop toolset. -# -rule init ( fop-command ? : java-home ? : java ? ) -{ - local has-command = $(.has-command) ; - - if $(fop-command) - { - .has-command = true ; - } - - if $(fop-command) || ! $(has-command) - { - fop-command = [ common.get-invocation-command fop : fop : $(fop-command) - : [ modules.peek : FOP_DIR ] ] ; - } - - if $(fop-command) - { - .FOP_COMMAND = $(fop-command) ; - } - - if $(java-home) || $(java) - { - .FOP_SETUP = ; - - - # JAVA_HOME is the location that java was installed to. - - if $(java-home) - { - .FOP_SETUP += [ common.variable-setting-command JAVA_HOME : $(java-home) ] ; - } - - # JAVACMD is the location that of the java executable, useful for a - # non-standard java installation, where the executable isn't at - # $JAVA_HOME/bin/java. - - if $(java) - { - .FOP_SETUP += [ common.variable-setting-command JAVACMD : $(java) ] ; - } - } -} - -actions render.pdf -{ - $(.FOP_SETUP) $(.FOP_COMMAND:E=fop) $(>) $(<) -} - -actions render.ps -{ - $(.FOP_SETUP) $(.FOP_COMMAND:E=fop) $(>) -ps $(<) -} diff --git a/jam-files/boost-build/tools/fortran.jam b/jam-files/boost-build/tools/fortran.jam deleted file mode 100644 index 37665825..00000000 --- a/jam-files/boost-build/tools/fortran.jam +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2004 Toon Knapen -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# -# This file contains common settings for all fortran tools -# - -import "class" : new ; -import feature : feature ; - -import type ; -import generators ; -import common ; - -type.register FORTRAN : f F for f77 ; -type.register FORTRAN90 : f90 F90 ; - -feature fortran : : free ; -feature fortran90 : : free ; - -class fortran-compiling-generator : generator -{ - rule __init__ ( id : source-types + : target-types + : requirements * : optional-properties * ) - { - generator.__init__ $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ; - } -} - -rule register-fortran-compiler ( id : source-types + : target-types + : requirements * : optional-properties * ) -{ - local g = [ new fortran-compiling-generator $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ] ; - generators.register $(g) ; -} - -class fortran90-compiling-generator : generator -{ - rule __init__ ( id : source-types + : target-types + : requirements * : optional-properties * ) - { - generator.__init__ $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ; - } -} - -rule register-fortran90-compiler ( id : source-types + : target-types + : requirements * : optional-properties * ) -{ - local g = [ new fortran90-compiling-generator $(id) : $(source-types) : $(target-types) : $(requirements) : $(optional-properties) ] ; - generators.register $(g) ; -} - -# FIXME: this is ugly, should find a better way (we'd want client code to -# register all generators as "generator.some-rule", not with "some-module.some-rule".) -IMPORT $(__name__) : register-fortran-compiler : : generators.register-fortran-compiler ; -IMPORT $(__name__) : register-fortran90-compiler : : generators.register-fortran90-compiler ; diff --git a/jam-files/boost-build/tools/gcc.jam b/jam-files/boost-build/tools/gcc.jam deleted file mode 100644 index f7b0da54..00000000 --- a/jam-files/boost-build/tools/gcc.jam +++ /dev/null @@ -1,1185 +0,0 @@ -# Copyright 2001 David Abrahams. -# Copyright 2002-2006 Rene Rivera. -# Copyright 2002-2003 Vladimir Prus. -# Copyright (c) 2005 Reece H. Dunn. -# Copyright 2006 Ilya Sokolov. -# Copyright 2007 Roland Schwarz -# Copyright 2007 Boris Gubenko. -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import "class" : new ; -import common ; -import errors ; -import feature ; -import generators ; -import os ; -import pch ; -import property ; -import property-set ; -import toolset ; -import type ; -import rc ; -import regex ; -import set ; -import unix ; -import fortran ; - - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - - -feature.extend toolset : gcc ; -# feature.subfeature toolset gcc : flavor : : optional ; - -toolset.inherit-generators gcc : unix : unix.link unix.link.dll ; -toolset.inherit-flags gcc : unix ; -toolset.inherit-rules gcc : unix ; - -generators.override gcc.prebuilt : builtin.prebuilt ; -generators.override gcc.searched-lib-generator : searched-lib-generator ; - -# Make gcc toolset object files use the "o" suffix on all platforms. -type.set-generated-target-suffix OBJ : gcc : o ; -type.set-generated-target-suffix OBJ : gcc windows : o ; -type.set-generated-target-suffix OBJ : gcc cygwin : o ; - -# Initializes the gcc toolset for the given version. If necessary, command may -# be used to specify where the compiler is located. The parameter 'options' is a -# space-delimited list of options, each one specified as -# option-value. Valid option names are: cxxflags, linkflags and -# linker-type. Accepted linker-type values are aix, darwin, gnu, hpux, osf or -# sun and the default value will be selected based on the current OS. -# Example: -# using gcc : 3.4 : : foo bar sun ; -# -# The compiler command to use is detected in a three step manner: -# 1) If an explicit command is specified by the user, it will be used and must available. -# 2) If only a certain version is specified, it is enforced: -# - either a command 'g++-VERSION' must be available -# - or the default command 'g++' must be available and match the exact version. -# 3) Without user-provided restrictions use default 'g++' -rule init ( version ? : command * : options * ) -{ - #1): use user-provided command - local tool-command = ; - if $(command) - { - tool-command = [ common.get-invocation-command-nodefault gcc : g++ : $(command) ] ; - if ! $(tool-command) - { - errors.error "toolset gcc initialization:" : - "provided command '$(command)' not found" : - "initialized from" [ errors.nearest-user-location ] ; - } - } - #2): enforce user-provided version - else if $(version) - { - tool-command = [ common.get-invocation-command-nodefault gcc : "g++-$(version[1])" ] ; - - #2.1) fallback: check whether "g++" reports the requested version - if ! $(tool-command) - { - tool-command = [ common.get-invocation-command-nodefault gcc : g++ ] ; - if $(tool-command) - { - local tool-command-string = $(tool-command:J=" ") ; - local tool-version = [ MATCH "^([0-9.]+)" : [ SHELL "$(tool-command-string) -dumpversion" ] ] ; - if $(tool-version) != $(version) - { - # Permit a match betwen two-digit version specified by the user - # (e.g. 4.4) and 3-digit version reported by gcc. - # Since only two digits are present in binary name anyway, - # insisting that user specify 3-digit version when - # configuring Boost.Build while it's not required on - # command like would be strange. - local stripped = [ MATCH "^([0-9]+\.[0-9]+).*" : $(tool-version) ] ; - if $(stripped) != $(version) - { - errors.error "toolset gcc initialization:" : - "version '$(version)' requested but 'g++-$(version)' not found and version '$(tool-version)' of default '$(tool-command)' does not match" : - "initialized from" [ errors.nearest-user-location ] ; - tool-command = ; - } - # Use full 3-digit version to be compatible with the 'using gcc ;' case - version = $(tool-version) ; - } - } - else - { - errors.error "toolset gcc initialization:" : - "version '$(version)' requested but neither 'g++-$(version)' nor default 'g++' found" : - "initialized from" [ errors.nearest-user-location ] ; - } - } - } - #3) default: no command and no version specified, try using default command "g++" - else - { - tool-command = [ common.get-invocation-command-nodefault gcc : g++ ] ; - if ! $(tool-command) - { - errors.error "toolset gcc initialization:" : - "no command provided, default command 'g++' not found" : - "initialized from" [ errors.nearest-user-location ] ; - } - } - - - # Information about the gcc command... - # The command. - local command = $(tool-command) ; - # The root directory of the tool install. - local root = [ feature.get-values : $(options) ] ; - # The bin directory where to find the command to execute. - local bin ; - # The flavor of compiler. - local flavor = [ feature.get-values : $(options) ] ; - # Autodetect the root and bin dir if not given. - if $(command) - { - bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ; - root ?= $(bin:D) ; - } - # The 'command' variable can have multiple elements. When calling - # the SHELL builtin we need a single string. - local command-string = $(command:J=" ") ; - # Autodetect the version and flavor if not given. - if $(command) - { - local machine = [ MATCH "^([^ ]+)" - : [ SHELL "$(command-string) -dumpmachine" ] ] ; - version ?= [ MATCH "^([0-9.]+)" - : [ SHELL "$(command-string) -dumpversion" ] ] ; - switch $(machine:L) - { - case *mingw* : flavor ?= mingw ; - } - } - - local condition ; - if $(flavor) - { - condition = [ common.check-init-parameters gcc - : version $(version) - : flavor $(flavor) - ] ; - } - else - { - condition = [ common.check-init-parameters gcc - : version $(version) - ] ; - condition = $(condition) ; #/ ; - } - - common.handle-options gcc : $(condition) : $(command) : $(options) ; - - local linker = [ feature.get-values : $(options) ] ; - # The logic below should actually be keyed on - if ! $(linker) - { - if [ os.name ] = OSF - { - linker = osf ; - } - else if [ os.name ] = HPUX - { - linker = hpux ; - } - else if [ os.name ] = AIX - { - linker = aix ; - } - else if [ os.name ] = SOLARIS - { - linker = sun ; - } - else - { - linker = gnu ; - } - } - init-link-flags gcc $(linker) $(condition) ; - - - # If gcc is installed in non-standard location, we'd need to add - # LD_LIBRARY_PATH when running programs created with it (for unit-test/run - # rules). - if $(command) - { - # On multilib 64-bit boxes, there are both 32-bit and 64-bit libraries - # and all must be added to LD_LIBRARY_PATH. The linker will pick the - # right onces. Note that we don't provide a clean way to build 32-bit - # binary with 64-bit compiler, but user can always pass -m32 manually. - local lib_path = $(root)/bin $(root)/lib $(root)/lib32 $(root)/lib64 ; - if $(.debug-configuration) - { - ECHO notice: using gcc libraries :: $(condition) :: $(lib_path) ; - } - toolset.flags gcc.link RUN_PATH $(condition) : $(lib_path) ; - } - - # If it's not a system gcc install we should adjust the various programs as - # needed to prefer using the install specific versions. This is essential - # for correct use of MinGW and for cross-compiling. - - local nl = " -" ; - - # - The archive builder. - local archiver = [ common.get-invocation-command gcc - : [ NORMALIZE_PATH [ MATCH "(.*)[$(nl)]+" : [ SHELL "$(command-string) -print-prog-name=ar" ] ] ] - : [ feature.get-values : $(options) ] - : $(bin) - : search-path ] ; - toolset.flags gcc.archive .AR $(condition) : $(archiver[1]) ; - if $(.debug-configuration) - { - ECHO notice: using gcc archiver :: $(condition) :: $(archiver[1]) ; - } - - # - Ranlib - local ranlib = [ common.get-invocation-command gcc - : [ NORMALIZE_PATH [ MATCH "(.*)[$(nl)]+" : [ SHELL "$(command-string) -print-prog-name=ranlib" ] ] ] - : [ feature.get-values : $(options) ] - : $(bin) - : search-path ] ; - toolset.flags gcc.archive .RANLIB $(condition) : $(ranlib[1]) ; - if $(.debug-configuration) - { - ECHO notice: using gcc ranlib :: $(condition) :: $(ranlib[1]) ; - } - - - # - The resource compiler. - local rc = - [ common.get-invocation-command-nodefault gcc - : windres : [ feature.get-values : $(options) ] : $(bin) : search-path ] ; - local rc-type = - [ feature.get-values : $(options) ] ; - rc-type ?= windres ; - if ! $(rc) - { - # If we can't find an RC compiler we fallback to a null RC compiler that - # creates empty object files. This allows the same Jamfiles to work - # across the board. The null RC uses the assembler to create the empty - # objects, so configure that. - rc = [ common.get-invocation-command gcc : as : : $(bin) : search-path ] ; - rc-type = null ; - } - rc.configure $(rc) : $(condition) : $(rc-type) ; -} - -if [ os.name ] = NT -{ - # This causes single-line command invocation to not go through .bat files, - # thus avoiding command-line length limitations. - JAMSHELL = % ; -} - -generators.register-c-compiler gcc.compile.c++.preprocess : CPP : PREPROCESSED_CPP : gcc ; -generators.register-c-compiler gcc.compile.c.preprocess : C : PREPROCESSED_C : gcc ; -generators.register-c-compiler gcc.compile.c++ : CPP : OBJ : gcc ; -generators.register-c-compiler gcc.compile.c : C : OBJ : gcc ; -generators.register-c-compiler gcc.compile.asm : ASM : OBJ : gcc ; -generators.register-fortran-compiler gcc.compile.fortran : FORTRAN FORTRAN90 : OBJ : gcc ; - -# pch support - -# The compiler looks for a precompiled header in each directory just before it -# looks for the include file in that directory. The name searched for is the -# name specified in the #include directive with ".gch" suffix appended. The -# logic in gcc-pch-generator will make sure that BASE_PCH suffix is appended to -# full name of the header. - -type.set-generated-target-suffix PCH : gcc : gch ; - -# GCC-specific pch generator. -class gcc-pch-generator : pch-generator -{ - import project ; - import property-set ; - import type ; - - rule run-pch ( project name ? : property-set : sources + ) - { - # Find the header in sources. Ignore any CPP sources. - local header ; - for local s in $(sources) - { - if [ type.is-derived [ $(s).type ] H ] - { - header = $(s) ; - } - } - - # Error handling: Base header file name should be the same as the base - # precompiled header name. - local header-name = [ $(header).name ] ; - local header-basename = $(header-name:B) ; - if $(header-basename) != $(name) - { - local location = [ $(project).project-module ] ; - errors.user-error "in" $(location)": pch target name `"$(name)"' should be the same as the base name of header file `"$(header-name)"'" ; - } - - local pch-file = [ generator.run $(project) $(name) : $(property-set) - : $(header) ] ; - - # return result of base class and pch-file property as usage-requirements - return - [ property-set.create $(pch-file) -Winvalid-pch ] - $(pch-file) - ; - } - - # Calls the base version specifying source's name as the name of the created - # target. As result, the PCH will be named whatever.hpp.gch, and not - # whatever.gch. - rule generated-targets ( sources + : property-set : project name ? ) - { - name = [ $(sources[1]).name ] ; - return [ generator.generated-targets $(sources) - : $(property-set) : $(project) $(name) ] ; - } -} - -# Note: the 'H' source type will catch both '.h' header and '.hpp' header. The -# latter have HPP type, but HPP type is derived from H. The type of compilation -# is determined entirely by the destination type. -generators.register [ new gcc-pch-generator gcc.compile.c.pch : H : C_PCH : on gcc ] ; -generators.register [ new gcc-pch-generator gcc.compile.c++.pch : H : CPP_PCH : on gcc ] ; - -# Override default do-nothing generators. -generators.override gcc.compile.c.pch : pch.default-c-pch-generator ; -generators.override gcc.compile.c++.pch : pch.default-cpp-pch-generator ; - -toolset.flags gcc.compile PCH_FILE on : ; - -# Declare flags and action for compilation. -toolset.flags gcc.compile OPTIONS off : -O0 ; -toolset.flags gcc.compile OPTIONS speed : -O3 ; -toolset.flags gcc.compile OPTIONS space : -Os ; - -toolset.flags gcc.compile OPTIONS off : -fno-inline ; -toolset.flags gcc.compile OPTIONS on : -Wno-inline ; -toolset.flags gcc.compile OPTIONS full : -finline-functions -Wno-inline ; - -toolset.flags gcc.compile OPTIONS off : -w ; -toolset.flags gcc.compile OPTIONS on : -Wall ; -toolset.flags gcc.compile OPTIONS all : -Wall -pedantic ; -toolset.flags gcc.compile OPTIONS on : -Werror ; - -toolset.flags gcc.compile OPTIONS on : -g ; -toolset.flags gcc.compile OPTIONS on : -pg ; -toolset.flags gcc.compile OPTIONS off : -fno-rtti ; - -rule setup-fpic ( targets * : sources * : properties * ) -{ - local link = [ feature.get-values link : $(properties) ] ; - if $(link) = shared - { - local target = [ feature.get-values target-os : $(properties) ] ; - - # This logic will add -fPIC for all compilations: - # - # lib a : a.cpp b ; - # obj b : b.cpp ; - # exe c : c.cpp a d ; - # obj d : d.cpp ; - # - # This all is fine, except that 'd' will be compiled with -fPIC even though - # it is not needed, as 'd' is used only in exe. However, it is hard to - # detect where a target is going to be used. Alternatively, we can set -fPIC - # only when main target type is LIB but than 'b' would be compiled without - # -fPIC which would lead to link errors on x86-64. So, compile everything - # with -fPIC. - # - # Yet another alternative would be to create a propagated - # feature and set it when building shared libraries, but that would be hard - # to implement and would increase the target path length even more. - - # On Windows, fPIC is default, specifying -fPIC explicitly leads to - # a warning. - if $(target) != cygwin && $(target) != windows - { - OPTIONS on $(targets) += -fPIC ; - } - } -} - -rule setup-address-model ( targets * : sources * : properties * ) -{ - local model = [ feature.get-values address-model : $(properties) ] ; - if $(model) - { - local option ; - local os = [ feature.get-values target-os : $(properties) ] ; - if $(os) = aix - { - if $(model) = 32 - { - option = -maix32 ; - } - else - { - option = -maix64 ; - } - } - else if $(os) = hpux - { - if $(model) = 32 - { - option = -milp32 ; - } - else - { - option = -mlp64 ; - } - } - else - { - if $(model) = 32 - { - option = -m32 ; - } - else if $(model) = 64 - { - option = -m64 ; - } - # For darwin, the model can be 32_64. darwin.jam will handle that - # on its own. - } - OPTIONS on $(targets) += $(option) ; - } -} - - -# FIXME: this should not use os.name. -if [ os.name ] != NT && [ os.name ] != OSF && [ os.name ] != HPUX && [ os.name ] != AIX -{ - # OSF does have an option called -soname but it does not seem to work as - # expected, therefore it has been disabled. - HAVE_SONAME = "" ; - SONAME_OPTION = -h ; -} - -# HPUX, for some reason, seem to use '+h', not '-h'. -if [ os.name ] = HPUX -{ - HAVE_SONAME = "" ; - SONAME_OPTION = +h ; -} - -toolset.flags gcc.compile USER_OPTIONS ; -toolset.flags gcc.compile.c++ USER_OPTIONS ; -toolset.flags gcc.compile DEFINES ; -toolset.flags gcc.compile INCLUDES ; -toolset.flags gcc.compile.c++ TEMPLATE_DEPTH ; -toolset.flags gcc.compile.fortran USER_OPTIONS ; - -rule compile.c++.pch ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; -} - -actions compile.c++.pch -{ - "$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -rule compile.c.pch ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; -} - -actions compile.c.pch -{ - "$(CONFIG_COMMAND)" -x c-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -rule compile.c++.preprocess ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - - # Some extensions are compiled as C++ by default. For others, we need to - # pass -x c++. We could always pass -x c++ but distcc does not work with it. - if ! $(>:S) in .cc .cp .cxx .cpp .c++ .C - { - LANG on $(<) = "-x c++" ; - } - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; -} - -rule compile.c.preprocess ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - - # If we use the name g++ then default file suffix -> language mapping does - # not work. So have to pass -x option. Maybe, we can work around this by - # allowing the user to specify both C and C++ compiler names. - #if $(>:S) != .c - #{ - LANG on $(<) = "-x c" ; - #} - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; -} - -rule compile.c++ ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - - # Some extensions are compiled as C++ by default. For others, we need to - # pass -x c++. We could always pass -x c++ but distcc does not work with it. - if ! $(>:S) in .cc .cp .cxx .cpp .c++ .C - { - LANG on $(<) = "-x c++" ; - } - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; - - # Here we want to raise the template-depth parameter value to something - # higher than the default value of 17. Note that we could do this using the - # feature.set-default rule but we do not want to set the default value for - # all toolsets as well. - # - # TODO: This 'modified default' has been inherited from some 'older Boost - # Build implementation' and has most likely been added to make some Boost - # library parts compile correctly. We should see what exactly prompted this - # and whether we can get around the problem more locally. - local template-depth = [ on $(<) return $(TEMPLATE_DEPTH) ] ; - if ! $(template-depth) - { - TEMPLATE_DEPTH on $(<) = 128 ; - } -} - -rule compile.c ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - - # If we use the name g++ then default file suffix -> language mapping does - # not work. So have to pass -x option. Maybe, we can work around this by - # allowing the user to specify both C and C++ compiler names. - #if $(>:S) != .c - #{ - LANG on $(<) = "-x c" ; - #} - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; -} - -rule compile.fortran ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; -} - -actions compile.c++ bind PCH_FILE -{ - "$(CONFIG_COMMAND)" $(LANG) -ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<:W)" "$(>:W)" -} - -actions compile.c bind PCH_FILE -{ - "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++.preprocess bind PCH_FILE -{ - "$(CONFIG_COMMAND)" $(LANG) -ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" "$(>:W)" -E >"$(<:W)" -} - -actions compile.c.preprocess bind PCH_FILE -{ - "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" "$(>)" -E >$(<) -} - -actions compile.fortran -{ - "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -rule compile.asm ( targets * : sources * : properties * ) -{ - setup-fpic $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - LANG on $(<) = "-x assembler-with-cpp" ; -} - -actions compile.asm -{ - "$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -# The class which check that we don't try to use the static -# property while creating or using shared library, since it's not supported by -# gcc/libc. -class gcc-linking-generator : unix-linking-generator -{ - rule run ( project name ? : property-set : sources + ) - { - # TODO: Replace this with the use of a target-os property. - local no-static-link = ; - if [ modules.peek : UNIX ] - { - switch [ modules.peek : JAMUNAME ] - { - case * : no-static-link = true ; - } - } - - local properties = [ $(property-set).raw ] ; - local reason ; - if $(no-static-link) && static in $(properties) - { - if shared in $(properties) - { - reason = - "On gcc, DLL can't be build with 'static'." ; - } - else if [ type.is-derived $(self.target-types[1]) EXE ] - { - for local s in $(sources) - { - local type = [ $(s).type ] ; - if $(type) && [ type.is-derived $(type) SHARED_LIB ] - { - reason = - "On gcc, using DLLS together with the" - "static options is not possible " ; - } - } - } - } - if $(reason) - { - ECHO warning: - $(reason) ; - ECHO warning: - "It is suggested to use 'static' together" - "with 'static'." ; - return ; - } - else - { - local generated-targets = [ unix-linking-generator.run $(project) - $(name) : $(property-set) : $(sources) ] ; - return $(generated-targets) ; - } - } -} - -# The set of permissible input types is different on mingw. -# So, define two sets of generators, with mingw generators -# selected when target-os=windows. - -local g ; -g = [ new gcc-linking-generator gcc.mingw.link - : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB - : EXE - : gcc windows ] ; -$(g).set-rule-name gcc.link ; -generators.register $(g) ; - -g = [ new gcc-linking-generator gcc.mingw.link.dll - : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB - : IMPORT_LIB SHARED_LIB - : gcc windows ] ; -$(g).set-rule-name gcc.link.dll ; -generators.register $(g) ; - -generators.register - [ new gcc-linking-generator gcc.link - : LIB OBJ - : EXE - : gcc ] ; -generators.register - [ new gcc-linking-generator gcc.link.dll - : LIB OBJ - : SHARED_LIB - : gcc ] ; - -generators.override gcc.mingw.link : gcc.link ; -generators.override gcc.mingw.link.dll : gcc.link.dll ; - -# Cygwin is similar to msvc and mingw in that it uses import libraries. -# While in simple cases, it can directly link to a shared library, -# it is believed to be slower, and not always possible. Define cygwin-specific -# generators here. - -g = [ new gcc-linking-generator gcc.cygwin.link - : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB - : EXE - : gcc cygwin ] ; -$(g).set-rule-name gcc.link ; -generators.register $(g) ; - -g = [ new gcc-linking-generator gcc.cygwin.link.dll - : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB - : IMPORT_LIB SHARED_LIB - : gcc cygwin ] ; -$(g).set-rule-name gcc.link.dll ; -generators.register $(g) ; - -generators.override gcc.cygwin.link : gcc.link ; -generators.override gcc.cygwin.link.dll : gcc.link.dll ; - -# Declare flags for linking. -# First, the common flags. -toolset.flags gcc.link OPTIONS on : -g ; -toolset.flags gcc.link OPTIONS on : -pg ; -toolset.flags gcc.link USER_OPTIONS ; -toolset.flags gcc.link LINKPATH ; -toolset.flags gcc.link FINDLIBS-ST ; -toolset.flags gcc.link FINDLIBS-SA ; -toolset.flags gcc.link LIBRARIES ; - -toolset.flags gcc.link.dll .IMPLIB-COMMAND windows : "-Wl,--out-implib," ; -toolset.flags gcc.link.dll .IMPLIB-COMMAND cygwin : "-Wl,--out-implib," ; - -# For static we made sure there are no dynamic libraries in the -# link. On HP-UX not all system libraries exist as archived libraries (for -# example, there is no libunwind.a), so, on this platform, the -static option -# cannot be specified. -if [ os.name ] != HPUX -{ - toolset.flags gcc.link OPTIONS static : -static ; -} - -# Now, the vendor specific flags. -# The parameter linker can be either aix, darwin, gnu, hpux, osf or sun. -rule init-link-flags ( toolset linker condition ) -{ - switch $(linker) - { - case aix : - { - # - # On AIX we *have* to use the native linker. - # - # Using -brtl, the AIX linker will look for libraries with both the .a - # and .so extensions, such as libfoo.a and libfoo.so. Without -brtl, the - # AIX linker looks only for libfoo.a. Note that libfoo.a is an archived - # file that may contain shared objects and is different from static libs - # as on Linux. - # - # The -bnoipath strips the prepending (relative) path of libraries from - # the loader section in the target library or executable. Hence, during - # load-time LIBPATH (identical to LD_LIBRARY_PATH) or a hard-coded - # -blibpath (*similar* to -lrpath/-lrpath-link) is searched. Without - # this option, the prepending (relative) path + library name is - # hard-coded in the loader section, causing *only* this path to be - # searched during load-time. Note that the AIX linker does not have an - # -soname equivalent, this is as close as it gets. - # - # The above options are definately for AIX 5.x, and most likely also for - # AIX 4.x and AIX 6.x. For details about the AIX linker see: - # http://download.boulder.ibm.com/ibmdl/pub/software/dw/aix/es-aix_ll.pdf - # - - toolset.flags $(toolset).link OPTIONS : -Wl,-brtl -Wl,-bnoipath - : unchecked ; - } - - case darwin : - { - # On Darwin, the -s option to ld does not work unless we pass -static, - # and passing -static unconditionally is a bad idea. So, don't pass -s. - # at all, darwin.jam will use separate 'strip' invocation. - toolset.flags $(toolset).link RPATH $(condition) : : unchecked ; - toolset.flags $(toolset).link RPATH_LINK $(condition) : : unchecked ; - } - - case gnu : - { - # Strip the binary when no debugging is needed. We use --strip-all flag - # as opposed to -s since icc (intel's compiler) is generally - # option-compatible with and inherits from the gcc toolset, but does not - # support -s. - toolset.flags $(toolset).link OPTIONS $(condition)/on : -Wl,--strip-all : unchecked ; - toolset.flags $(toolset).link RPATH $(condition) : : unchecked ; - toolset.flags $(toolset).link RPATH_LINK $(condition) : : unchecked ; - toolset.flags $(toolset).link START-GROUP $(condition) : -Wl,--start-group : unchecked ; - toolset.flags $(toolset).link END-GROUP $(condition) : -Wl,--end-group : unchecked ; - - # gnu ld has the ability to change the search behaviour for libraries - # referenced by -l switch. These modifiers are -Bstatic and -Bdynamic - # and change search for -l switches that follow them. The following list - # shows the tried variants. - # The search stops at the first variant that has a match. - # *nix: -Bstatic -lxxx - # libxxx.a - # - # *nix: -Bdynamic -lxxx - # libxxx.so - # libxxx.a - # - # windows (mingw,cygwin) -Bstatic -lxxx - # libxxx.a - # xxx.lib - # - # windows (mingw,cygwin) -Bdynamic -lxxx - # libxxx.dll.a - # xxx.dll.a - # libxxx.a - # xxx.lib - # cygxxx.dll (*) - # libxxx.dll - # xxx.dll - # libxxx.a - # - # (*) This is for cygwin - # Please note that -Bstatic and -Bdynamic are not a guarantee that a - # static or dynamic lib indeed gets linked in. The switches only change - # search patterns! - - # On *nix mixing shared libs with static runtime is not a good idea. - toolset.flags $(toolset).link FINDLIBS-ST-PFX $(condition)/shared - : -Wl,-Bstatic : unchecked ; - toolset.flags $(toolset).link FINDLIBS-SA-PFX $(condition)/shared - : -Wl,-Bdynamic : unchecked ; - - # On windows allow mixing of static and dynamic libs with static - # runtime. - toolset.flags $(toolset).link FINDLIBS-ST-PFX $(condition)/static/windows - : -Wl,-Bstatic : unchecked ; - toolset.flags $(toolset).link FINDLIBS-SA-PFX $(condition)/static/windows - : -Wl,-Bdynamic : unchecked ; - toolset.flags $(toolset).link OPTIONS $(condition)/static/windows - : -Wl,-Bstatic : unchecked ; - } - - case hpux : - { - toolset.flags $(toolset).link OPTIONS $(condition)/on - : -Wl,-s : unchecked ; - toolset.flags $(toolset).link OPTIONS $(condition)/shared - : -fPIC : unchecked ; - } - - case osf : - { - # No --strip-all, just -s. - toolset.flags $(toolset).link OPTIONS $(condition)/on - : -Wl,-s : unchecked ; - toolset.flags $(toolset).link RPATH $(condition) : - : unchecked ; - # This does not supports -R. - toolset.flags $(toolset).link RPATH_OPTION $(condition) : -rpath - : unchecked ; - # -rpath-link is not supported at all. - } - - case sun : - { - toolset.flags $(toolset).link OPTIONS $(condition)/on - : -Wl,-s : unchecked ; - toolset.flags $(toolset).link RPATH $(condition) : - : unchecked ; - # Solaris linker does not have a separate -rpath-link, but allows to use - # -L for the same purpose. - toolset.flags $(toolset).link LINKPATH $(condition) : - : unchecked ; - - # This permits shared libraries with non-PIC code on Solaris. - # VP, 2004/09/07: Now that we have -fPIC hardcode in link.dll, the - # following is not needed. Whether -fPIC should be hardcoded, is a - # separate question. - # AH, 2004/10/16: it is still necessary because some tests link against - # static libraries that were compiled without PIC. - toolset.flags $(toolset).link OPTIONS $(condition)/shared - : -mimpure-text : unchecked ; - } - - case * : - { - errors.user-error - "$(toolset) initialization: invalid linker '$(linker)'" : - "The value '$(linker)' specified for is not recognized." : - "Possible values are 'aix', 'darwin', 'gnu', 'hpux', 'osf' or 'sun'" ; - } - } -} - -# Enclose the RPATH variable on 'targets' in (double) quotes, -# unless it's already enclosed in single quotes. -# This special casing is done because it's common to pass -# '$ORIGIN' to linker -- and it has to have single quotes -# to prevent expansion by shell -- and if we add double -# quotes then preventing properties of single quotes disappear. -rule quote-rpath ( targets * ) -{ - local r = [ on $(targets[1]) return $(RPATH) ] ; - if ! [ MATCH "('.*')" : $(r) ] - { - r = "\"$(r)\"" ; - } - RPATH on $(targets) = $(r) ; -} - -# Declare actions for linking. -rule link ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - SPACE on $(targets) = " " ; - # Serialize execution of the 'link' action, since running N links in - # parallel is just slower. For now, serialize only gcc links, it might be a - # good idea to serialize all links. - JAM_SEMAPHORE on $(targets) = gcc-link-semaphore ; - quote-rpath $(targets) ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS) - -} - -# Default value. Mostly for the sake of intel-linux that inherits from gcc, but -# does not have the same logic to set the .AR variable. We can put the same -# logic in intel-linux, but that's hardly worth the trouble as on Linux, 'ar' is -# always available. -.AR = ar ; -.RANLIB = ranlib ; - -toolset.flags gcc.archive AROPTIONS ; - -rule archive ( targets * : sources * : properties * ) -{ - # Always remove archive and start again. Here is the rationale from - # - # Andre Hentz: - # - # I had a file, say a1.c, that was included into liba.a. I moved a1.c to - # a2.c, updated my Jamfiles and rebuilt. My program was crashing with absurd - # errors. After some debugging I traced it back to the fact that a1.o was - # *still* in liba.a - # - # Rene Rivera: - # - # Originally removing the archive was done by splicing an RM onto the - # archive action. That makes archives fail to build on NT when they have - # many files because it will no longer execute the action directly and blow - # the line length limit. Instead we remove the file in a different action, - # just before building the archive. - # - local clean.a = $(targets[1])(clean) ; - TEMPORARY $(clean.a) ; - NOCARE $(clean.a) ; - LOCATE on $(clean.a) = [ on $(targets[1]) return $(LOCATE) ] ; - DEPENDS $(clean.a) : $(sources) ; - DEPENDS $(targets) : $(clean.a) ; - common.RmTemps $(clean.a) : $(targets) ; -} - -# Declare action for creating static libraries. -# The letter 'r' means to add files to the archive with replacement. Since we -# remove archive, we don't care about replacement, but there's no option "add -# without replacement". -# The letter 'c' suppresses the warning in case the archive does not exists yet. -# That warning is produced only on some platforms, for whatever reasons. -actions piecemeal archive -{ - "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)" - "$(.RANLIB)" "$(<)" -} - -rule link.dll ( targets * : sources * : properties * ) -{ - setup-threading $(targets) : $(sources) : $(properties) ; - setup-address-model $(targets) : $(sources) : $(properties) ; - SPACE on $(targets) = " " ; - JAM_SEMAPHORE on $(targets) = gcc-link-semaphore ; - quote-rpath $(targets) ; -} - -# Differs from 'link' above only by -shared. -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) "$(.IMPLIB-COMMAND)$(<[1])" -o "$(<[-1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,$(<[-1]:D=) -shared $(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS) -} - -rule setup-threading ( targets * : sources * : properties * ) -{ - local threading = [ feature.get-values threading : $(properties) ] ; - if $(threading) = multi - { - local target = [ feature.get-values target-os : $(properties) ] ; - local option ; - local libs ; - - switch $(target) - { - case windows : - { - option = -mthreads ; - } - case cygwin : - { - option = -mthreads ; - } - case solaris : - { - option = -pthreads ; - libs = rt ; - } - case beos : - { - # BeOS has no threading options, so do not set anything here. - } - case *bsd : - { - option = -pthread ; - # There is no -lrt on BSD. - } - case sgi : - { - # gcc on IRIX does not support multi-threading so do not set anything - # here. - } - case darwin : - { - # Darwin has no threading options so do not set anything here. - } - case * : - { - option = -pthread ; - libs = rt ; - } - } - - if $(option) - { - OPTIONS on $(targets) += $(option) ; - } - if $(libs) - { - FINDLIBS-SA on $(targets) += $(libs) ; - } - } -} - -local rule cpu-flags ( toolset variable : architecture : instruction-set + : values + : default ? ) -{ - if $(default) - { - toolset.flags $(toolset) $(variable) - $(architecture)/ - : $(values) ; - } - toolset.flags $(toolset) $(variable) - /$(instruction-set) - $(architecture)/$(instruction-set) - : $(values) ; -} - -# Set architecture/instruction-set options. -# -# x86 and compatible -# The 'native' option appeared in gcc 4.2 so we cannot safely use it -# as default. Use conservative i386 instead. -cpu-flags gcc OPTIONS : x86 : native : -march=native ; -cpu-flags gcc OPTIONS : x86 : i386 : -march=i386 : default ; -cpu-flags gcc OPTIONS : x86 : i486 : -march=i486 ; -cpu-flags gcc OPTIONS : x86 : i586 : -march=i586 ; -cpu-flags gcc OPTIONS : x86 : i686 : -march=i686 ; -cpu-flags gcc OPTIONS : x86 : pentium : -march=pentium ; -cpu-flags gcc OPTIONS : x86 : pentium-mmx : -march=pentium-mmx ; -cpu-flags gcc OPTIONS : x86 : pentiumpro : -march=pentiumpro ; -cpu-flags gcc OPTIONS : x86 : pentium2 : -march=pentium2 ; -cpu-flags gcc OPTIONS : x86 : pentium3 : -march=pentium3 ; -cpu-flags gcc OPTIONS : x86 : pentium3m : -march=pentium3m ; -cpu-flags gcc OPTIONS : x86 : pentium-m : -march=pentium-m ; -cpu-flags gcc OPTIONS : x86 : pentium4 : -march=pentium4 ; -cpu-flags gcc OPTIONS : x86 : pentium4m : -march=pentium4m ; -cpu-flags gcc OPTIONS : x86 : prescott : -march=prescott ; -cpu-flags gcc OPTIONS : x86 : nocona : -march=nocona ; -cpu-flags gcc OPTIONS : x86 : core2 : -march=core2 ; -cpu-flags gcc OPTIONS : x86 : k6 : -march=k6 ; -cpu-flags gcc OPTIONS : x86 : k6-2 : -march=k6-2 ; -cpu-flags gcc OPTIONS : x86 : k6-3 : -march=k6-3 ; -cpu-flags gcc OPTIONS : x86 : athlon : -march=athlon ; -cpu-flags gcc OPTIONS : x86 : athlon-tbird : -march=athlon-tbird ; -cpu-flags gcc OPTIONS : x86 : athlon-4 : -march=athlon-4 ; -cpu-flags gcc OPTIONS : x86 : athlon-xp : -march=athlon-xp ; -cpu-flags gcc OPTIONS : x86 : athlon-mp : -march=athlon-mp ; -## -cpu-flags gcc OPTIONS : x86 : k8 : -march=k8 ; -cpu-flags gcc OPTIONS : x86 : opteron : -march=opteron ; -cpu-flags gcc OPTIONS : x86 : athlon64 : -march=athlon64 ; -cpu-flags gcc OPTIONS : x86 : athlon-fx : -march=athlon-fx ; -cpu-flags gcc OPTIONS : x86 : winchip-c6 : -march=winchip-c6 ; -cpu-flags gcc OPTIONS : x86 : winchip2 : -march=winchip2 ; -cpu-flags gcc OPTIONS : x86 : c3 : -march=c3 ; -cpu-flags gcc OPTIONS : x86 : c3-2 : -march=c3-2 ; -# Sparc -cpu-flags gcc OPTIONS : sparc : c3 : -mcpu=c3 : default ; -cpu-flags gcc OPTIONS : sparc : v7 : -mcpu=v7 ; -cpu-flags gcc OPTIONS : sparc : cypress : -mcpu=cypress ; -cpu-flags gcc OPTIONS : sparc : v8 : -mcpu=v8 ; -cpu-flags gcc OPTIONS : sparc : supersparc : -mcpu=supersparc ; -cpu-flags gcc OPTIONS : sparc : sparclite : -mcpu=sparclite ; -cpu-flags gcc OPTIONS : sparc : hypersparc : -mcpu=hypersparc ; -cpu-flags gcc OPTIONS : sparc : sparclite86x : -mcpu=sparclite86x ; -cpu-flags gcc OPTIONS : sparc : f930 : -mcpu=f930 ; -cpu-flags gcc OPTIONS : sparc : f934 : -mcpu=f934 ; -cpu-flags gcc OPTIONS : sparc : sparclet : -mcpu=sparclet ; -cpu-flags gcc OPTIONS : sparc : tsc701 : -mcpu=tsc701 ; -cpu-flags gcc OPTIONS : sparc : v9 : -mcpu=v9 ; -cpu-flags gcc OPTIONS : sparc : ultrasparc : -mcpu=ultrasparc ; -cpu-flags gcc OPTIONS : sparc : ultrasparc3 : -mcpu=ultrasparc3 ; -# RS/6000 & PowerPC -cpu-flags gcc OPTIONS : power : 403 : -mcpu=403 ; -cpu-flags gcc OPTIONS : power : 505 : -mcpu=505 ; -cpu-flags gcc OPTIONS : power : 601 : -mcpu=601 ; -cpu-flags gcc OPTIONS : power : 602 : -mcpu=602 ; -cpu-flags gcc OPTIONS : power : 603 : -mcpu=603 ; -cpu-flags gcc OPTIONS : power : 603e : -mcpu=603e ; -cpu-flags gcc OPTIONS : power : 604 : -mcpu=604 ; -cpu-flags gcc OPTIONS : power : 604e : -mcpu=604e ; -cpu-flags gcc OPTIONS : power : 620 : -mcpu=620 ; -cpu-flags gcc OPTIONS : power : 630 : -mcpu=630 ; -cpu-flags gcc OPTIONS : power : 740 : -mcpu=740 ; -cpu-flags gcc OPTIONS : power : 7400 : -mcpu=7400 ; -cpu-flags gcc OPTIONS : power : 7450 : -mcpu=7450 ; -cpu-flags gcc OPTIONS : power : 750 : -mcpu=750 ; -cpu-flags gcc OPTIONS : power : 801 : -mcpu=801 ; -cpu-flags gcc OPTIONS : power : 821 : -mcpu=821 ; -cpu-flags gcc OPTIONS : power : 823 : -mcpu=823 ; -cpu-flags gcc OPTIONS : power : 860 : -mcpu=860 ; -cpu-flags gcc OPTIONS : power : 970 : -mcpu=970 ; -cpu-flags gcc OPTIONS : power : 8540 : -mcpu=8540 ; -cpu-flags gcc OPTIONS : power : power : -mcpu=power ; -cpu-flags gcc OPTIONS : power : power2 : -mcpu=power2 ; -cpu-flags gcc OPTIONS : power : power3 : -mcpu=power3 ; -cpu-flags gcc OPTIONS : power : power4 : -mcpu=power4 ; -cpu-flags gcc OPTIONS : power : power5 : -mcpu=power5 ; -cpu-flags gcc OPTIONS : power : powerpc : -mcpu=powerpc ; -cpu-flags gcc OPTIONS : power : powerpc64 : -mcpu=powerpc64 ; -cpu-flags gcc OPTIONS : power : rios : -mcpu=rios ; -cpu-flags gcc OPTIONS : power : rios1 : -mcpu=rios1 ; -cpu-flags gcc OPTIONS : power : rios2 : -mcpu=rios2 ; -cpu-flags gcc OPTIONS : power : rsc : -mcpu=rsc ; -cpu-flags gcc OPTIONS : power : rs64a : -mcpu=rs64 ; -# AIX variant of RS/6000 & PowerPC -toolset.flags gcc AROPTIONS 64/aix : "-X 64" ; diff --git a/jam-files/boost-build/tools/gcc.py b/jam-files/boost-build/tools/gcc.py deleted file mode 100644 index 2a3e675e..00000000 --- a/jam-files/boost-build/tools/gcc.py +++ /dev/null @@ -1,796 +0,0 @@ -# Status: being ported by Steven Watanabe -# Base revision: 47077 -# TODO: common.jam needs to be ported -# TODO: generators.jam needs to have register_c_compiler. -# -# Copyright 2001 David Abrahams. -# Copyright 2002-2006 Rene Rivera. -# Copyright 2002-2003 Vladimir Prus. -# Copyright (c) 2005 Reece H. Dunn. -# Copyright 2006 Ilya Sokolov. -# Copyright 2007 Roland Schwarz -# Copyright 2007 Boris Gubenko. -# Copyright 2008 Steven Watanabe -# -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import os -import subprocess -import re - -import bjam - -from b2.tools import unix, common, rc, pch, builtin -from b2.build import feature, type, toolset, generators -from b2.util.utility import os_name, on_windows -from b2.manager import get_manager -from b2.build.generators import Generator -from b2.build.toolset import flags -from b2.util.utility import to_seq - -__debug = None - -def debug(): - global __debug - if __debug is None: - __debug = "--debug-configuration" in bjam.variable("ARGV") - return __debug - -feature.extend('toolset', ['gcc']) - - -toolset.inherit_generators('gcc', [], 'unix', ['unix.link', 'unix.link.dll']) -toolset.inherit_flags('gcc', 'unix') -toolset.inherit_rules('gcc', 'unix') - -generators.override('gcc.prebuilt', 'builtin.prebuilt') -generators.override('gcc.searched-lib-generator', 'searched-lib-generator') - -# Target naming is determined by types/lib.jam and the settings below this -# comment. -# -# On *nix: -# libxxx.a static library -# libxxx.so shared library -# -# On windows (mingw): -# libxxx.lib static library -# xxx.dll DLL -# xxx.lib import library -# -# On windows (cygwin) i.e. cygwin -# libxxx.a static library -# xxx.dll DLL -# libxxx.dll.a import library -# -# Note: user can always override by using the @rule -# This settings have been choosen, so that mingw -# is in line with msvc naming conventions. For -# cygwin the cygwin naming convention has been choosen. - -# Make the "o" suffix used for gcc toolset on all -# platforms -type.set_generated_target_suffix('OBJ', ['gcc'], 'o') -type.set_generated_target_suffix('STATIC_LIB', ['gcc', 'cygwin'], 'a') - -type.set_generated_target_suffix('IMPORT_LIB', ['gcc', 'cygwin'], 'dll.a') -type.set_generated_target_prefix('IMPORT_LIB', ['gcc', 'cygwin'], 'lib') - -__machine_match = re.compile('^([^ ]+)') -__version_match = re.compile('^([0-9.]+)') - -def init(version = None, command = None, options = None): - """ - Initializes the gcc toolset for the given version. If necessary, command may - be used to specify where the compiler is located. The parameter 'options' is a - space-delimited list of options, each one specified as - option-value. Valid option names are: cxxflags, linkflags and - linker-type. Accepted linker-type values are gnu, darwin, osf, hpux or sun - and the default value will be selected based on the current OS. - Example: - using gcc : 3.4 : : foo bar sun ; - """ - - options = to_seq(options) - command = to_seq(command) - - # Information about the gcc command... - # The command. - command = to_seq(common.get_invocation_command('gcc', 'g++', command)) - # The root directory of the tool install. - root = feature.get_values('', options) ; - # The bin directory where to find the command to execute. - bin = None - # The flavor of compiler. - flavor = feature.get_values('', options) - # Autodetect the root and bin dir if not given. - if command: - if not bin: - bin = common.get_absolute_tool_path(command[-1]) - if not root: - root = os.path.dirname(bin) - # Autodetect the version and flavor if not given. - if command: - machine_info = subprocess.Popen(command + ['-dumpmachine'], stdout=subprocess.PIPE).communicate()[0] - machine = __machine_match.search(machine_info).group(1) - - version_info = subprocess.Popen(command + ['-dumpversion'], stdout=subprocess.PIPE).communicate()[0] - version = __version_match.search(version_info).group(1) - if not flavor and machine.find('mingw') != -1: - flavor = 'mingw' - - condition = None - if flavor: - condition = common.check_init_parameters('gcc', None, - ('version', version), - ('flavor', flavor)) - else: - condition = common.check_init_parameters('gcc', None, - ('version', version)) - - if command: - command = command[0] - - common.handle_options('gcc', condition, command, options) - - linker = feature.get_values('', options) - if not linker: - if os_name() == 'OSF': - linker = 'osf' - elif os_name() == 'HPUX': - linker = 'hpux' ; - else: - linker = 'gnu' - - init_link_flags('gcc', linker, condition) - - # If gcc is installed in non-standard location, we'd need to add - # LD_LIBRARY_PATH when running programs created with it (for unit-test/run - # rules). - if command: - # On multilib 64-bit boxes, there are both 32-bit and 64-bit libraries - # and all must be added to LD_LIBRARY_PATH. The linker will pick the - # right onces. Note that we don't provide a clean way to build 32-bit - # binary with 64-bit compiler, but user can always pass -m32 manually. - lib_path = [os.path.join(root, 'bin'), - os.path.join(root, 'lib'), - os.path.join(root, 'lib32'), - os.path.join(root, 'lib64')] - if debug(): - print 'notice: using gcc libraries ::', condition, '::', lib_path - toolset.flags('gcc.link', 'RUN_PATH', condition, lib_path) - - # If it's not a system gcc install we should adjust the various programs as - # needed to prefer using the install specific versions. This is essential - # for correct use of MinGW and for cross-compiling. - - # - The archive builder. - archiver = common.get_invocation_command('gcc', - 'ar', feature.get_values('', options), [bin], path_last=True) - toolset.flags('gcc.archive', '.AR', condition, [archiver]) - if debug(): - print 'notice: using gcc archiver ::', condition, '::', archiver - - # - The resource compiler. - rc_command = common.get_invocation_command_nodefault('gcc', - 'windres', feature.get_values('', options), [bin], path_last=True) - rc_type = feature.get_values('', options) - - if not rc_type: - rc_type = 'windres' - - if not rc_command: - # If we can't find an RC compiler we fallback to a null RC compiler that - # creates empty object files. This allows the same Jamfiles to work - # across the board. The null RC uses the assembler to create the empty - # objects, so configure that. - rc_command = common.get_invocation_command('gcc', 'as', [], [bin], path_last=True) - rc_type = 'null' - rc.configure(rc_command, condition, '' + rc_type) - -###if [ os.name ] = NT -###{ -### # This causes single-line command invocation to not go through .bat files, -### # thus avoiding command-line length limitations. -### JAMSHELL = % ; -###} - -#FIXME: when register_c_compiler is moved to -# generators, these should be updated -builtin.register_c_compiler('gcc.compile.c++', ['CPP'], ['OBJ'], ['gcc']) -builtin.register_c_compiler('gcc.compile.c', ['C'], ['OBJ'], ['gcc']) -builtin.register_c_compiler('gcc.compile.asm', ['ASM'], ['OBJ'], ['gcc']) - -# pch support - -# The compiler looks for a precompiled header in each directory just before it -# looks for the include file in that directory. The name searched for is the -# name specified in the #include directive with ".gch" suffix appended. The -# logic in gcc-pch-generator will make sure that BASE_PCH suffix is appended to -# full name of the header. - -type.set_generated_target_suffix('PCH', ['gcc'], 'gch') - -# GCC-specific pch generator. -class GccPchGenerator(pch.PchGenerator): - - # Inherit the __init__ method - - def run_pch(self, project, name, prop_set, sources): - # Find the header in sources. Ignore any CPP sources. - header = None - for s in sources: - if type.is_derived(s.type, 'H'): - header = s - - # Error handling: Base header file name should be the same as the base - # precompiled header name. - header_name = header.name - header_basename = os.path.basename(header_name).rsplit('.', 1)[0] - if header_basename != name: - location = project.project_module - ###FIXME: - raise Exception() - ### errors.user-error "in" $(location)": pch target name `"$(name)"' should be the same as the base name of header file `"$(header-name)"'" ; - - pch_file = Generator.run(self, project, name, prop_set, [header]) - - # return result of base class and pch-file property as usage-requirements - # FIXME: what about multiple results from generator.run? - return (property_set.create('' + pch_file[0], '-Winvalid-pch'), - pch_file) - - # Calls the base version specifying source's name as the name of the created - # target. As result, the PCH will be named whatever.hpp.gch, and not - # whatever.gch. - def generated_targets(self, sources, prop_set, project, name = None): - name = sources[0].name - return Generator.generated_targets(self, sources, - prop_set, project, name) - -# Note: the 'H' source type will catch both '.h' header and '.hpp' header. The -# latter have HPP type, but HPP type is derived from H. The type of compilation -# is determined entirely by the destination type. -generators.register(GccPchGenerator('gcc.compile.c.pch', False, ['H'], ['C_PCH'], ['on', 'gcc' ])) -generators.register(GccPchGenerator('gcc.compile.c++.pch', False, ['H'], ['CPP_PCH'], ['on', 'gcc' ])) - -# Override default do-nothing generators. -generators.override('gcc.compile.c.pch', 'pch.default-c-pch-generator') -generators.override('gcc.compile.c++.pch', 'pch.default-cpp-pch-generator') - -flags('gcc.compile', 'PCH_FILE', ['on'], ['']) - -# Declare flags and action for compilation -flags('gcc.compile', 'OPTIONS', ['off'], ['-O0']) -flags('gcc.compile', 'OPTIONS', ['speed'], ['-O3']) -flags('gcc.compile', 'OPTIONS', ['space'], ['-Os']) - -flags('gcc.compile', 'OPTIONS', ['off'], ['-fno-inline']) -flags('gcc.compile', 'OPTIONS', ['on'], ['-Wno-inline']) -flags('gcc.compile', 'OPTIONS', ['full'], ['-finline-functions', '-Wno-inline']) - -flags('gcc.compile', 'OPTIONS', ['off'], ['-w']) -flags('gcc.compile', 'OPTIONS', ['on'], ['-Wall']) -flags('gcc.compile', 'OPTIONS', ['all'], ['-Wall', '-pedantic']) -flags('gcc.compile', 'OPTIONS', ['on'], ['-Werror']) - -flags('gcc.compile', 'OPTIONS', ['on'], ['-g']) -flags('gcc.compile', 'OPTIONS', ['on'], ['-pg']) -flags('gcc.compile', 'OPTIONS', ['off'], ['-fno-rtti']) - -# On cygwin and mingw, gcc generates position independent code by default, and -# warns if -fPIC is specified. This might not be the right way of checking if -# we're using cygwin. For example, it's possible to run cygwin gcc from NT -# shell, or using crosscompiling. But we'll solve that problem when it's time. -# In that case we'll just add another parameter to 'init' and move this login -# inside 'init'. -if not os_name () in ['CYGWIN', 'NT']: - # This logic will add -fPIC for all compilations: - # - # lib a : a.cpp b ; - # obj b : b.cpp ; - # exe c : c.cpp a d ; - # obj d : d.cpp ; - # - # This all is fine, except that 'd' will be compiled with -fPIC even though - # it's not needed, as 'd' is used only in exe. However, it's hard to detect - # where a target is going to be used. Alternative, we can set -fPIC only - # when main target type is LIB but than 'b' will be compiled without -fPIC. - # In x86-64 that will lead to link errors. So, compile everything with - # -fPIC. - # - # Yet another alternative would be to create propagated - # feature, and set it when building shared libraries, but that's hard to - # implement and will increase target path length even more. - flags('gcc.compile', 'OPTIONS', ['shared'], ['-fPIC']) - -if os_name() != 'NT' and os_name() != 'OSF' and os_name() != 'HPUX': - # OSF does have an option called -soname but it doesn't seem to work as - # expected, therefore it has been disabled. - HAVE_SONAME = '' - SONAME_OPTION = '-h' - - -flags('gcc.compile', 'USER_OPTIONS', [], ['']) -flags('gcc.compile.c++', 'USER_OPTIONS',[], ['']) -flags('gcc.compile', 'DEFINES', [], ['']) -flags('gcc.compile', 'INCLUDES', [], ['']) - -engine = get_manager().engine() - -engine.register_action('gcc.compile.c++.pch', - '"$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"') - -engine.register_action('gcc.compile.c.pch', - '"$(CONFIG_COMMAND)" -x c-header $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"') - - -def gcc_compile_cpp(targets, sources, properties): - # Some extensions are compiled as C++ by default. For others, we need to - # pass -x c++. We could always pass -x c++ but distcc does not work with it. - extension = os.path.splitext (sources [0]) [1] - lang = '' - if not extension in ['.cc', '.cp', '.cxx', '.cpp', '.c++', '.C']: - lang = '-x c++' - get_manager().engine().set_target_variable (targets, 'LANG', lang) - engine.add_dependency(targets, bjam.call('get-target-variable', targets, 'PCH_FILE')) - -def gcc_compile_c(targets, sources, properties): - engine = get_manager().engine() - # If we use the name g++ then default file suffix -> language mapping does - # not work. So have to pass -x option. Maybe, we can work around this by - # allowing the user to specify both C and C++ compiler names. - #if $(>:S) != .c - #{ - engine.set_target_variable (targets, 'LANG', '-x c') - #} - engine.add_dependency(targets, bjam.call('get-target-variable', targets, 'PCH_FILE')) - -engine.register_action( - 'gcc.compile.c++', - '"$(CONFIG_COMMAND)" $(LANG) -ftemplate-depth-128 $(OPTIONS) ' + - '$(USER_OPTIONS) -D$(DEFINES) -I"$(PCH_FILE:D)" -I"$(INCLUDES)" ' + - '-c -o "$(<:W)" "$(>:W)"', - function=gcc_compile_cpp, - bound_list=['PCH_FILE']) - -engine.register_action( - 'gcc.compile.c', - '"$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) ' + - '-I"$(PCH_FILE:D)" -I"$(INCLUDES)" -c -o "$(<)" "$(>)"', - function=gcc_compile_c, - bound_list=['PCH_FILE']) - -def gcc_compile_asm(targets, sources, properties): - get_manager().engine().set_target_variable(targets, 'LANG', '-x assembler-with-cpp') - -engine.register_action( - 'gcc.compile.asm', - '"$(CONFIG_COMMAND)" $(LANG) $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"', - function=gcc_compile_asm) - - -class GccLinkingGenerator(unix.UnixLinkingGenerator): - """ - The class which check that we don't try to use the static - property while creating or using shared library, since it's not supported by - gcc/libc. - """ - def run(self, project, name, ps, sources): - # TODO: Replace this with the use of a target-os property. - - no_static_link = False - if bjam.variable('UNIX'): - no_static_link = True; - ##FIXME: what does this mean? -## { -## switch [ modules.peek : JAMUNAME ] -## { -## case * : no-static-link = true ; -## } -## } - - reason = None - if no_static_link and ps.get('runtime-link') == 'static': - if ps.get('link') == 'shared': - reason = "On gcc, DLL can't be build with 'static'." - elif type.is_derived(self.target_types[0], 'EXE'): - for s in sources: - source_type = s.type() - if source_type and type.is_derived(source_type, 'SHARED_LIB'): - reason = "On gcc, using DLLS together with the " +\ - "static options is not possible " - if reason: - print 'warning:', reason - print 'warning:',\ - "It is suggested to use 'static' together",\ - "with 'static'." ; - return - else: - generated_targets = unix.UnixLinkingGenerator.run(self, project, - name, ps, sources) - return generated_targets - -if on_windows(): - flags('gcc.link.dll', '.IMPLIB-COMMAND', [], ['-Wl,--out-implib,']) - generators.register( - GccLinkingGenerator('gcc.link', True, - ['OBJ', 'SEARCHED_LIB', 'STATIC_LIB', 'IMPORT_LIB'], - [ 'EXE' ], - [ 'gcc' ])) - generators.register( - GccLinkingGenerator('gcc.link.dll', True, - ['OBJ', 'SEARCHED_LIB', 'STATIC_LIB', 'IMPORT_LIB'], - ['IMPORT_LIB', 'SHARED_LIB'], - ['gcc'])) -else: - generators.register( - GccLinkingGenerator('gcc.link', True, - ['LIB', 'OBJ'], - ['EXE'], - ['gcc'])) - generators.register( - GccLinkingGenerator('gcc.link.dll', True, - ['LIB', 'OBJ'], - ['SHARED_LIB'], - ['gcc'])) - -# Declare flags for linking. -# First, the common flags. -flags('gcc.link', 'OPTIONS', ['on'], ['-g']) -flags('gcc.link', 'OPTIONS', ['on'], ['-pg']) -flags('gcc.link', 'USER_OPTIONS', [], ['']) -flags('gcc.link', 'LINKPATH', [], ['']) -flags('gcc.link', 'FINDLIBS-ST', [], ['']) -flags('gcc.link', 'FINDLIBS-SA', [], ['']) -flags('gcc.link', 'LIBRARIES', [], ['']) - -# For static we made sure there are no dynamic libraries in the -# link. On HP-UX not all system libraries exist as archived libraries (for -# example, there is no libunwind.a), so, on this platform, the -static option -# cannot be specified. -if os_name() != 'HPUX': - flags('gcc.link', 'OPTIONS', ['static'], ['-static']) - -# Now, the vendor specific flags. -# The parameter linker can be either gnu, darwin, osf, hpux or sun. -def init_link_flags(toolset, linker, condition): - """ - Now, the vendor specific flags. - The parameter linker can be either gnu, darwin, osf, hpux or sun. - """ - toolset_link = toolset + '.link' - if linker == 'gnu': - # Strip the binary when no debugging is needed. We use --strip-all flag - # as opposed to -s since icc (intel's compiler) is generally - # option-compatible with and inherits from the gcc toolset, but does not - # support -s. - - # FIXME: what does unchecked translate to? - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/off', condition), ['-Wl,--strip-all']) # : unchecked ; - flags(toolset_link, 'RPATH', condition, ['']) # : unchecked ; - flags(toolset_link, 'RPATH_LINK', condition, ['']) # : unchecked ; - flags(toolset_link, 'START-GROUP', condition, ['-Wl,--start-group'])# : unchecked ; - flags(toolset_link, 'END-GROUP', condition, ['-Wl,--end-group']) # : unchecked ; - - # gnu ld has the ability to change the search behaviour for libraries - # referenced by -l switch. These modifiers are -Bstatic and -Bdynamic - # and change search for -l switches that follow them. The following list - # shows the tried variants. - # The search stops at the first variant that has a match. - # *nix: -Bstatic -lxxx - # libxxx.a - # - # *nix: -Bdynamic -lxxx - # libxxx.so - # libxxx.a - # - # windows (mingw,cygwin) -Bstatic -lxxx - # libxxx.a - # xxx.lib - # - # windows (mingw,cygwin) -Bdynamic -lxxx - # libxxx.dll.a - # xxx.dll.a - # libxxx.a - # xxx.lib - # cygxxx.dll (*) - # libxxx.dll - # xxx.dll - # libxxx.a - # - # (*) This is for cygwin - # Please note that -Bstatic and -Bdynamic are not a guarantee that a - # static or dynamic lib indeed gets linked in. The switches only change - # search patterns! - - # On *nix mixing shared libs with static runtime is not a good idea. - flags(toolset_link, 'FINDLIBS-ST-PFX', - map(lambda x: x + '/shared', condition), - ['-Wl,-Bstatic']) # : unchecked ; - flags(toolset_link, 'FINDLIBS-SA-PFX', - map(lambda x: x + '/shared', condition), - ['-Wl,-Bdynamic']) # : unchecked ; - - # On windows allow mixing of static and dynamic libs with static - # runtime. - flags(toolset_link, 'FINDLIBS-ST-PFX', - map(lambda x: x + '/static/windows', condition), - ['-Wl,-Bstatic']) # : unchecked ; - flags(toolset_link, 'FINDLIBS-SA-PFX', - map(lambda x: x + '/static/windows', condition), - ['-Wl,-Bdynamic']) # : unchecked ; - flags(toolset_link, 'OPTIONS', - map(lambda x: x + '/static/windows', condition), - ['-Wl,-Bstatic']) # : unchecked ; - - elif linker == 'darwin': - # On Darwin, the -s option to ld does not work unless we pass -static, - # and passing -static unconditionally is a bad idea. So, don't pass -s. - # at all, darwin.jam will use separate 'strip' invocation. - flags(toolset_link, 'RPATH', condition, ['']) # : unchecked ; - flags(toolset_link, 'RPATH_LINK', condition, ['']) # : unchecked ; - - elif linker == 'osf': - # No --strip-all, just -s. - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/off', condition), ['-Wl,-s']) - # : unchecked ; - flags(toolset_link, 'RPATH', condition, ['']) # : unchecked ; - # This does not supports -R. - flags(toolset_link, 'RPATH_OPTION', condition, ['-rpath']) # : unchecked ; - # -rpath-link is not supported at all. - - elif linker == 'sun': - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/off', condition), ['-Wl,-s']) - # : unchecked ; - flags(toolset_link, 'RPATH', condition, ['']) # : unchecked ; - # Solaris linker does not have a separate -rpath-link, but allows to use - # -L for the same purpose. - flags(toolset_link, 'LINKPATH', condition, ['']) # : unchecked ; - - # This permits shared libraries with non-PIC code on Solaris. - # VP, 2004/09/07: Now that we have -fPIC hardcode in link.dll, the - # following is not needed. Whether -fPIC should be hardcoded, is a - # separate question. - # AH, 2004/10/16: it is still necessary because some tests link against - # static libraries that were compiled without PIC. - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/shared', condition), ['-mimpure-text']) - # : unchecked ; - - elif linker == 'hpux': - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/off', condition), - ['-Wl,-s']) # : unchecked ; - flags(toolset_link, 'OPTIONS', map(lambda x: x + '/shared', condition), - ['-fPIC']) # : unchecked ; - - else: - # FIXME: - errors.user_error( - "$(toolset) initialization: invalid linker '$(linker)' " + - "The value '$(linker)' specified for is not recognized. " + - "Possible values are 'gnu', 'darwin', 'osf', 'hpux' or 'sun'") - -# Declare actions for linking. -def gcc_link(targets, sources, properties): - engine = get_manager().engine() - engine.set_target_variable(targets, 'SPACE', ' ') - # Serialize execution of the 'link' action, since running N links in - # parallel is just slower. For now, serialize only gcc links, it might be a - # good idea to serialize all links. - engine.set_target_variable(targets, 'JAM_SEMAPHORE', 'gcc-link-semaphore') - -engine.register_action( - 'gcc.link', - '"$(CONFIG_COMMAND)" -L"$(LINKPATH)" ' + - '-Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,"$(RPATH)" ' + - '-Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" ' + - '$(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) ' + - '-l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) ' + - '$(OPTIONS) $(USER_OPTIONS)', - function=gcc_link, - bound_list=['LIBRARIES']) - -# Default value. Mostly for the sake of intel-linux that inherits from gcc, but -# does not have the same logic to set the .AR variable. We can put the same -# logic in intel-linux, but that's hardly worth the trouble as on Linux, 'ar' is -# always available. -__AR = 'ar' - -flags('gcc.archive', 'AROPTIONS', [], ['']) - -def gcc_archive(targets, sources, properties): - # Always remove archive and start again. Here's rationale from - # - # Andre Hentz: - # - # I had a file, say a1.c, that was included into liba.a. I moved a1.c to - # a2.c, updated my Jamfiles and rebuilt. My program was crashing with absurd - # errors. After some debugging I traced it back to the fact that a1.o was - # *still* in liba.a - # - # Rene Rivera: - # - # Originally removing the archive was done by splicing an RM onto the - # archive action. That makes archives fail to build on NT when they have - # many files because it will no longer execute the action directly and blow - # the line length limit. Instead we remove the file in a different action, - # just before building the archive. - clean = targets[0] + '(clean)' - bjam.call('TEMPORARY', clean) - bjam.call('NOCARE', clean) - engine = get_manager().engine() - engine.set_target_variable('LOCATE', clean, bjam.call('get-target-variable', targets, 'LOCATE')) - engine.add_dependency(clean, sources) - engine.add_dependency(targets, clean) - engine.set_update_action('common.RmTemps', clean, targets) - -# Declare action for creating static libraries. -# The letter 'r' means to add files to the archive with replacement. Since we -# remove archive, we don't care about replacement, but there's no option "add -# without replacement". -# The letter 'c' suppresses the warning in case the archive does not exists yet. -# That warning is produced only on some platforms, for whatever reasons. -engine.register_action('gcc.archive', - '"$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)"', - function=gcc_archive, - flags=['piecemeal']) - -def gcc_link_dll(targets, sources, properties): - engine = get_manager().engine() - engine.set_target_variable(targets, 'SPACE', ' ') - engine.set_target_variable(targets, 'JAM_SEMAPHORE', 'gcc-link-semaphore') - engine.set_target_variable(targets, "HAVE_SONAME", HAVE_SONAME) - engine.set_target_variable(targets, "SONAME_OPTION", SONAME_OPTION) - -engine.register_action( - 'gcc.link.dll', - # Differ from 'link' above only by -shared. - '"$(CONFIG_COMMAND)" -L"$(LINKPATH)" ' + - '-Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,"$(RPATH)" ' + - '"$(.IMPLIB-COMMAND)$(<[1])" -o "$(<[-1])" ' + - '$(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,$(<[-1]:D=) ' + - '-shared $(START-GROUP) "$(>)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) ' + - '-l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) ' + - '$(OPTIONS) $(USER_OPTIONS)', - function = gcc_link_dll, - bound_list=['LIBRARIES']) - -# Set up threading support. It's somewhat contrived, so perform it at the end, -# to avoid cluttering other code. - -if on_windows(): - flags('gcc', 'OPTIONS', ['multi'], ['-mthreads']) -elif bjam.variable('UNIX'): - jamuname = bjam.variable('JAMUNAME') - host_os_name = jamuname[0] - if host_os_name.startswith('SunOS'): - flags('gcc', 'OPTIONS', ['multi'], ['-pthreads']) - flags('gcc', 'FINDLIBS-SA', [], ['rt']) - elif host_os_name == 'BeOS': - # BeOS has no threading options, don't set anything here. - pass - elif host_os_name.endswith('BSD'): - flags('gcc', 'OPTIONS', ['multi'], ['-pthread']) - # there is no -lrt on BSD - elif host_os_name == 'DragonFly': - flags('gcc', 'OPTIONS', ['multi'], ['-pthread']) - # there is no -lrt on BSD - DragonFly is a FreeBSD variant, - # which anoyingly doesn't say it's a *BSD. - elif host_os_name == 'IRIX': - # gcc on IRIX does not support multi-threading, don't set anything here. - pass - elif host_os_name == 'Darwin': - # Darwin has no threading options, don't set anything here. - pass - else: - flags('gcc', 'OPTIONS', ['multi'], ['-pthread']) - flags('gcc', 'FINDLIBS-SA', [], ['rt']) - -def cpu_flags(toolset, variable, architecture, instruction_set, values, default=None): - #FIXME: for some reason this fails. Probably out of date feature code -## if default: -## flags(toolset, variable, -## ['' + architecture + '/'], -## values) - flags(toolset, variable, - #FIXME: same as above - [##'/' + instruction_set, - '' + architecture + '/' + instruction_set], - values) - -# Set architecture/instruction-set options. -# -# x86 and compatible -flags('gcc', 'OPTIONS', ['x86/32'], ['-m32']) -flags('gcc', 'OPTIONS', ['x86/64'], ['-m64']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'i386', ['-march=i386'], default=True) -cpu_flags('gcc', 'OPTIONS', 'x86', 'i486', ['-march=i486']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'i586', ['-march=i586']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'i686', ['-march=i686']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium', ['-march=pentium']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium-mmx', ['-march=pentium-mmx']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentiumpro', ['-march=pentiumpro']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium2', ['-march=pentium2']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium3', ['-march=pentium3']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium3m', ['-march=pentium3m']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium-m', ['-march=pentium-m']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium4', ['-march=pentium4']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'pentium4m', ['-march=pentium4m']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'prescott', ['-march=prescott']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'nocona', ['-march=nocona']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'k6', ['-march=k6']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'k6-2', ['-march=k6-2']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'k6-3', ['-march=k6-3']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon', ['-march=athlon']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon-tbird', ['-march=athlon-tbird']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon-4', ['-march=athlon-4']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon-xp', ['-march=athlon-xp']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon-mp', ['-march=athlon-mp']) -## -cpu_flags('gcc', 'OPTIONS', 'x86', 'k8', ['-march=k8']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'opteron', ['-march=opteron']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon64', ['-march=athlon64']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'athlon-fx', ['-march=athlon-fx']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'winchip-c6', ['-march=winchip-c6']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'winchip2', ['-march=winchip2']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'c3', ['-march=c3']) -cpu_flags('gcc', 'OPTIONS', 'x86', 'c3-2', ['-march=c3-2']) -# Sparc -flags('gcc', 'OPTIONS', ['sparc/32'], ['-m32']) -flags('gcc', 'OPTIONS', ['sparc/64'], ['-m64']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'c3', ['-mcpu=c3'], default=True) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'v7', ['-mcpu=v7']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'cypress', ['-mcpu=cypress']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'v8', ['-mcpu=v8']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'supersparc', ['-mcpu=supersparc']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'sparclite', ['-mcpu=sparclite']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'hypersparc', ['-mcpu=hypersparc']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'sparclite86x', ['-mcpu=sparclite86x']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'f930', ['-mcpu=f930']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'f934', ['-mcpu=f934']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'sparclet', ['-mcpu=sparclet']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'tsc701', ['-mcpu=tsc701']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'v9', ['-mcpu=v9']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'ultrasparc', ['-mcpu=ultrasparc']) -cpu_flags('gcc', 'OPTIONS', 'sparc', 'ultrasparc3', ['-mcpu=ultrasparc3']) -# RS/6000 & PowerPC -flags('gcc', 'OPTIONS', ['power/32'], ['-m32']) -flags('gcc', 'OPTIONS', ['power/64'], ['-m64']) -cpu_flags('gcc', 'OPTIONS', 'power', '403', ['-mcpu=403']) -cpu_flags('gcc', 'OPTIONS', 'power', '505', ['-mcpu=505']) -cpu_flags('gcc', 'OPTIONS', 'power', '601', ['-mcpu=601']) -cpu_flags('gcc', 'OPTIONS', 'power', '602', ['-mcpu=602']) -cpu_flags('gcc', 'OPTIONS', 'power', '603', ['-mcpu=603']) -cpu_flags('gcc', 'OPTIONS', 'power', '603e', ['-mcpu=603e']) -cpu_flags('gcc', 'OPTIONS', 'power', '604', ['-mcpu=604']) -cpu_flags('gcc', 'OPTIONS', 'power', '604e', ['-mcpu=604e']) -cpu_flags('gcc', 'OPTIONS', 'power', '620', ['-mcpu=620']) -cpu_flags('gcc', 'OPTIONS', 'power', '630', ['-mcpu=630']) -cpu_flags('gcc', 'OPTIONS', 'power', '740', ['-mcpu=740']) -cpu_flags('gcc', 'OPTIONS', 'power', '7400', ['-mcpu=7400']) -cpu_flags('gcc', 'OPTIONS', 'power', '7450', ['-mcpu=7450']) -cpu_flags('gcc', 'OPTIONS', 'power', '750', ['-mcpu=750']) -cpu_flags('gcc', 'OPTIONS', 'power', '801', ['-mcpu=801']) -cpu_flags('gcc', 'OPTIONS', 'power', '821', ['-mcpu=821']) -cpu_flags('gcc', 'OPTIONS', 'power', '823', ['-mcpu=823']) -cpu_flags('gcc', 'OPTIONS', 'power', '860', ['-mcpu=860']) -cpu_flags('gcc', 'OPTIONS', 'power', '970', ['-mcpu=970']) -cpu_flags('gcc', 'OPTIONS', 'power', '8540', ['-mcpu=8540']) -cpu_flags('gcc', 'OPTIONS', 'power', 'power', ['-mcpu=power']) -cpu_flags('gcc', 'OPTIONS', 'power', 'power2', ['-mcpu=power2']) -cpu_flags('gcc', 'OPTIONS', 'power', 'power3', ['-mcpu=power3']) -cpu_flags('gcc', 'OPTIONS', 'power', 'power4', ['-mcpu=power4']) -cpu_flags('gcc', 'OPTIONS', 'power', 'power5', ['-mcpu=power5']) -cpu_flags('gcc', 'OPTIONS', 'power', 'powerpc', ['-mcpu=powerpc']) -cpu_flags('gcc', 'OPTIONS', 'power', 'powerpc64', ['-mcpu=powerpc64']) -cpu_flags('gcc', 'OPTIONS', 'power', 'rios', ['-mcpu=rios']) -cpu_flags('gcc', 'OPTIONS', 'power', 'rios1', ['-mcpu=rios1']) -cpu_flags('gcc', 'OPTIONS', 'power', 'rios2', ['-mcpu=rios2']) -cpu_flags('gcc', 'OPTIONS', 'power', 'rsc', ['-mcpu=rsc']) -cpu_flags('gcc', 'OPTIONS', 'power', 'rs64a', ['-mcpu=rs64']) -# AIX variant of RS/6000 & PowerPC -flags('gcc', 'OPTIONS', ['power/32/aix'], ['-maix32']) -flags('gcc', 'OPTIONS', ['power/64/aix'], ['-maix64']) -flags('gcc', 'AROPTIONS', ['power/64/aix'], ['-X 64']) diff --git a/jam-files/boost-build/tools/generate.jam b/jam-files/boost-build/tools/generate.jam deleted file mode 100644 index 6732fa35..00000000 --- a/jam-files/boost-build/tools/generate.jam +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Declares main target 'generate' used to produce targets by calling a -# user-provided rule that takes and produces virtual targets. - -import "class" : new ; -import errors ; -import feature ; -import project ; -import property ; -import property-set ; -import targets ; -import regex ; - - -feature.feature generating-rule : : free ; - - -class generated-target-class : basic-target -{ - import errors ; - import indirect ; - import virtual-target ; - - rule __init__ ( name : project : sources * : requirements * - : default-build * : usage-requirements * ) - { - basic-target.__init__ $(name) : $(project) : $(sources) - : $(requirements) : $(default-build) : $(usage-requirements) ; - - if ! [ $(self.requirements).get ] - { - errors.user-error "The generate rule requires the " - "property to be set" ; - } - } - - rule construct ( name : sources * : property-set ) - { - local result ; - local gr = [ $(property-set).get ] ; - - # FIXME: this is a copy-paste from virtual-target.jam. We should add a - # utility rule to call a rule like this. - local rule-name = [ MATCH ^@(.*) : $(gr) ] ; - if $(rule-name) - { - if $(gr[2]) - { - local target-name = [ full-name ] ; - errors.user-error "Multiple properties" - "encountered for target $(target-name)." ; - } - - result = [ indirect.call $(rule-name) $(self.project) $(name) - : $(property-set) : $(sources) ] ; - - if ! $(result) - { - ECHO "warning: Unable to construct" [ full-name ] ; - } - } - - local ur ; - local targets ; - - if $(result) - { - if [ class.is-a $(result[1]) : property-set ] - { - ur = $(result[1]) ; - targets = $(result[2-]) ; - } - else - { - ur = [ property-set.empty ] ; - targets = $(result) ; - } - } - # FIXME: the following loop should be doable using sequence.transform or - # some similar utility rule. - local rt ; - for local t in $(targets) - { - rt += [ virtual-target.register $(t) ] ; - } - return $(ur) $(rt) ; - } -} - - -rule generate ( name : sources * : requirements * : default-build * - : usage-requirements * ) -{ - local project = [ project.current ] ; - - targets.main-target-alternative - [ new generated-target-class $(name) : $(project) - : [ targets.main-target-sources $(sources) : $(name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - : [ targets.main-target-usage-requirements $(usage-requirements) : $(project) ] - ] ; -} - -IMPORT $(__name__) : generate : : generate ; diff --git a/jam-files/boost-build/tools/gettext.jam b/jam-files/boost-build/tools/gettext.jam deleted file mode 100644 index 99a43ffe..00000000 --- a/jam-files/boost-build/tools/gettext.jam +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module support GNU gettext internationalization utilities. -# -# It provides two main target rules: 'gettext.catalog', used for -# creating machine-readable catalogs from translations files, and -# 'gettext.update', used for update translation files from modified -# sources. -# -# To add i18n support to your application you should follow these -# steps. -# -# - Decide on a file name which will contain translations and -# what main target name will be used to update it. For example:: -# -# gettext.update update-russian : russian.po a.cpp my_app ; -# -# - Create the initial translation file by running:: -# -# bjam update-russian -# -# - Edit russian.po. For example, you might change fields like LastTranslator. -# -# - Create a main target for final message catalog:: -# -# gettext.catalog russian : russian.po ; -# -# The machine-readable catalog will be updated whenever you update -# "russian.po". The "russian.po" file will be updated only on explicit -# request. When you're ready to update translations, you should -# -# - Run:: -# -# bjam update-russian -# -# - Edit "russian.po" in appropriate editor. -# -# The next bjam run will convert "russian.po" into machine-readable form. -# -# By default, translations are marked by 'i18n' call. The 'gettext.keyword' -# feature can be used to alter this. - - -import targets ; -import property-set ; -import virtual-target ; -import "class" : new ; -import project ; -import type ; -import generators ; -import errors ; -import feature : feature ; -import toolset : flags ; -import regex ; - -.path = "" ; - -# Initializes the gettext module. -rule init ( path ? # Path where all tools are located. If not specified, - # they should be in PATH. - ) -{ - if $(.initialized) && $(.path) != $(path) - { - errors.error "Attempt to reconfigure with different path" ; - } - .initialized = true ; - if $(path) - { - .path = $(path)/ ; - } -} - -# Creates a main target 'name', which, when updated, will cause -# file 'existing-translation' to be updated with translations -# extracted from 'sources'. It's possible to specify main target -# in sources --- it which case all target from dependency graph -# of those main targets will be scanned, provided they are of -# appropricate type. The 'gettext.types' feature can be used to -# control the types. -# -# The target will be updated only if explicitly requested on the -# command line. -rule update ( name : existing-translation sources + : requirements * ) -{ - local project = [ project.current ] ; - - targets.main-target-alternative - [ new typed-target $(name) : $(project) : gettext.UPDATE : - $(existing-translation) $(sources) - : [ targets.main-target-requirements $(requirements) : $(project) ] - ] ; - $(project).mark-target-as-explicit $(name) ; -} - - -# The human editable source, containing translation. -type.register gettext.PO : po ; -# The machine readable message catalog. -type.register gettext.catalog : mo ; -# Intermediate type produce by extracting translations from -# sources. -type.register gettext.POT : pot ; -# Pseudo type used to invoke update-translations generator -type.register gettext.UPDATE ; - -# Identifies the keyword that should be used when scanning sources. -# Default: i18n -feature gettext.keyword : : free ; -# Contains space-separated list of sources types which should be scanned. -# Default: "C CPP" -feature gettext.types : : free ; - -generators.register-standard gettext.compile : gettext.PO : gettext.catalog ; - -class update-translations-generator : generator -{ - import regex : split ; - import property-set ; - - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - # The rule should be called with at least two sources. The first source - # is the translation (.po) file to update. The remaining sources are targets - # which should be scanned for new messages. All sources files for those targets - # will be found and passed to the 'xgettext' utility, which extracts the - # messages for localization. Those messages will be merged to the .po file. - rule run ( project name ? : property-set : sources * : multiple ? ) - { - local types = [ $(property-set).get ] ; - types ?= "C CPP" ; - types = [ regex.split $(types) " " ] ; - - local keywords = [ $(property-set).get ] ; - property-set = [ property-set.create $(keywords:G=) ] ; - - # First deterime the list of sources that must be scanned for - # messages. - local all-sources ; - # CONSIDER: I'm not sure if the logic should be the same as for 'stage': - # i.e. following dependency properties as well. - for local s in $(sources[2-]) - { - all-sources += [ virtual-target.traverse $(s) : : include-sources ] ; - } - local right-sources ; - for local s in $(all-sources) - { - if [ $(s).type ] in $(types) - { - right-sources += $(s) ; - } - } - - local .constructed ; - if $(right-sources) - { - # Create the POT file, which will contain list of messages extracted - # from the sources. - local extract = - [ new action $(right-sources) : gettext.extract : $(property-set) ] ; - local new-messages = [ new file-target $(name) : gettext.POT - : $(project) : $(extract) ] ; - - # Create a notfile target which will update the existing translation file - # with new messages. - local a = [ new action $(sources[1]) $(new-messages) - : gettext.update-po-dispatch ] ; - local r = [ new notfile-target $(name) : $(project) : $(a) ] ; - .constructed = [ virtual-target.register $(r) ] ; - } - else - { - errors.error "No source could be scanned by gettext tools" ; - } - return $(.constructed) ; - } -} -generators.register [ new update-translations-generator gettext.update : : gettext.UPDATE ] ; - -flags gettext.extract KEYWORD ; -actions extract -{ - $(.path)xgettext -k$(KEYWORD:E=i18n) -o $(<) $(>) -} - -# Does realy updating of po file. The tricky part is that -# we're actually updating one of the sources: -# $(<) is the NOTFILE target we're updating -# $(>[1]) is the PO file to be really updated. -# $(>[2]) is the PO file created from sources. -# -# When file to be updated does not exist (during the -# first run), we need to copy the file created from sources. -# In all other cases, we need to update the file. -rule update-po-dispatch -{ - NOCARE $(>[1]) ; - gettext.create-po $(<) : $(>) ; - gettext.update-po $(<) : $(>) ; - _ on $(<) = " " ; - ok on $(<) = "" ; - EXISTING_PO on $(<) = $(>[1]) ; -} - -# Due to fancy interaction of existing and updated, this rule can be called with -# one source, in which case we copy the lonely source into EXISTING_PO, or with -# two sources, in which case the action body expands to nothing. I'd really like -# to have "missing" action modifier. -actions quietly existing updated create-po bind EXISTING_PO -{ - cp$(_)"$(>[1])"$(_)"$(EXISTING_PO)"$($(>[2]:E=ok)) -} - -actions updated update-po bind EXISTING_PO -{ - $(.path)msgmerge$(_)-U$(_)"$(EXISTING_PO)"$(_)"$(>[1])" -} - -actions gettext.compile -{ - $(.path)msgfmt -o $(<) $(>) -} - -IMPORT $(__name__) : update : : gettext.update ; diff --git a/jam-files/boost-build/tools/gfortran.jam b/jam-files/boost-build/tools/gfortran.jam deleted file mode 100644 index 0aa69b85..00000000 --- a/jam-files/boost-build/tools/gfortran.jam +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2004 Toon Knapen -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import toolset : flags ; -import feature ; -import fortran ; - -rule init ( version ? : command * : options * ) -{ -} - -# Declare flags and action for compilation -flags gfortran OPTIONS ; - -flags gfortran OPTIONS off : -O0 ; -flags gfortran OPTIONS speed : -O3 ; -flags gfortran OPTIONS space : -Os ; - -flags gfortran OPTIONS on : -g ; -flags gfortran OPTIONS on : -pg ; - -flags gfortran OPTIONS shared/LIB : -fPIC ; - -flags gfortran DEFINES ; -flags gfortran INCLUDES ; - -rule compile.fortran -{ -} - -actions compile.fortran -{ - gcc -Wall $(OPTIONS) -D$(DEFINES) -I$(INCLUDES) -c -o "$(<)" "$(>)" -} - -generators.register-fortran-compiler gfortran.compile.fortran : FORTRAN FORTRAN90 : OBJ ; diff --git a/jam-files/boost-build/tools/hp_cxx.jam b/jam-files/boost-build/tools/hp_cxx.jam deleted file mode 100644 index 86cd783e..00000000 --- a/jam-files/boost-build/tools/hp_cxx.jam +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2001 David Abrahams. -# Copyright 2004, 2005 Markus Schoepflin. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# -# HP CXX compiler -# See http://h30097.www3.hp.com/cplus/?jumpid=reg_R1002_USEN -# -# -# Notes on this toolset: -# -# - Because of very subtle issues with the default ansi mode, strict_ansi mode -# is used for compilation. One example of things that don't work correctly in -# the default ansi mode is overload resolution of function templates when -# mixed with non-template functions. -# -# - For template instantiation "-timplicit_local" is used. Previously, -# "-tlocal" has been tried to avoid the need for a template repository -# but this doesn't work with manually instantiated templates. "-tweak" -# has not been used to avoid the stream of warning messages issued by -# ar or ld when creating a library or linking an application. -# -# - Debug symbols are generated with "-g3", as this works both in debug and -# release mode. When compiling C++ code without optimization, we additionally -# use "-gall", which generates full symbol table information for all classes, -# structs, and unions. As this turns off optimization, it can't be used when -# optimization is needed. -# - -import feature generators common ; -import toolset : flags ; - -feature.extend toolset : hp_cxx ; -feature.extend c++abi : cxxarm ; - -# Inherit from Unix toolset to get library ordering magic. -toolset.inherit hp_cxx : unix ; - -generators.override hp_cxx.prebuilt : builtin.lib-generator ; -generators.override hp_cxx.prebuilt : builtin.prebuilt ; -generators.override hp_cxx.searched-lib-generator : searched-lib-generator ; - - -rule init ( version ? : command * : options * ) -{ - local condition = [ common.check-init-parameters hp_cxx : version $(version) ] ; - - local command = [ common.get-invocation-command hp_cxx : cxx : $(command) ] ; - - if $(command) - { - local root = [ common.get-absolute-tool-path $(command[-1]) ] ; - - if $(root) - { - flags hp_cxx .root $(condition) : "\"$(root)\"/" ; - } - } - # If we can't find 'cxx' anyway, at least show 'cxx' in the commands - command ?= cxx ; - - common.handle-options hp_cxx : $(condition) : $(command) : $(options) ; -} - -generators.register-c-compiler hp_cxx.compile.c++ : CPP : OBJ : hp_cxx ; -generators.register-c-compiler hp_cxx.compile.c : C : OBJ : hp_cxx ; - - - -# No static linking as far as I can tell. -# flags cxx LINKFLAGS static : -bstatic ; -flags hp_cxx.compile OPTIONS on : -g3 ; -flags hp_cxx.compile OPTIONS off/on : -gall ; -flags hp_cxx.link OPTIONS on : -g ; -flags hp_cxx.link OPTIONS off : -s ; - -flags hp_cxx.compile OPTIONS off : -O0 ; -flags hp_cxx.compile OPTIONS speed/on : -O2 ; -flags hp_cxx.compile OPTIONS speed : -O2 ; - -# This (undocumented) macro needs to be defined to get all C function -# overloads required by the C++ standard. -flags hp_cxx.compile.c++ OPTIONS : -D__CNAME_OVERLOADS ; - -# Added for threading support -flags hp_cxx.compile OPTIONS multi : -pthread ; -flags hp_cxx.link OPTIONS multi : -pthread ; - -flags hp_cxx.compile OPTIONS space/on : size ; -flags hp_cxx.compile OPTIONS space : -O1 ; -flags hp_cxx.compile OPTIONS off : -inline none ; - -# The compiler versions tried (up to V6.5-040) hang when compiling Boost code -# with full inlining enabled. So leave it at the default level for now. -# -# flags hp_cxx.compile OPTIONS full : -inline all ; - -flags hp_cxx.compile OPTIONS on : -pg ; -flags hp_cxx.link OPTIONS on : -pg ; - -# Selection of the object model. This flag is needed on both the C++ compiler -# and linker command line. - -# Unspecified ABI translates to '-model ansi' as most -# standard-conforming. -flags hp_cxx.compile.c++ OPTIONS : -model ansi : : hack-hack ; -flags hp_cxx.compile.c++ OPTIONS cxxarm : -model arm ; -flags hp_cxx.link OPTIONS : -model ansi : : hack-hack ; -flags hp_cxx.link OPTIONS cxxarm : -model arm ; - -# Display a descriptive tag together with each compiler message. This tag can -# be used by the user to explicitely suppress the compiler message. -flags hp_cxx.compile OPTIONS : -msg_display_tag ; - -flags hp_cxx.compile OPTIONS ; -flags hp_cxx.compile.c++ OPTIONS ; -flags hp_cxx.compile DEFINES ; -flags hp_cxx.compile INCLUDES ; -flags hp_cxx.link OPTIONS ; - -flags hp_cxx.link LIBPATH ; -flags hp_cxx.link LIBRARIES ; -flags hp_cxx.link FINDLIBS-ST ; -flags hp_cxx.link FINDLIBS-SA ; - -flags hp_cxx.compile.c++ TEMPLATE_DEPTH ; - -actions link bind LIBRARIES -{ - $(CONFIG_COMMAND) -noimplicit_include $(OPTIONS) -o "$(<)" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) -lrt -lm -} - -# When creating dynamic libraries, we don't want to be warned about unresolved -# symbols, therefore all unresolved symbols are marked as expected by -# '-expect_unresolved *'. This also mirrors the behaviour of the GNU tool -# chain. - -actions link.dll bind LIBRARIES -{ - $(CONFIG_COMMAND) -shared -expect_unresolved \* -noimplicit_include $(OPTIONS) -o "$(<[1])" -L$(LIBPATH) "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) -lm -} - - -# Note: Relaxed ANSI mode (-std) is used for compilation because in strict ANSI -# C89 mode (-std1) the compiler doesn't accept C++ comments in C files. As -std -# is the default, no special flag is needed. -actions compile.c -{ - $(.root:E=)cc -c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)" -} - -# Note: The compiler is forced to compile the files as C++ (-x cxx) because -# otherwise it will silently ignore files with no file extension. -# -# Note: We deliberately don't suppress any warnings on the compiler command -# line, the user can always do this in a customized toolset later on. - -rule compile.c++ -{ - # We preprocess the TEMPLATE_DEPTH command line option here because we found - # no way to do it correctly in the actual action code. There we either get - # the -pending_instantiations parameter when no c++-template-depth property - # has been specified or we get additional quotes around - # "-pending_instantiations ". - local template-depth = [ on $(1) return $(TEMPLATE_DEPTH) ] ; - TEMPLATE_DEPTH on $(1) = "-pending_instantiations "$(template-depth) ; -} - -actions compile.c++ -{ - $(CONFIG_COMMAND) -x cxx -c -std strict_ansi -nopure_cname -noimplicit_include -timplicit_local -ptr "$(<[1]:D)/cxx_repository" $(OPTIONS) $(TEMPLATE_DEPTH) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)" -} - -# Always create archive from scratch. See the gcc toolet for rationale. -RM = [ common.rm-command ] ; -actions together piecemeal archive -{ - $(RM) "$(<)" - ar rc $(<) $(>) -} diff --git a/jam-files/boost-build/tools/hpfortran.jam b/jam-files/boost-build/tools/hpfortran.jam deleted file mode 100644 index 96e8d18b..00000000 --- a/jam-files/boost-build/tools/hpfortran.jam +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2004 Toon Knapen -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import toolset : flags ; -import feature ; -import fortran ; - -rule init ( version ? : command * : options * ) -{ -} - -# Declare flags and action for compilation -flags hpfortran OPTIONS off : -O0 ; -flags hpfortran OPTIONS speed : -O3 ; -flags hpfortran OPTIONS space : -O1 ; - -flags hpfortran OPTIONS on : -g ; -flags hpfortran OPTIONS on : -pg ; - -flags hpfortran DEFINES ; -flags hpfortran INCLUDES ; - -rule compile.fortran -{ -} - -actions compile.fortran -{ - f77 +DD64 $(OPTIONS) -D$(DEFINES) -I$(INCLUDES) -c -o "$(<)" "$(>)" -} - -generators.register-fortran-compiler hpfortran.compile.fortran : FORTRAN : OBJ ; diff --git a/jam-files/boost-build/tools/ifort.jam b/jam-files/boost-build/tools/ifort.jam deleted file mode 100644 index eb7c1988..00000000 --- a/jam-files/boost-build/tools/ifort.jam +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (C) 2004 Toon Knapen -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import toolset : flags ; -import feature ; -import fortran ; - -rule init ( version ? : command * : options * ) -{ -} - -# Declare flags and action for compilation -flags ifort OPTIONS ; - -flags ifort OPTIONS off : /Od ; -flags ifort OPTIONS speed : /O3 ; -flags ifort OPTIONS space : /O1 ; - -flags ifort OPTIONS on : /debug:full ; -flags ifort OPTIONS on : /Qprof_gen ; - -flags ifort.compile FFLAGS off/shared : /MD ; -flags ifort.compile FFLAGS on/shared : /MDd ; -flags ifort.compile FFLAGS off/static/single : /ML ; -flags ifort.compile FFLAGS on/static/single : /MLd ; -flags ifort.compile FFLAGS off/static/multi : /MT ; -flags ifort.compile FFLAGS on/static/multi : /MTd ; - -flags ifort DEFINES ; -flags ifort INCLUDES ; - -rule compile.fortran -{ -} - -actions compile.fortran -{ - ifort $(FFLAGS) $(OPTIONS) /names:lowercase /D$(DEFINES) /I"$(INCLUDES)" /c /object:"$(<)" "$(>)" -} - -generators.register-fortran-compiler ifort.compile.fortran : FORTRAN : OBJ ; diff --git a/jam-files/boost-build/tools/intel-darwin.jam b/jam-files/boost-build/tools/intel-darwin.jam deleted file mode 100644 index aa0fd8fb..00000000 --- a/jam-files/boost-build/tools/intel-darwin.jam +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright Vladimir Prus 2004. -# Copyright Noel Belcourt 2007. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt -# or copy at http://www.boost.org/LICENSE_1_0.txt) - -import intel ; -import feature : feature ; -import os ; -import toolset ; -import toolset : flags ; -import gcc ; -import common ; -import errors ; -import generators ; - -feature.extend-subfeature toolset intel : platform : darwin ; - -toolset.inherit-generators intel-darwin - intel darwin - : gcc - # Don't inherit PCH generators. They were not tested, and probably - # don't work for this compiler. - : gcc.mingw.link gcc.mingw.link.dll gcc.compile.c.pch gcc.compile.c++.pch - ; - -generators.override intel-darwin.prebuilt : builtin.lib-generator ; -generators.override intel-darwin.prebuilt : builtin.prebuilt ; -generators.override intel-darwin.searched-lib-generator : searched-lib-generator ; - -toolset.inherit-rules intel-darwin : gcc ; -toolset.inherit-flags intel-darwin : gcc - : off on full space - off all on - x86/32 - x86/64 - ; - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - -# vectorization diagnostics -feature vectorize : off on full ; - -# Initializes the intel-darwin toolset -# version in mandatory -# name (default icc) is used to invoke the specified intel complier -# compile and link options allow you to specify addition command line options for each version -rule init ( version ? : command * : options * ) -{ - local condition = [ common.check-init-parameters intel-darwin - : version $(version) ] ; - - command = [ common.get-invocation-command intel-darwin : icc - : $(command) : /opt/intel_cc_80/bin ] ; - - common.handle-options intel-darwin : $(condition) : $(command) : $(options) ; - - gcc.init-link-flags intel-darwin darwin $(condition) ; - - # handle - # local library-path = [ feature.get-values : $(options) ] ; - # flags intel-darwin.link USER_OPTIONS $(condition) : [ feature.get-values : $(options) ] ; - - local root = [ feature.get-values : $(options) ] ; - local bin ; - if $(command) || $(root) - { - bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ; - root ?= $(bin:D) ; - - if $(root) - { - # Libraries required to run the executable may be in either - # $(root)/lib (10.1 and earlier) - # or - # $(root)/lib/architecture-name (11.0 and later: - local lib_path = $(root)/lib $(root:P)/lib/$(bin:B) ; - if $(.debug-configuration) - { - ECHO notice: using intel libraries :: $(condition) :: $(lib_path) ; - } - flags intel-darwin.link RUN_PATH $(condition) : $(lib_path) ; - } - } - - local m = [ MATCH (..).* : $(version) ] ; - local n = [ MATCH (.)\\. : $(m) ] ; - if $(n) { - m = $(n) ; - } - - local major = $(m) ; - - if $(major) = "9" { - flags intel-darwin.compile OPTIONS $(condition)/off : -Ob0 ; - flags intel-darwin.compile OPTIONS $(condition)/on : -Ob1 ; - flags intel-darwin.compile OPTIONS $(condition)/full : -Ob2 ; - flags intel-darwin.compile OPTIONS $(condition)/off : -vec-report0 ; - flags intel-darwin.compile OPTIONS $(condition)/on : -vec-report1 ; - flags intel-darwin.compile OPTIONS $(condition)/full : -vec-report5 ; - flags intel-darwin.link OPTIONS $(condition)/static : -static -static-libcxa -lstdc++ -lpthread ; - flags intel-darwin.link OPTIONS $(condition)/shared : -shared-libcxa -lstdc++ -lpthread ; - } - else { - flags intel-darwin.compile OPTIONS $(condition)/off : -inline-level=0 ; - flags intel-darwin.compile OPTIONS $(condition)/on : -inline-level=1 ; - flags intel-darwin.compile OPTIONS $(condition)/full : -inline-level=2 ; - flags intel-darwin.compile OPTIONS $(condition)/off : -vec-report0 ; - flags intel-darwin.compile OPTIONS $(condition)/on : -vec-report1 ; - flags intel-darwin.compile OPTIONS $(condition)/full : -vec-report5 ; - flags intel-darwin.link OPTIONS $(condition)/static : -static -static-intel -lstdc++ -lpthread ; - flags intel-darwin.link OPTIONS $(condition)/shared : -shared-intel -lstdc++ -lpthread ; - } - - local minor = [ MATCH ".*\\.(.).*" : $(version) ] ; - - # wchar_t char_traits workaround for compilers older than 10.2 - if $(major) = "9" || ( $(major) = "10" && ( $(minor) = "0" || $(minor) = "1" ) ) { - flags intel-darwin.compile DEFINES $(condition) : __WINT_TYPE__=int : unchecked ; - } -} - -SPACE = " " ; - -flags intel-darwin.compile OPTIONS ; -flags intel-darwin.compile OPTIONS ; -# flags intel-darwin.compile INCLUDES ; - -flags intel-darwin.compile OPTIONS space : -O1 ; # no specific space optimization flag in icc - -# -cpu-type-em64t = prescott nocona ; -flags intel-darwin.compile OPTIONS $(cpu-type-em64t)/32 : -m32 ; # -mcmodel=small ; -flags intel-darwin.compile OPTIONS $(cpu-type-em64t)/64 : -m64 ; # -mcmodel=large ; - -flags intel-darwin.compile.c OPTIONS off : -w0 ; -flags intel-darwin.compile.c OPTIONS on : -w1 ; -flags intel-darwin.compile.c OPTIONS all : -w2 ; - -flags intel-darwin.compile.c++ OPTIONS off : -w0 ; -flags intel-darwin.compile.c++ OPTIONS on : -w1 ; -flags intel-darwin.compile.c++ OPTIONS all : -w2 ; - -actions compile.c -{ - "$(CONFIG_COMMAND)" -xc $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" -xc++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -flags intel-darwin ARFLAGS ; - -# Default value. Mostly for the sake of intel-linux -# that inherits from gcc, but does not has the same -# logic to set the .AR variable. We can put the same -# logic in intel-linux, but that's hardly worth the trouble -# as on Linux, 'ar' is always available. -.AR = ar ; - -rule archive ( targets * : sources * : properties * ) -{ - # Always remove archive and start again. Here's rationale from - # Andre Hentz: - # - # I had a file, say a1.c, that was included into liba.a. - # I moved a1.c to a2.c, updated my Jamfiles and rebuilt. - # My program was crashing with absurd errors. - # After some debugging I traced it back to the fact that a1.o was *still* - # in liba.a - # - # Rene Rivera: - # - # Originally removing the archive was done by splicing an RM - # onto the archive action. That makes archives fail to build on NT - # when they have many files because it will no longer execute the - # action directly and blow the line length limit. Instead we - # remove the file in a different action, just before the building - # of the archive. - # - local clean.a = $(targets[1])(clean) ; - TEMPORARY $(clean.a) ; - NOCARE $(clean.a) ; - LOCATE on $(clean.a) = [ on $(targets[1]) return $(LOCATE) ] ; - DEPENDS $(clean.a) : $(sources) ; - DEPENDS $(targets) : $(clean.a) ; - common.RmTemps $(clean.a) : $(targets) ; -} - -actions piecemeal archive -{ - "$(.AR)" $(AROPTIONS) rc "$(<)" "$(>)" - "ranlib" -cs "$(<)" -} - -flags intel-darwin.link USER_OPTIONS ; - -# Declare actions for linking -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; - # Serialize execution of the 'link' action, since - # running N links in parallel is just slower. - JAM_SEMAPHORE on $(targets) = intel-darwin-link-semaphore ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) -} - -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" -single_module -dynamiclib -install_name "$(<[1]:D=)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) -} diff --git a/jam-files/boost-build/tools/intel-linux.jam b/jam-files/boost-build/tools/intel-linux.jam deleted file mode 100644 index d9164add..00000000 --- a/jam-files/boost-build/tools/intel-linux.jam +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright (c) 2003 Michael Stevens -# Copyright (c) 2011 Bryce Lelbach -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import toolset ; -import feature ; -import toolset : flags ; - -import intel ; -import gcc ; -import common ; -import errors ; -import generators ; -import type ; -import numbers ; - -feature.extend-subfeature toolset intel : platform : linux ; - -toolset.inherit-generators intel-linux - intel linux : gcc : gcc.mingw.link gcc.mingw.link.dll ; -generators.override intel-linux.prebuilt : builtin.lib-generator ; -generators.override intel-linux.prebuilt : builtin.prebuilt ; -generators.override intel-linux.searched-lib-generator : searched-lib-generator ; - -# Override default do-nothing generators. -generators.override intel-linux.compile.c.pch : pch.default-c-pch-generator ; -generators.override intel-linux.compile.c++.pch : pch.default-cpp-pch-generator ; - -type.set-generated-target-suffix PCH : intel linux : pchi ; - -toolset.inherit-rules intel-linux : gcc ; -toolset.inherit-flags intel-linux : gcc - : off on full - space speed - off all on - ; - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - -# Initializes the intel-linux toolset -# version in mandatory -# name (default icpc) is used to invoke the specified intel-linux complier -# compile and link options allow you to specify addition command line options for each version -rule init ( version ? : command * : options * ) -{ - local condition = [ common.check-init-parameters intel-linux - : version $(version) ] ; - - if $(.debug-configuration) - { - ECHO "notice: intel-linux version is" $(version) ; - } - - local default_path ; - - # Intel C++ Composer XE 2011 for Linux, aka Intel C++ Compiler XE 12.0, - # aka intel-linux-12.0. In this version, Intel thankfully decides to install - # to a sane 'intel' folder in /opt. - if [ MATCH "(12[.]0|12)" : $(version) ] - { default_path = /opt/intel/bin ; } - # Intel C++ Compiler 11.1. - else if [ MATCH "(11[.]1)" : $(version) ] - { default_path = /opt/intel_cce_11.1.064.x86_64/bin ; } - # Intel C++ Compiler 11.0. - else if [ MATCH "(11[.]0|11)" : $(version) ] - { default_path = /opt/intel_cce_11.0.074.x86_64/bin ; } - # Intel C++ Compiler 10.1. - else if [ MATCH "(10[.]1)" : $(version) ] - { default_path = /opt/intel_cce_10.1.013_x64/bin ; } - # Intel C++ Compiler 9.1. - else if [ MATCH "(9[.]1)" : $(version) ] - { default_path = /opt/intel_cc_91/bin ; } - # Intel C++ Compiler 9.0. - else if [ MATCH "(9[.]0|9)" : $(version) ] - { default_path = /opt/intel_cc_90/bin ; } - # Intel C++ Compiler 8.1. - else if [ MATCH "(8[.]1)" : $(version) ] - { default_path = /opt/intel_cc_81/bin ; } - # Intel C++ Compiler 8.0 - this used to be the default, so now it's the - # fallback. - else - { default_path = /opt/intel_cc_80/bin ; } - - if $(.debug-configuration) - { - ECHO "notice: default search path for intel-linux is" $(default_path) ; - } - - command = [ common.get-invocation-command intel-linux : icpc - : $(command) : $(default_path) ] ; - - common.handle-options intel-linux : $(condition) : $(command) : $(options) ; - - gcc.init-link-flags intel-linux gnu $(condition) ; - - local root = [ feature.get-values : $(options) ] ; - local bin ; - if $(command) || $(root) - { - bin ?= [ common.get-absolute-tool-path $(command[-1]) ] ; - root ?= $(bin:D) ; - - local command-string = $(command:J=" ") ; - local version-output = [ SHELL "$(command-string) --version" ] ; - local real-version = [ MATCH "([0-9.]+)" : $(version-output) ] ; - local major = [ MATCH "([0-9]+).*" : $(real-version) ] ; - - # If we failed to determine major version, use the behaviour for - # the current compiler. - if $(major) && [ numbers.less $(major) 10 ] - { - flags intel-linux.compile OPTIONS $(condition)/off : "-Ob0" ; - flags intel-linux.compile OPTIONS $(condition)/on : "-Ob1" ; - flags intel-linux.compile OPTIONS $(condition)/full : "-Ob2" ; - flags intel-linux.compile OPTIONS $(condition)/space : "-O1" ; - flags intel-linux.compile OPTIONS $(condition)/speed : "-O3 -ip" ; - } - else if $(major) && [ numbers.less $(major) 11 ] - { - flags intel-linux.compile OPTIONS $(condition)/off : "-inline-level=0" ; - flags intel-linux.compile OPTIONS $(condition)/on : "-inline-level=1" ; - flags intel-linux.compile OPTIONS $(condition)/full : "-inline-level=2" ; - flags intel-linux.compile OPTIONS $(condition)/space : "-O1" ; - flags intel-linux.compile OPTIONS $(condition)/speed : "-O3 -ip" ; - } - else # newer version of intel do have -Os (at least 11+, don't know about 10) - { - flags intel-linux.compile OPTIONS $(condition)/off : "-inline-level=0" ; - flags intel-linux.compile OPTIONS $(condition)/on : "-inline-level=1" ; - flags intel-linux.compile OPTIONS $(condition)/full : "-inline-level=2" ; - flags intel-linux.compile OPTIONS $(condition)/space : "-Os" ; - flags intel-linux.compile OPTIONS $(condition)/speed : "-O3 -ip" ; - } - - if $(root) - { - # Libraries required to run the executable may be in either - # $(root)/lib (10.1 and earlier) - # or - # $(root)/lib/architecture-name (11.0 and later: - local lib_path = $(root)/lib $(root:P)/lib/$(bin:B) ; - if $(.debug-configuration) - { - ECHO notice: using intel libraries :: $(condition) :: $(lib_path) ; - } - flags intel-linux.link RUN_PATH $(condition) : $(lib_path) ; - } - } -} - -SPACE = " " ; - -flags intel-linux.compile OPTIONS off : -w0 ; -flags intel-linux.compile OPTIONS on : -w1 ; -flags intel-linux.compile OPTIONS all : -w2 ; - -rule compile.c++ ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-fpic $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; -} - -actions compile.c++ bind PCH_FILE -{ - "$(CONFIG_COMMAND)" -c -xc++ $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -use-pch"$(PCH_FILE)" -c -o "$(<)" "$(>)" -} - -rule compile.c ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-fpic $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; - DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ; -} - -actions compile.c bind PCH_FILE -{ - "$(CONFIG_COMMAND)" -c -xc $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -use-pch"$(PCH_FILE)" -c -o "$(<)" "$(>)" -} - -rule compile.c++.pch ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-fpic $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; -} -# -# Compiling a pch first deletes any existing *.pchi file, as Intel's compiler -# won't over-write an existing pch: instead it creates filename$1.pchi, filename$2.pchi -# etc - which appear not to do anything except take up disk space :-( -# -actions compile.c++.pch -{ - rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c++-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -pch-create "$(<)" "$(>)" -} - -actions compile.fortran -{ - "ifort" -c $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -rule compile.c.pch ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-fpic $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; -} - -actions compile.c.pch -{ - rm -f "$(<)" && "$(CONFIG_COMMAND)" -x c-header $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -pch-create "$(<)" "$(>)" -} - -rule link ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; - SPACE on $(targets) = " " ; - JAM_SEMAPHORE on $(targets) = intel-linux-link-semaphore ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS) -} - -rule link.dll ( targets * : sources * : properties * ) -{ - gcc.setup-threading $(targets) : $(sources) : $(properties) ; - gcc.setup-address-model $(targets) : $(sources) : $(properties) ; - SPACE on $(targets) = " " ; - JAM_SEMAPHORE on $(targets) = intel-linux-link-semaphore ; -} - -# Differ from 'link' above only by -shared. -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -o "$(<)" -Wl,-soname$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) $(OPTIONS) $(USER_OPTIONS) -} - - - diff --git a/jam-files/boost-build/tools/intel-win.jam b/jam-files/boost-build/tools/intel-win.jam deleted file mode 100644 index 691b5dce..00000000 --- a/jam-files/boost-build/tools/intel-win.jam +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright Vladimir Prus 2004. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt -# or copy at http://www.boost.org/LICENSE_1_0.txt) - -# Importing common is needed because the rules we inherit here depend on it. -# That is nasty. -import common ; -import errors ; -import feature ; -import intel ; -import msvc ; -import os ; -import toolset ; -import generators ; -import type ; - -feature.extend-subfeature toolset intel : platform : win ; - -toolset.inherit-generators intel-win intel win : msvc ; -toolset.inherit-flags intel-win : msvc : : YLOPTION ; -toolset.inherit-rules intel-win : msvc ; - -# Override default do-nothing generators. -generators.override intel-win.compile.c.pch : pch.default-c-pch-generator ; -generators.override intel-win.compile.c++.pch : pch.default-cpp-pch-generator ; -generators.override intel-win.compile.rc : rc.compile.resource ; -generators.override intel-win.compile.mc : mc.compile ; - -toolset.flags intel-win.compile PCH_SOURCE on : ; - -toolset.add-requirements intel-win,shared:multi ; - -# Initializes the intel toolset for windows -rule init ( version ? : # the compiler version - command * : # the command to invoke the compiler itself - options * # Additional option: - # either 'vc6', 'vc7', 'vc7.1' - # or 'native'(default). - ) -{ - local compatibility = - [ feature.get-values : $(options) ] ; - local condition = [ common.check-init-parameters intel-win - : version $(version) : compatibility $(compatibility) ] ; - - command = [ common.get-invocation-command intel-win : icl.exe : - $(command) ] ; - - common.handle-options intel-win : $(condition) : $(command) : $(options) ; - - local root ; - if $(command) - { - root = [ common.get-absolute-tool-path $(command[-1]) ] ; - root = $(root)/ ; - } - - local setup ; - setup = [ GLOB $(root) : iclvars_*.bat ] ; - if ! $(setup) - { - setup = $(root)/iclvars.bat ; - } - setup = "call \""$(setup)"\" > nul " ; - - if [ os.name ] = NT - { - setup = $(setup)" -" ; - } - else - { - setup = "cmd /S /C "$(setup)" \"&&\" " ; - } - - toolset.flags intel-win.compile .CC $(condition) : $(setup)icl ; - toolset.flags intel-win.link .LD $(condition) : $(setup)xilink ; - toolset.flags intel-win.archive .LD $(condition) : $(setup)xilink /lib ; - toolset.flags intel-win.link .MT $(condition) : $(setup)mt -nologo ; - toolset.flags intel-win.compile .MC $(condition) : $(setup)mc ; - toolset.flags intel-win.compile .RC $(condition) : $(setup)rc ; - - local m = [ MATCH (.).* : $(version) ] ; - local major = $(m[1]) ; - - local C++FLAGS ; - - C++FLAGS += /nologo ; - - # Reduce the number of spurious error messages - C++FLAGS += /Qwn5 /Qwd985 ; - - # Enable ADL - C++FLAGS += -Qoption,c,--arg_dep_lookup ; #"c" works for C++, too - - # Disable Microsoft "secure" overloads in Dinkumware libraries since they - # cause compile errors with Intel versions 9 and 10. - C++FLAGS += -D_SECURE_SCL=0 ; - - if $(major) > 5 - { - C++FLAGS += /Zc:forScope ; # Add support for correct for loop scoping. - } - - # Add options recognized only by intel7 and above. - if $(major) >= 7 - { - C++FLAGS += /Qansi_alias ; - } - - if $(compatibility) = vc6 - { - C++FLAGS += - # Emulate VC6 - /Qvc6 - - # No wchar_t support in vc6 dinkum library. Furthermore, in vc6 - # compatibility-mode, wchar_t is not a distinct type from unsigned - # short. - -DBOOST_NO_INTRINSIC_WCHAR_T - ; - } - else - { - if $(major) > 5 - { - # Add support for wchar_t - C++FLAGS += /Zc:wchar_t - # Tell the dinkumware library about it. - -D_NATIVE_WCHAR_T_DEFINED - ; - } - } - - if $(compatibility) && $(compatibility) != native - { - C++FLAGS += /Q$(base-vc) ; - } - else - { - C++FLAGS += - -Qoption,cpp,--arg_dep_lookup - # The following options were intended to disable the Intel compiler's - # 'bug-emulation' mode, but were later reported to be causing ICE with - # Intel-Win 9.0. It is not yet clear which options can be safely used. - # -Qoption,cpp,--const_string_literals - # -Qoption,cpp,--new_for_init - # -Qoption,cpp,--no_implicit_typename - # -Qoption,cpp,--no_friend_injection - # -Qoption,cpp,--no_microsoft_bugs - ; - } - - toolset.flags intel-win CFLAGS $(condition) : $(C++FLAGS) ; - # By default, when creating PCH, intel adds 'i' to the explicitly - # specified name of the PCH file. Of course, Boost.Build is not - # happy when compiler produces not the file it was asked for. - # The option below stops this behaviour. - toolset.flags intel-win CFLAGS : -Qpchi- ; - - if ! $(compatibility) - { - # If there's no backend version, assume 7.1. - compatibility = vc7.1 ; - } - - local extract-version = [ MATCH ^vc(.*) : $(compatibility) ] ; - if ! $(extract-version) - { - errors.user-error "Invalid value for compatibility option:" - $(compatibility) ; - } - - # Depending on the settings, running of tests require some runtime DLLs. - toolset.flags intel-win RUN_PATH $(condition) : $(root) ; - - msvc.configure-version-specific intel-win : $(extract-version[1]) : $(condition) ; -} - -toolset.flags intel-win.link LIBRARY_OPTION intel : "" ; - -toolset.flags intel-win YLOPTION ; - diff --git a/jam-files/boost-build/tools/intel.jam b/jam-files/boost-build/tools/intel.jam deleted file mode 100644 index 67038aa2..00000000 --- a/jam-files/boost-build/tools/intel.jam +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright Vladimir Prus 2004. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt -# or copy at http://www.boost.org/LICENSE_1_0.txt) - -# This is a generic 'intel' toolset. Depending on the current -# system, it forwards either to 'intel-linux' or 'intel-win' -# modules. - -import feature ; -import os ; -import toolset ; - -feature.extend toolset : intel ; -feature.subfeature toolset intel : platform : : propagated link-incompatible ; - -rule init ( * : * ) -{ - if [ os.name ] = LINUX - { - toolset.using intel-linux : - $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - else if [ os.name ] = MACOSX - { - toolset.using intel-darwin : - $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - else - { - toolset.using intel-win : - $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } -} diff --git a/jam-files/boost-build/tools/lex.jam b/jam-files/boost-build/tools/lex.jam deleted file mode 100644 index 75d64131..00000000 --- a/jam-files/boost-build/tools/lex.jam +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2003 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -import type ; -import generators ; -import feature ; -import property ; - - -feature.feature flex.prefix : : free ; -type.register LEX : l ; -type.register LEX++ : ll ; -generators.register-standard lex.lex : LEX : C ; -generators.register-standard lex.lex : LEX++ : CPP ; - -rule init ( ) -{ -} - -rule lex ( target : source : properties * ) -{ - local r = [ property.select flex.prefix : $(properties) ] ; - if $(r) - { - PREFIX on $(<) = $(r:G=) ; - } -} - -actions lex -{ - flex -P$(PREFIX) -o$(<) $(>) -} diff --git a/jam-files/boost-build/tools/make.jam b/jam-files/boost-build/tools/make.jam deleted file mode 100644 index 08567285..00000000 --- a/jam-files/boost-build/tools/make.jam +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2003 Dave Abrahams -# Copyright 2003 Douglas Gregor -# Copyright 2006 Rene Rivera -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module defines the 'make' main target rule. - -import "class" : new ; -import errors : error ; -import project ; -import property ; -import property-set ; -import regex ; -import targets ; - - -class make-target-class : basic-target -{ - import type regex virtual-target ; - import "class" : new ; - - rule __init__ ( name : project : sources * : requirements * - : default-build * : usage-requirements * ) - { - basic-target.__init__ $(name) : $(project) : $(sources) : - $(requirements) : $(default-build) : $(usage-requirements) ; - } - - rule construct ( name : source-targets * : property-set ) - { - local action-name = [ $(property-set).get ] ; - # 'm' will always be set -- we add '@' ourselves in the 'make' rule - # below. - local m = [ MATCH ^@(.*) : $(action-name) ] ; - - local a = [ new action $(source-targets) : $(m[1]) : $(property-set) ] ; - local t = [ new file-target $(self.name) exact : [ type.type - $(self.name) ] : $(self.project) : $(a) ] ; - return [ property-set.empty ] [ virtual-target.register $(t) ] ; - } -} - - -# Declares the 'make' main target. -# -rule make ( target-name : sources * : generating-rule + : requirements * : - usage-requirements * ) -{ - local project = [ project.current ] ; - - # The '@' sign causes the feature.jam module to qualify rule name with the - # module name of current project, if needed. - local m = [ MATCH ^(@).* : $(generating-rule) ] ; - if ! $(m) - { - generating-rule = @$(generating-rule) ; - } - requirements += $(generating-rule) ; - - targets.main-target-alternative - [ new make-target-class $(target-name) : $(project) - : [ targets.main-target-sources $(sources) : $(target-name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build : $(project) ] - : [ targets.main-target-usage-requirements $(usage-requirements) : - $(project) ] ] ; -} - - -IMPORT $(__name__) : make : : make ; diff --git a/jam-files/boost-build/tools/make.py b/jam-files/boost-build/tools/make.py deleted file mode 100644 index 10baa1cb..00000000 --- a/jam-files/boost-build/tools/make.py +++ /dev/null @@ -1,59 +0,0 @@ -# Status: ported. -# Base revision: 64068 - -# Copyright 2003 Dave Abrahams -# Copyright 2003 Douglas Gregor -# Copyright 2006 Rene Rivera -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module defines the 'make' main target rule. - -from b2.build.targets import BasicTarget -from b2.build.virtual_target import Action, FileTarget -from b2.build import type -from b2.manager import get_manager -import b2.build.property_set - - -class MakeTarget(BasicTarget): - - def construct(self, name, source_targets, property_set): - - action_name = property_set.get("")[0] - action = Action(get_manager(), source_targets, action_name[1:], property_set) - target = FileTarget(self.name(), type.type(self.name()), - self.project(), action, exact=True) - return [ b2.build.property_set.empty(), - [self.project().manager().virtual_targets().register(target)]] - -def make (target_name, sources, generating_rule, - requirements=None, usage_requirements=None): - - target_name = target_name[0] - generating_rule = generating_rule[0] - if generating_rule[0] != '@': - generating_rule = '@' + generating_rule - - if not requirements: - requirements = [] - - - requirements.append("%s" % generating_rule) - - m = get_manager() - targets = m.targets() - project = m.projects().current() - engine = m.engine() - engine.register_bjam_action(generating_rule) - - targets.main_target_alternative(MakeTarget( - target_name, project, - targets.main_target_sources(sources, target_name), - targets.main_target_requirements(requirements, project), - targets.main_target_default_build([], project), - targets.main_target_usage_requirements(usage_requirements or [], project))) - -get_manager().projects().add_rule("make", make) - diff --git a/jam-files/boost-build/tools/mc.jam b/jam-files/boost-build/tools/mc.jam deleted file mode 100644 index 57837773..00000000 --- a/jam-files/boost-build/tools/mc.jam +++ /dev/null @@ -1,44 +0,0 @@ -#~ Copyright 2005 Alexey Pakhunov. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Support for Microsoft message compiler tool. -# Notes: -# - there's just message compiler tool, there's no tool for -# extracting message strings from sources -# - This file allows to use Microsoft message compiler -# with any toolset. In msvc.jam, there's more specific -# message compiling action. - -import common ; -import generators ; -import feature : feature get-values ; -import toolset : flags ; -import type ; -import rc ; - -rule init ( ) -{ -} - -type.register MC : mc ; - - -# Command line options -feature mc-input-encoding : ansi unicode : free ; -feature mc-output-encoding : unicode ansi : free ; -feature mc-set-customer-bit : no yes : free ; - -flags mc.compile MCFLAGS ansi : -a ; -flags mc.compile MCFLAGS unicode : -u ; -flags mc.compile MCFLAGS ansi : -A ; -flags mc.compile MCFLAGS unicode : -U ; -flags mc.compile MCFLAGS no : ; -flags mc.compile MCFLAGS yes : -c ; - -generators.register-standard mc.compile : MC : H RC ; - -actions compile -{ - mc $(MCFLAGS) -h "$(<[1]:DW)" -r "$(<[2]:DW)" "$(>:W)" -} diff --git a/jam-files/boost-build/tools/message.jam b/jam-files/boost-build/tools/message.jam deleted file mode 100644 index 212d8542..00000000 --- a/jam-files/boost-build/tools/message.jam +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2008 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Defines main target type 'message', that prints a message when built for the -# first time. - -import project ; -import "class" : new ; -import targets ; -import property-set ; - -class message-target-class : basic-target -{ - rule __init__ ( name-and-dir : project : * ) - { - basic-target.__init__ $(name-and-dir) : $(project) ; - self.3 = $(3) ; - self.4 = $(4) ; - self.5 = $(5) ; - self.6 = $(6) ; - self.7 = $(7) ; - self.8 = $(8) ; - self.9 = $(9) ; - self.built = ; - } - - rule construct ( name : source-targets * : property-set ) - { - if ! $(self.built) - { - for i in 3 4 5 6 7 8 9 - { - if $(self.$(i)) - { - ECHO $(self.$(i)) ; - } - } - self.built = 1 ; - } - - return [ property-set.empty ] ; - } -} - - -rule message ( name : * ) -{ - local project = [ project.current ] ; - - targets.main-target-alternative - [ new message-target-class $(name) : $(project) - : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) ] ; -} -IMPORT $(__name__) : message : : message ; \ No newline at end of file diff --git a/jam-files/boost-build/tools/message.py b/jam-files/boost-build/tools/message.py deleted file mode 100644 index cc0b946f..00000000 --- a/jam-files/boost-build/tools/message.py +++ /dev/null @@ -1,46 +0,0 @@ -# Status: ported. -# Base revision: 64488. -# -# Copyright 2008, 2010 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Defines main target type 'message', that prints a message when built for the -# first time. - -import b2.build.targets as targets -import b2.build.property_set as property_set - -from b2.manager import get_manager - -class MessageTargetClass(targets.BasicTarget): - - def __init__(self, name, project, *args): - - targets.BasicTarget.__init__(self, name, project, []) - self.args = args - self.built = False - - def construct(self, name, sources, ps): - - if not self.built: - for arg in self.args: - if type(arg) == type([]): - arg = " ".join(arg) - print arg - self.built = True - - return (property_set.empty(), []) - -def message(name, *args): - - if type(name) == type([]): - name = name[0] - - t = get_manager().targets() - - project = get_manager().projects().current() - - return t.main_target_alternative(MessageTargetClass(*((name, project) + args))) - -get_manager().projects().add_rule("message", message) diff --git a/jam-files/boost-build/tools/midl.jam b/jam-files/boost-build/tools/midl.jam deleted file mode 100644 index 0aa5dda3..00000000 --- a/jam-files/boost-build/tools/midl.jam +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2005 Alexey Pakhunov. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# Microsoft Interface Definition Language (MIDL) related routines - -import common ; -import generators ; -import feature : feature get-values ; -import os ; -import scanner ; -import toolset : flags ; -import type ; - -rule init ( ) -{ -} - -type.register IDL : idl ; - -# A type library (.tlb) is generated by MIDL compiler and can be included -# to resources of an application (.rc). In order to be found by a resource -# compiler its target type should be derived from 'H' - otherwise -# the property '' will be ignored. -type.register MSTYPELIB : tlb : H ; - - -# Register scanner for MIDL files -class midl-scanner : scanner -{ - import path property-set regex scanner type virtual-target ; - - rule __init__ ( includes * ) - { - scanner.__init__ ; - - self.includes = $(includes) ; - - # List of quoted strings - self.re-strings = "[ \t]*\"([^\"]*)\"([ \t]*,[ \t]*\"([^\"]*)\")*[ \t]*" ; - - # 'import' and 'importlib' directives - self.re-import = "import"$(self.re-strings)"[ \t]*;" ; - self.re-importlib = "importlib[ \t]*[(]"$(self.re-strings)"[)][ \t]*;" ; - - # C preprocessor 'include' directive - self.re-include-angle = "#[ \t]*include[ \t]*<(.*)>" ; - self.re-include-quoted = "#[ \t]*include[ \t]*\"(.*)\"" ; - } - - rule pattern ( ) - { - # Match '#include', 'import' and 'importlib' directives - return "((#[ \t]*include|import(lib)?).+(<(.*)>|\"(.*)\").+)" ; - } - - rule process ( target : matches * : binding ) - { - local included-angle = [ regex.transform $(matches) : $(self.re-include-angle) : 1 ] ; - local included-quoted = [ regex.transform $(matches) : $(self.re-include-quoted) : 1 ] ; - local imported = [ regex.transform $(matches) : $(self.re-import) : 1 3 ] ; - local imported_tlbs = [ regex.transform $(matches) : $(self.re-importlib) : 1 3 ] ; - - # CONSIDER: the new scoping rule seem to defeat "on target" variables. - local g = [ on $(target) return $(HDRGRIST) ] ; - local b = [ NORMALIZE_PATH $(binding:D) ] ; - - # Attach binding of including file to included targets. - # When target is directly created from virtual target - # this extra information is unnecessary. But in other - # cases, it allows to distinguish between two headers of the - # same name included from different places. - local g2 = $(g)"#"$(b) ; - - included-angle = $(included-angle:G=$(g)) ; - included-quoted = $(included-quoted:G=$(g2)) ; - imported = $(imported:G=$(g2)) ; - imported_tlbs = $(imported_tlbs:G=$(g2)) ; - - local all = $(included-angle) $(included-quoted) $(imported) ; - - INCLUDES $(target) : $(all) ; - DEPENDS $(target) : $(imported_tlbs) ; - NOCARE $(all) $(imported_tlbs) ; - SEARCH on $(included-angle) = $(self.includes:G=) ; - SEARCH on $(included-quoted) = $(b) $(self.includes:G=) ; - SEARCH on $(imported) = $(b) $(self.includes:G=) ; - SEARCH on $(imported_tlbs) = $(b) $(self.includes:G=) ; - - scanner.propagate - [ type.get-scanner CPP : [ property-set.create $(self.includes) ] ] : - $(included-angle) $(included-quoted) : $(target) ; - - scanner.propagate $(__name__) : $(imported) : $(target) ; - } -} - -scanner.register midl-scanner : include ; -type.set-scanner IDL : midl-scanner ; - - -# Command line options -feature midl-stubless-proxy : yes no : propagated ; -feature midl-robust : yes no : propagated ; - -flags midl.compile.idl MIDLFLAGS yes : /Oicf ; -flags midl.compile.idl MIDLFLAGS no : /Oic ; -flags midl.compile.idl MIDLFLAGS yes : /robust ; -flags midl.compile.idl MIDLFLAGS no : /no_robust ; - -# Architecture-specific options -architecture-x86 = x86 ; -address-model-32 = 32 ; -address-model-64 = 64 ; - -flags midl.compile.idl MIDLFLAGS $(architecture-x86)/$(address-model-32) : /win32 ; -flags midl.compile.idl MIDLFLAGS $(architecture-x86)/64 : /x64 ; -flags midl.compile.idl MIDLFLAGS ia64/$(address-model-64) : /ia64 ; - - -flags midl.compile.idl DEFINES ; -flags midl.compile.idl UNDEFS ; -flags midl.compile.idl INCLUDES ; - - -generators.register-c-compiler midl.compile.idl : IDL : MSTYPELIB H C(%_i) C(%_proxy) C(%_dlldata) ; - - -# MIDL does not always generate '%_proxy.c' and '%_dlldata.c'. This behavior -# depends on contents of the source IDL file. Calling TOUCH_FILE below ensures -# that both files will be created so bjam will not try to recreate them -# constantly. -TOUCH_FILE = [ common.file-touch-command ] ; - -actions compile.idl -{ - midl /nologo @"@($(<[1]:W).rsp:E=$(nl)"$(>:W)" $(nl)-D$(DEFINES) $(nl)"-I$(INCLUDES)" $(nl)-U$(UNDEFS) $(nl)$(MIDLFLAGS) $(nl)/tlb "$(<[1]:W)" $(nl)/h "$(<[2]:W)" $(nl)/iid "$(<[3]:W)" $(nl)/proxy "$(<[4]:W)" $(nl)/dlldata "$(<[5]:W)")" - $(TOUCH_FILE) "$(<[4]:W)" - $(TOUCH_FILE) "$(<[5]:W)" -} diff --git a/jam-files/boost-build/tools/mipspro.jam b/jam-files/boost-build/tools/mipspro.jam deleted file mode 100644 index 417eaefc..00000000 --- a/jam-files/boost-build/tools/mipspro.jam +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright Noel Belcourt 2007. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import property ; -import generators ; -import os ; -import toolset : flags ; -import feature ; -import fortran ; -import type ; -import common ; - -feature.extend toolset : mipspro ; -toolset.inherit mipspro : unix ; -generators.override mipspro.prebuilt : builtin.lib-generator ; -generators.override mipspro.searched-lib-generator : searched-lib-generator ; - -# Documentation and toolchain description located -# http://www.sgi.com/products/software/irix/tools/ - -rule init ( version ? : command * : options * ) -{ - local condition = [ - common.check-init-parameters mipspro : version $(version) ] ; - - command = [ common.get-invocation-command mipspro : CC : $(command) ] ; - - common.handle-options mipspro : $(condition) : $(command) : $(options) ; - - command_c = $(command_c[1--2]) $(command[-1]:B=cc) ; - - toolset.flags mipspro CONFIG_C_COMMAND $(condition) : $(command_c) ; - - # fortran support - local command = [ - common.get-invocation-command mipspro : f77 : $(command) : $(install_dir) ] ; - - command_f = $(command_f[1--2]) $(command[-1]:B=f77) ; - toolset.flags mipspro CONFIG_F_COMMAND $(condition) : $(command_f) ; - - # set link flags - flags mipspro.link FINDLIBS-ST : [ - feature.get-values : $(options) ] : unchecked ; - - flags mipspro.link FINDLIBS-SA : [ - feature.get-values : $(options) ] : unchecked ; -} - -# Declare generators -generators.register-c-compiler mipspro.compile.c : C : OBJ : mipspro ; -generators.register-c-compiler mipspro.compile.c++ : CPP : OBJ : mipspro ; -generators.register-fortran-compiler mipspro.compile.fortran : FORTRAN : OBJ : mipspro ; - -cpu-arch-32 = - / - /32 ; - -cpu-arch-64 = - /64 ; - -flags mipspro.compile OPTIONS $(cpu-arch-32) : -n32 ; -flags mipspro.compile OPTIONS $(cpu-arch-64) : -64 ; - -# Declare flags and actions for compilation -flags mipspro.compile OPTIONS on : -g ; -# flags mipspro.compile OPTIONS on : -xprofile=tcov ; -flags mipspro.compile OPTIONS off : -w ; -flags mipspro.compile OPTIONS on : -ansiW -diag_suppress 1429 ; # suppress long long is nonstandard warning -flags mipspro.compile OPTIONS all : -fullwarn ; -flags mipspro.compile OPTIONS speed : -Ofast ; -flags mipspro.compile OPTIONS space : -O2 ; -flags mipspro.compile OPTIONS : -LANG:std ; -flags mipspro.compile.c++ OPTIONS off : -INLINE:none ; -flags mipspro.compile.c++ OPTIONS ; -flags mipspro.compile DEFINES ; -flags mipspro.compile INCLUDES ; - - -flags mipspro.compile.fortran OPTIONS ; - -actions compile.c -{ - "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" -FE:template_in_elf_section -ptused $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.fortran -{ - "$(CONFIG_F_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -# Declare flags and actions for linking -flags mipspro.link OPTIONS on : -g ; -# Strip the binary when no debugging is needed -# flags mipspro.link OPTIONS off : -s ; -# flags mipspro.link OPTIONS on : -xprofile=tcov ; -# flags mipspro.link OPTIONS multi : -mt ; - -flags mipspro.link OPTIONS $(cpu-arch-32) : -n32 ; -flags mipspro.link OPTIONS $(cpu-arch-64) : -64 ; - -flags mipspro.link OPTIONS speed : -Ofast ; -flags mipspro.link OPTIONS space : -O2 ; -flags mipspro.link OPTIONS ; -flags mipspro.link LINKPATH ; -flags mipspro.link FINDLIBS-ST ; -flags mipspro.link FINDLIBS-SA ; -flags mipspro.link FINDLIBS-SA multi : pthread ; -flags mipspro.link LIBRARIES ; -flags mipspro.link LINK-RUNTIME static : static ; -flags mipspro.link LINK-RUNTIME shared : dynamic ; -flags mipspro.link RPATH ; - -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -FE:template_in_elf_section -ptused $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -lm -} - -# Slight mods for dlls -rule link.dll ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -} - -# Declare action for creating static libraries -actions piecemeal archive -{ - ar -cr "$(<)" "$(>)" -} diff --git a/jam-files/boost-build/tools/mpi.jam b/jam-files/boost-build/tools/mpi.jam deleted file mode 100644 index 0fe490be..00000000 --- a/jam-files/boost-build/tools/mpi.jam +++ /dev/null @@ -1,583 +0,0 @@ -# Support for the Message Passing Interface (MPI) -# -# (C) Copyright 2005, 2006 Trustees of Indiana University -# (C) Copyright 2005 Douglas Gregor -# -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt.) -# -# Authors: Douglas Gregor -# Andrew Lumsdaine -# -# ==== MPI Configuration ==== -# -# For many users, MPI support can be enabled simply by adding the following -# line to your user-config.jam file: -# -# using mpi ; -# -# This should auto-detect MPI settings based on the MPI wrapper compiler in -# your path, e.g., "mpic++". If the wrapper compiler is not in your path, or -# has a different name, you can pass the name of the wrapper compiler as the -# first argument to the mpi module: -# -# using mpi : /opt/mpich2-1.0.4/bin/mpiCC ; -# -# If your MPI implementation does not have a wrapper compiler, or the MPI -# auto-detection code does not work with your MPI's wrapper compiler, -# you can pass MPI-related options explicitly via the second parameter to the -# mpi module: -# -# using mpi : : lammpio lammpi++ -# mpi lam -# dl ; -# -# To see the results of MPI auto-detection, pass "--debug-configuration" on -# the bjam command line. -# -# The (optional) fourth argument configures Boost.MPI for running -# regression tests. These parameters specify the executable used to -# launch jobs (default: "mpirun") followed by any necessary arguments -# to this to run tests and tell the program to expect the number of -# processors to follow (default: "-np"). With the default parameters, -# for instance, the test harness will execute, e.g., -# -# mpirun -np 4 all_gather_test -# -# ==== Linking Against the MPI Libraries === -# -# To link against the MPI libraries, import the "mpi" module and add the -# following requirement to your target: -# -# /mpi//mpi -# -# Since MPI support is not always available, you should check -# "mpi.configured" before trying to link against the MPI libraries. - -import "class" : new ; -import common ; -import feature : feature ; -import generators ; -import os ; -import project ; -import property ; -import testing ; -import toolset ; -import type ; -import path ; - -# Make this module a project -project.initialize $(__name__) ; -project mpi ; - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - -# Assuming the first part of the command line is the given prefix -# followed by some non-empty value, remove the first argument. Returns -# either nothing (if there was no prefix or no value) or a pair -# -# value rest-of-cmdline -# -# This is a subroutine of cmdline_to_features -rule add_feature ( prefix name cmdline ) -{ - local match = [ MATCH "^$(prefix)([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ; - - # If there was no value associated with the prefix, abort - if ! $(match) { - return ; - } - - local value = $(match[1]) ; - - if [ MATCH " +" : $(value) ] { - value = "\"$(value)\"" ; - } - - return "<$(name)>$(value)" $(match[2]) ; -} - -# Strip any end-of-line characters off the given string and return the -# result. -rule strip-eol ( string ) -{ - local match = [ MATCH "^(([A-Za-z0-9~`\.!@#$%^&*()_+={};:'\",.<>/?\\| -]|[|])*).*$" : $(string) ] ; - - if $(match) - { - return $(match[1]) ; - } - else - { - return $(string) ; - } -} - -# Split a command-line into a set of features. Certain kinds of -# compiler flags are recognized (e.g., -I, -D, -L, -l) and replaced -# with their Boost.Build equivalents (e.g., , , -# , ). All other arguments are introduced -# using the features in the unknown-features parameter, because we -# don't know how to deal with them. For instance, if your compile and -# correct. The incoming command line should be a string starting with -# an executable (e.g., g++ -I/include/path") and may contain any -# number of command-line arguments thereafter. The result is a list of -# features corresponding to the given command line, ignoring the -# executable. -rule cmdline_to_features ( cmdline : unknown-features ? ) -{ - local executable ; - local features ; - local otherflags ; - local result ; - - unknown-features ?= ; - - # Pull the executable out of the command line. At this point, the - # executable is just thrown away. - local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ; - executable = $(match[1]) ; - cmdline = $(match[2]) ; - - # List the prefix/feature pairs that we will be able to transform. - # Every kind of parameter not mentioned here will be placed in both - # cxxflags and linkflags, because we don't know where they should go. - local feature_kinds-D = "define" ; - local feature_kinds-I = "include" ; - local feature_kinds-L = "library-path" ; - local feature_kinds-l = "find-shared-library" ; - - while $(cmdline) { - - # Check for one of the feature prefixes we know about. If we - # find one (and the associated value is nonempty), convert it - # into a feature. - local match = [ MATCH "^(-.)(.*)" : $(cmdline) ] ; - local matched ; - if $(match) && $(match[2]) { - local prefix = $(match[1]) ; - if $(feature_kinds$(prefix)) { - local name = $(feature_kinds$(prefix)) ; - local add = [ add_feature $(prefix) $(name) $(cmdline) ] ; - - if $(add) { - - if $(add[1]) = pthread - { - # Uhm. It's not really nice that this MPI implementation - # uses -lpthread as opposed to -pthread. We do want to - # set multi, instead of -lpthread. - result += "multi" ; - MPI_EXTRA_REQUIREMENTS += "multi" ; - } - else - { - result += $(add[1]) ; - } - - cmdline = $(add[2]) ; - matched = yes ; - } - } - } - - # If we haven't matched a feature prefix, just grab the command-line - # argument itself. If we can map this argument to a feature - # (e.g., -pthread -> multi), then do so; otherwise, - # and add it to the list of "other" flags that we don't - # understand. - if ! $(matched) { - match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" : $(cmdline) ] ; - local value = $(match[1]) ; - cmdline = $(match[2]) ; - - # Check for multithreading support - if $(value) = "-pthread" || $(value) = "-pthreads" - { - result += "multi" ; - - # DPG: This is a hack intended to work around a BBv2 bug where - # requirements propagated from libraries are not checked for - # conflicts when BBv2 determines which "common" properties to - # apply to a target. In our case, the single property - # gets propagated from the common properties to Boost.MPI - # targets, even though multi is in the usage - # requirements of /mpi//mpi. - MPI_EXTRA_REQUIREMENTS += "multi" ; - } - else if [ MATCH "(.*[a-zA-Z0-9<>?-].*)" : $(value) ] { - otherflags += $(value) ; - } - } - } - - # If there are other flags that we don't understand, add them to the - # result as both and - if $(otherflags) { - for unknown in $(unknown-features) - { - result += "$(unknown)$(otherflags:J= )" ; - } - } - - return $(result) ; -} - -# Determine if it is safe to execute the given shell command by trying -# to execute it and determining whether the exit code is zero or -# not. Returns true for an exit code of zero, false otherwise. -local rule safe-shell-command ( cmdline ) -{ - local result = [ SHELL "$(cmdline) > /dev/null 2>/dev/null; if [ "$?" -eq "0" ]; then echo SSCOK; fi" ] ; - return [ MATCH ".*(SSCOK).*" : $(result) ] ; -} - -# Initialize the MPI module. -rule init ( mpicxx ? : options * : mpirun-with-options * ) -{ - if ! $(options) && $(.debug-configuration) - { - ECHO "===============MPI Auto-configuration===============" ; - } - - if ! $(mpicxx) && [ os.on-windows ] - { - # Try to auto-configure to the Microsoft Compute Cluster Pack - local cluster_pack_path_native = "C:\\Program Files\\Microsoft Compute Cluster Pack" ; - local cluster_pack_path = [ path.make $(cluster_pack_path_native) ] ; - if [ GLOB $(cluster_pack_path_native)\\Include : mpi.h ] - { - if $(.debug-configuration) - { - ECHO "Found Microsoft Compute Cluster Pack: $(cluster_pack_path_native)" ; - } - - # Pick up either the 32-bit or 64-bit library, depending on which address - # model the user has selected. Default to 32-bit. - options = $(cluster_pack_path)/Include - 64:$(cluster_pack_path)/Lib/amd64 - $(cluster_pack_path)/Lib/i386 - msmpi - msvc:_SECURE_SCL=0 - ; - - # Setup the "mpirun" equivalent (mpiexec) - .mpirun = "\"$(cluster_pack_path_native)\\Bin\\mpiexec.exe"\" ; - .mpirun_flags = -n ; - } - else if $(.debug-configuration) - { - ECHO "Did not find Microsoft Compute Cluster Pack in $(cluster_pack_path_native)." ; - } - } - - if ! $(options) - { - # Try to auto-detect options based on the wrapper compiler - local command = [ common.get-invocation-command mpi : mpic++ : $(mpicxx) ] ; - - if ! $(mpicxx) && ! $(command) - { - # Try "mpiCC", which is used by MPICH - command = [ common.get-invocation-command mpi : mpiCC ] ; - } - - if ! $(mpicxx) && ! $(command) - { - # Try "mpicxx", which is used by OpenMPI and MPICH2 - command = [ common.get-invocation-command mpi : mpicxx ] ; - } - - local result ; - local compile_flags ; - local link_flags ; - - if ! $(command) - { - # Do nothing: we'll complain later - } - # OpenMPI and newer versions of LAM-MPI have -showme:compile and - # -showme:link. - else if [ safe-shell-command "$(command) -showme:compile" ] && - [ safe-shell-command "$(command) -showme:link" ] - { - if $(.debug-configuration) - { - ECHO "Found recent LAM-MPI or Open MPI wrapper compiler: $(command)" ; - } - - compile_flags = [ SHELL "$(command) -showme:compile" ] ; - link_flags = [ SHELL "$(command) -showme:link" ] ; - - # Prepend COMPILER as the executable name, to match the format of - # other compilation commands. - compile_flags = "COMPILER $(compile_flags)" ; - link_flags = "COMPILER $(link_flags)" ; - } - # Look for LAM-MPI's -showme - else if [ safe-shell-command "$(command) -showme" ] - { - if $(.debug-configuration) - { - ECHO "Found older LAM-MPI wrapper compiler: $(command)" ; - } - - result = [ SHELL "$(command) -showme" ] ; - } - # Look for MPICH - else if [ safe-shell-command "$(command) -show" ] - { - if $(.debug-configuration) - { - ECHO "Found MPICH wrapper compiler: $(command)" ; - } - compile_flags = [ SHELL "$(command) -compile_info" ] ; - link_flags = [ SHELL "$(command) -link_info" ] ; - } - # Sun HPC and Ibm POE - else if [ SHELL "$(command) -v 2>/dev/null" ] - { - compile_flags = [ SHELL "$(command) -c -v -xtarget=native64 2>/dev/null" ] ; - - local back = [ MATCH "--------------------(.*)" : $(compile_flags) ] ; - if $(back) - { - # Sun HPC - if $(.debug-configuration) - { - ECHO "Found Sun MPI wrapper compiler: $(command)" ; - } - - compile_flags = [ MATCH "(.*)--------------------" : $(back) ] ; - compile_flags = [ MATCH "(.*)-v" : $(compile_flags) ] ; - link_flags = [ SHELL "$(command) -v -xtarget=native64 2>/dev/null" ] ; - link_flags = [ MATCH "--------------------(.*)" : $(link_flags) ] ; - link_flags = [ MATCH "(.*)--------------------" : $(link_flags) ] ; - - # strip out -v from compile options - local front = [ MATCH "(.*)-v" : $(link_flags) ] ; - local back = [ MATCH "-v(.*)" : $(link_flags) ] ; - link_flags = "$(front) $(back)" ; - front = [ MATCH "(.*)-xtarget=native64" : $(link_flags) ] ; - back = [ MATCH "-xtarget=native64(.*)" : $(link_flags) ] ; - link_flags = "$(front) $(back)" ; - } - else - { - # Ibm POE - if $(.debug-configuration) - { - ECHO "Found IBM MPI wrapper compiler: $(command)" ; - } - - # - compile_flags = [ SHELL "$(command) -c -v 2>/dev/null" ] ; - compile_flags = [ MATCH "(.*)exec: export.*" : $(compile_flags) ] ; - local front = [ MATCH "(.*)-v" : $(compile_flags) ] ; - local back = [ MATCH "-v(.*)" : $(compile_flags) ] ; - compile_flags = "$(front) $(back)" ; - front = [ MATCH "(.*)-c" : $(compile_flags) ] ; - back = [ MATCH "-c(.*)" : $(compile_flags) ] ; - compile_flags = "$(front) $(back)" ; - link_flags = $(compile_flags) ; - - # get location of mpif.h from mpxlf - local f_flags = [ SHELL "mpxlf -v 2>/dev/null" ] ; - f_flags = [ MATCH "(.*)exec: export.*" : $(f_flags) ] ; - front = [ MATCH "(.*)-v" : $(f_flags) ] ; - back = [ MATCH "-v(.*)" : $(f_flags) ] ; - f_flags = "$(front) $(back)" ; - f_flags = [ MATCH "xlf_r(.*)" : $(f_flags) ] ; - f_flags = [ MATCH "-F:mpxlf_r(.*)" : $(f_flags) ] ; - compile_flags = [ strip-eol $(compile_flags) ] ; - compile_flags = "$(compile_flags) $(f_flags)" ; - } - } - - if $(result) || $(compile_flags) && $(link_flags) - { - if $(result) - { - result = [ strip-eol $(result) ] ; - options = [ cmdline_to_features $(result) ] ; - } - else - { - compile_flags = [ strip-eol $(compile_flags) ] ; - link_flags = [ strip-eol $(link_flags) ] ; - - # Separately process compilation and link features, then combine - # them at the end. - local compile_features = [ cmdline_to_features $(compile_flags) - : "" ] ; - local link_features = [ cmdline_to_features $(link_flags) - : "" ] ; - options = $(compile_features) $(link_features) ; - } - - # If requested, display MPI configuration information. - if $(.debug-configuration) - { - if $(result) - { - ECHO " Wrapper compiler command line: $(result)" ; - } - else - { - local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" - : $(compile_flags) ] ; - ECHO "MPI compilation flags: $(match[2])" ; - local match = [ MATCH "^([^\" ]+|\"[^\"]+\") *(.*)$" - : $(link_flags) ] ; - ECHO "MPI link flags: $(match[2])" ; - } - } - } - else - { - if $(command) - { - ECHO "MPI auto-detection failed: unknown wrapper compiler $(command)" ; - ECHO "Please report this error to the Boost mailing list: http://www.boost.org" ; - } - else if $(mpicxx) - { - ECHO "MPI auto-detection failed: unable to find wrapper compiler $(mpicxx)" ; - } - else - { - ECHO "MPI auto-detection failed: unable to find wrapper compiler `mpic++' or `mpiCC'" ; - } - ECHO "You will need to manually configure MPI support." ; - } - - } - - # Find mpirun (or its equivalent) and its flags - if ! $(.mpirun) - { - .mpirun = - [ common.get-invocation-command mpi : mpirun : $(mpirun-with-options[1]) ] ; - .mpirun_flags = $(mpirun-with-options[2-]) ; - .mpirun_flags ?= -np ; - } - - if $(.debug-configuration) - { - if $(options) - { - echo "MPI build features: " ; - ECHO $(options) ; - } - - if $(.mpirun) - { - echo "MPI launcher: $(.mpirun) $(.mpirun_flags)" ; - } - - ECHO "====================================================" ; - } - - if $(options) - { - .configured = true ; - - # Set up the "mpi" alias - alias mpi : : : : $(options) ; - } -} - -# States whether MPI has bee configured -rule configured ( ) -{ - return $(.configured) ; -} - -# Returs the "extra" requirements needed to build MPI. These requirements are -# part of the /mpi//mpi library target, but they need to be added to anything -# that uses MPI directly to work around bugs in BBv2's propagation of -# requirements. -rule extra-requirements ( ) -{ - return $(MPI_EXTRA_REQUIREMENTS) ; -} - -# Support for testing; borrowed from Python -type.register RUN_MPI_OUTPUT ; -type.register RUN_MPI : : TEST ; - -class mpi-test-generator : generator -{ - import property-set ; - - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - self.composing = true ; - } - - rule run ( project name ? : property-set : sources * : multiple ? ) - { - # Generate an executable from the sources. This is the executable we will run. - local executable = - [ generators.construct $(project) $(name) : EXE : $(property-set) : $(sources) ] ; - - result = - [ construct-result $(executable[2-]) : $(project) $(name)-run : $(property-set) ] ; - } -} - -# Use mpi-test-generator to generate MPI tests from sources -generators.register - [ new mpi-test-generator mpi.capture-output : : RUN_MPI_OUTPUT ] ; - -generators.register-standard testing.expect-success - : RUN_MPI_OUTPUT : RUN_MPI ; - -# The number of processes to spawn when executing an MPI test. -feature mpi:processes : : free incidental ; - -# The flag settings on testing.capture-output do not -# apply to mpi.capture output at the moment. -# Redo this explicitly. -toolset.flags mpi.capture-output ARGS ; -rule capture-output ( target : sources * : properties * ) -{ - # Use the standard capture-output rule to run the tests - testing.capture-output $(target) : $(sources[1]) : $(properties) ; - - # Determine the number of processes we should run on. - local num_processes = [ property.select : $(properties) ] ; - num_processes = $(num_processes:G=) ; - - # serialize the MPI tests to avoid overloading systems - JAM_SEMAPHORE on $(target) = mpi-run-semaphore ; - - # We launch MPI processes using the "mpirun" equivalent specified by the user. - LAUNCHER on $(target) = - [ on $(target) return $(.mpirun) $(.mpirun_flags) $(num_processes) ] ; -} - -# Creates a set of test cases to be run through the MPI launcher. The name, sources, -# and requirements are the same as for any other test generator. However, schedule is -# a list of numbers, which indicates how many processes each test run will use. For -# example, passing 1 2 7 will run the test with 1 process, then 2 processes, then 7 -# 7 processes. The name provided is just the base name: the actual tests will be -# the name followed by a hypen, then the number of processes. -rule mpi-test ( name : sources * : requirements * : schedule * ) -{ - sources ?= $(name).cpp ; - schedule ?= 1 2 3 4 7 8 13 17 ; - - local result ; - for processes in $(schedule) - { - result += [ testing.make-test - run-mpi : $(sources) /boost/mpi//boost_mpi - : $(requirements) msvc:static $(processes) : $(name)-$(processes) ] ; - } - return $(result) ; -} diff --git a/jam-files/boost-build/tools/msvc-config.jam b/jam-files/boost-build/tools/msvc-config.jam deleted file mode 100644 index 6c71e3b0..00000000 --- a/jam-files/boost-build/tools/msvc-config.jam +++ /dev/null @@ -1,12 +0,0 @@ -#~ Copyright 2005 Rene Rivera. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Automatic configuration for VisualStudio toolset. To use, just import this module. - -import toolset : using ; - -ECHO "warning: msvc-config.jam is deprecated. Use 'using msvc : all ;' instead." ; - -using msvc : all ; - diff --git a/jam-files/boost-build/tools/msvc.jam b/jam-files/boost-build/tools/msvc.jam deleted file mode 100644 index e33a66d2..00000000 --- a/jam-files/boost-build/tools/msvc.jam +++ /dev/null @@ -1,1392 +0,0 @@ -# Copyright (c) 2003 David Abrahams. -# Copyright (c) 2005 Vladimir Prus. -# Copyright (c) 2005 Alexey Pakhunov. -# Copyright (c) 2006 Bojan Resnik. -# Copyright (c) 2006 Ilya Sokolov. -# Copyright (c) 2007 Rene Rivera -# Copyright (c) 2008 Jurko Gospodnetic -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -################################################################################ -# -# MSVC Boost Build toolset module. -# -------------------------------- -# -# All toolset versions need to have their location either auto-detected or -# explicitly specified except for the special 'default' version that expects the -# environment to find the needed tools or report an error. -# -################################################################################ - -import "class" : new ; -import common ; -import errors ; -import feature ; -import generators ; -import mc ; -import midl ; -import os ; -import path ; -import pch ; -import property ; -import rc ; -import toolset ; -import type ; - - -type.register MANIFEST : manifest ; -feature.feature embed-manifest : on off : incidental propagated ; - -type.register PDB : pdb ; - -################################################################################ -# -# Public rules. -# -################################################################################ - -# Initialize a specific toolset version configuration. As the result, path to -# compiler and, possible, program names are set up, and will be used when that -# version of compiler is requested. For example, you might have: -# -# using msvc : 6.5 : cl.exe ; -# using msvc : 7.0 : Y:/foo/bar/cl.exe ; -# -# The version parameter may be ommited: -# -# using msvc : : Z:/foo/bar/cl.exe ; -# -# The following keywords have special meanings when specified as versions: -# - all - all detected but not yet used versions will be marked as used -# with their default options. -# - default - this is an equivalent to an empty version. -# -# Depending on a supplied version, detected configurations and presence 'cl.exe' -# in the path different results may be achieved. The following table describes -# the possible scenarios: -# -# Nothing "x.y" -# Passed Nothing "x.y" detected, detected, -# version detected detected cl.exe in path cl.exe in path -# -# default Error Use "x.y" Create "default" Use "x.y" -# all None Use all None Use all -# x.y - Use "x.y" - Use "x.y" -# a.b Error Error Create "a.b" Create "a.b" -# -# "x.y" - refers to a detected version; -# "a.b" - refers to an undetected version. -# -# FIXME: Currently the command parameter and the property parameter -# seem to overlap in duties. Remove this duplication. This seems to be related -# to why someone started preparing to replace init with configure rules. -# -rule init ( - # The msvc version being configured. When omitted the tools invoked when no - # explicit version is given will be configured. - version ? - - # The command used to invoke the compiler. If not specified: - # - if version is given, default location for that version will be - # searched - # - # - if version is not given, default locations for MSVC 9.0, 8.0, 7.1, 7.0 - # and 6.* will be searched - # - # - if compiler is not found in the default locations, PATH will be - # searched. - : command * - - # Options may include: - # - # All options shared by multiple toolset types as handled by the - # common.handle-options() rule, e.g. , , , - # & . - # - # - # - # - # - # - # - # Exact tool names to be used by this msvc toolset configuration. - # - # - # Command through which to pipe the output of running the compiler. - # For example to pass the output to STLfilt. - # - # - # Global setup command to invoke before running any of the msvc tools. - # It will be passed additional option parameters depending on the actual - # target platform. - # - # - # - # - # Platform specific setup command to invoke before running any of the - # msvc tools used when builing a target for a specific platform, e.g. - # when building a 32 or 64 bit executable. - : options * -) -{ - if $(command) - { - options += $(command) ; - } - configure $(version) : $(options) ; -} - - -# 'configure' is a newer version of 'init'. The parameter 'command' is passed as -# a part of the 'options' list. See the 'init' rule comment for more detailed -# information. -# -rule configure ( version ? : options * ) -{ - switch $(version) - { - case "all" : - if $(options) - { - errors.error "MSVC toolset configuration: options should be" - "empty when '$(version)' is specified." ; - } - - # Configure (i.e. mark as used) all registered versions. - local all-versions = [ $(.versions).all ] ; - if ! $(all-versions) - { - if $(.debug-configuration) - { - ECHO "notice: [msvc-cfg] Asked to configure all registered" - "msvc toolset versions when there are none currently" - "registered." ; - } - } - else - { - for local v in $(all-versions) - { - # Note that there is no need to skip already configured - # versions here as this will request configure-really rule - # to configure the version using default options which will - # in turn cause it to simply do nothing in case the version - # has already been configured. - configure-really $(v) ; - } - } - - case "default" : - configure-really : $(options) ; - - case * : - configure-really $(version) : $(options) ; - } -} - - -# Sets up flag definitions dependent on the compiler version used. -# - 'version' is the version of compiler in N.M format. -# - 'conditions' is the property set to be used as flag conditions. -# - 'toolset' is the toolset for which flag settings are to be defined. -# This makes the rule reusable for other msvc-option-compatible compilers. -# -rule configure-version-specific ( toolset : version : conditions ) -{ - toolset.push-checking-for-flags-module unchecked ; - # Starting with versions 7.0, the msvc compiler have the /Zc:forScope and - # /Zc:wchar_t options that improve C++ standard conformance, but those - # options are off by default. If we are sure that the msvc version is at - # 7.*, add those options explicitly. We can be sure either if user specified - # version 7.* explicitly or if we auto-detected the version ourselves. - if ! [ MATCH ^(6\\.) : $(version) ] - { - toolset.flags $(toolset).compile CFLAGS $(conditions) : /Zc:forScope /Zc:wchar_t ; - toolset.flags $(toolset).compile.c++ C++FLAGS $(conditions) : /wd4675 ; - - # Explicitly disable the 'function is deprecated' warning. Some msvc - # versions have a bug, causing them to emit the deprecation warning even - # with /W0. - toolset.flags $(toolset).compile CFLAGS $(conditions)/off : /wd4996 ; - - if [ MATCH ^([78]\\.) : $(version) ] - { - # 64-bit compatibility warning deprecated since 9.0, see - # http://msdn.microsoft.com/en-us/library/yt4xw8fh.aspx - toolset.flags $(toolset).compile CFLAGS $(conditions)/all : /Wp64 ; - } - } - - # - # Processor-specific optimization. - # - - if [ MATCH ^([67]) : $(version) ] - { - # 8.0 deprecates some of the options. - toolset.flags $(toolset).compile CFLAGS $(conditions)/speed $(conditions)/space : /Ogiy /Gs ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/speed : /Ot ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/space : /Os ; - - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/ : /GB ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/i386 : /G3 ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/i486 : /G4 ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/$(.cpu-type-g5) : /G5 ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/$(.cpu-type-g6) : /G6 ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-i386)/$(.cpu-type-g7) : /G7 ; - - # Improve floating-point accuracy. Otherwise, some of C++ Boost's "math" - # tests will fail. - toolset.flags $(toolset).compile CFLAGS $(conditions) : /Op ; - - # 7.1 and below have single-threaded static RTL. - toolset.flags $(toolset).compile CFLAGS $(conditions)/off/static/single : /ML ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/on/static/single : /MLd ; - } - else - { - # 8.0 and above adds some more options. - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/ : /favor:blend ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/$(.cpu-type-em64t) : /favor:EM64T ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/$(.cpu-arch-amd64)/$(.cpu-type-amd64) : /favor:AMD64 ; - - # 8.0 and above only has multi-threaded static RTL. - toolset.flags $(toolset).compile CFLAGS $(conditions)/off/static/single : /MT ; - toolset.flags $(toolset).compile CFLAGS $(conditions)/on/static/single : /MTd ; - - # Specify target machine type so the linker will not need to guess. - toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-amd64) : /MACHINE:X64 ; - toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-i386) : /MACHINE:X86 ; - toolset.flags $(toolset).link LINKFLAGS $(conditions)/$(.cpu-arch-ia64) : /MACHINE:IA64 ; - - # Make sure that manifest will be generated even if there is no - # dependencies to put there. - toolset.flags $(toolset).link LINKFLAGS $(conditions)/off : /MANIFEST ; - } - toolset.pop-checking-for-flags-module ; -} - - -# Registers this toolset including all of its flags, features & generators. Does -# nothing on repeated calls. -# -rule register-toolset ( ) -{ - if ! msvc in [ feature.values toolset ] - { - register-toolset-really ; - } -} - - -# Declare action for creating static libraries. If library exists, remove it -# before adding files. See -# http://article.gmane.org/gmane.comp.lib.boost.build/4241 for rationale. -if [ os.name ] in NT -{ - # The 'DEL' command would issue a message to stdout if the file does not - # exist, so need a check. - actions archive - { - if exist "$(<[1])" DEL "$(<[1])" - $(.LD) $(AROPTIONS) /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - } -} -else -{ - actions archive - { - $(.RM) "$(<[1])" - $(.LD) $(AROPTIONS) /out:"$(<[1])" @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - } -} - - -# For the assembler the following options are turned on by default: -# -# -Zp4 align structures to 4 bytes -# -Cp preserve case of user identifiers -# -Cx preserve case in publics, externs -# -actions compile.asm -{ - $(.ASM) -c -Zp4 -Cp -Cx -D$(DEFINES) $(ASMFLAGS) $(USER_ASMFLAGS) -Fo "$(<:W)" "$(>:W)" -} - - -rule compile.c ( targets + : sources * : properties * ) -{ - C++FLAGS on $(targets[1]) = ; - get-rspline $(targets) : -TC ; - compile-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ; -} - - -rule compile.c.preprocess ( targets + : sources * : properties * ) -{ - C++FLAGS on $(targets[1]) = ; - get-rspline $(targets) : -TC ; - preprocess-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ; -} - - -rule compile.c.pch ( targets + : sources * : properties * ) -{ - C++FLAGS on $(targets[1]) = ; - get-rspline $(targets[1]) : -TC ; - get-rspline $(targets[2]) : -TC ; - local pch-source = [ on $(<) return $(PCH_SOURCE) ] ; - if $(pch-source) - { - DEPENDS $(<) : $(pch-source) ; - compile-c-c++-pch-s $(targets) : $(sources) $(pch-source) ; - } - else - { - compile-c-c++-pch $(targets) : $(sources) ; - } -} - -toolset.flags msvc YLOPTION : "-Yl" ; - -# Action for running the C/C++ compiler without using precompiled headers. -# -# WARNING: Synchronize any changes this in action with intel-win -# -# Notes regarding PDB generation, for when we use on/database -# -# 1. PDB_CFLAG is only set for on/database, ensuring that the /Fd flag is dropped if PDB_CFLAG is empty -# -# 2. When compiling executables's source files, PDB_NAME is set on a per-source file basis by rule compile-c-c++. -# The linker will pull these into the executable's PDB -# -# 3. When compiling library's source files, PDB_NAME is updated to .pdb for each source file by rule archive, -# as in this case the compiler must be used to create a single PDB for our library. -# -actions compile-c-c++ bind PDB_NAME -{ - $(.CC) @"@($(<[1]:W).rsp:E="$(>[1]:W)" -Fo"$(<[1]:W)" $(PDB_CFLAG)"$(PDB_NAME)" -Yu"$(>[3]:D=)" -Fp"$(>[2]:W)" $(CC_RSPLINE))" $(.CC.FILTER) -} - -actions preprocess-c-c++ bind PDB_NAME -{ - $(.CC) @"@($(<[1]:W).rsp:E="$(>[1]:W)" -E $(PDB_CFLAG)"$(PDB_NAME)" -Yu"$(>[3]:D=)" -Fp"$(>[2]:W)" $(CC_RSPLINE))" >"$(<[1]:W)" -} - -rule compile-c-c++ ( targets + : sources * ) -{ - DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_HEADER) ] ; - DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_FILE) ] ; - PDB_NAME on $(<) = $(<:S=.pdb) ; -} - -rule preprocess-c-c++ ( targets + : sources * ) -{ - DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_HEADER) ] ; - DEPENDS $(<[1]) : [ on $(<[1]) return $(PCH_FILE) ] ; - PDB_NAME on $(<) = $(<:S=.pdb) ; -} - -# Action for running the C/C++ compiler using precompiled headers. In addition -# to whatever else it needs to compile, this action also adds a temporary source -# .cpp file used to compile the precompiled headers themselves. -# -# The global .escaped-double-quote variable is used to avoid messing up Emacs -# syntax highlighting in the messy N-quoted code below. -actions compile-c-c++-pch -{ - $(.CC) @"@($(<[1]:W).rsp:E="$(>[2]:W)" -Fo"$(<[2]:W)" -Yc"$(>[1]:D=)" $(YLOPTION)"__bjam_pch_symbol_$(>[1]:D=)" -Fp"$(<[1]:W)" $(CC_RSPLINE))" "@($(<[1]:W).cpp:E=#include $(.escaped-double-quote)$(>[1]:D=)$(.escaped-double-quote)$(.nl))" $(.CC.FILTER) -} - - -# Action for running the C/C++ compiler using precompiled headers. An already -# built source file for compiling the precompiled headers is expected to be -# given as one of the source parameters. -actions compile-c-c++-pch-s -{ - $(.CC) @"@($(<[1]:W).rsp:E="$(>[2]:W)" -Fo"$(<[2]:W)" -Yc"$(>[1]:D=)" $(YLOPTION)"__bjam_pch_symbol_$(>[1]:D=)" -Fp"$(<[1]:W)" $(CC_RSPLINE))" $(.CC.FILTER) -} - - -rule compile.c++ ( targets + : sources * : properties * ) -{ - get-rspline $(targets) : -TP ; - compile-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ; -} - -rule compile.c++.preprocess ( targets + : sources * : properties * ) -{ - get-rspline $(targets) : -TP ; - preprocess-c-c++ $(<) : $(>) [ on $(<) return $(PCH_FILE) ] [ on $(<) return $(PCH_HEADER) ] ; -} - - -rule compile.c++.pch ( targets + : sources * : properties * ) -{ - get-rspline $(targets[1]) : -TP ; - get-rspline $(targets[2]) : -TP ; - local pch-source = [ on $(<) return $(PCH_SOURCE) ] ; - if $(pch-source) - { - DEPENDS $(<) : $(pch-source) ; - compile-c-c++-pch-s $(targets) : $(sources) $(pch-source) ; - } - else - { - compile-c-c++-pch $(targets) : $(sources) ; - } -} - - -# See midl.jam for details. -# -actions compile.idl -{ - $(.IDL) /nologo @"@($(<[1]:W).rsp:E=$(.nl)"$(>:W)" $(.nl)-D$(DEFINES) $(.nl)"-I$(INCLUDES:W)" $(.nl)-U$(UNDEFS) $(.nl)$(MIDLFLAGS) $(.nl)/tlb "$(<[1]:W)" $(.nl)/h "$(<[2]:W)" $(.nl)/iid "$(<[3]:W)" $(.nl)/proxy "$(<[4]:W)" $(.nl)/dlldata "$(<[5]:W)")" - $(.TOUCH_FILE) "$(<[4]:W)" - $(.TOUCH_FILE) "$(<[5]:W)" -} - - -actions compile.mc -{ - $(.MC) $(MCFLAGS) -h "$(<[1]:DW)" -r "$(<[2]:DW)" "$(>:W)" -} - - -actions compile.rc -{ - $(.RC) -l 0x409 -U$(UNDEFS) -D$(DEFINES) -I"$(INCLUDES:W)" -fo "$(<:W)" "$(>:W)" -} - - -rule link ( targets + : sources * : properties * ) -{ - if on in $(properties) - { - msvc.manifest $(targets) : $(sources) : $(properties) ; - } -} - -rule link.dll ( targets + : sources * : properties * ) -{ - DEPENDS $(<) : [ on $(<) return $(DEF_FILE) ] ; - if on in $(properties) - { - msvc.manifest.dll $(targets) : $(sources) : $(properties) ; - } -} - -# Incremental linking a DLL causes no end of problems: if the actual exports do -# not change, the import .lib file is never updated. Therefore, the .lib is -# always out-of-date and gets rebuilt every time. I am not sure that incremental -# linking is such a great idea in general, but in this case I am sure we do not -# want it. - -# Windows manifest is a new way to specify dependencies on managed DotNet -# assemblies and Windows native DLLs. The manifests are embedded as resources -# and are useful in any PE target (both DLL and EXE). - -if [ os.name ] in NT -{ - actions link bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE - { - $(.LD) $(LINKFLAGS) /out:"$(<[1]:W)" /LIBPATH:"$(LINKPATH:W)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - if %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% - } - - actions manifest - { - if exist "$(<[1]).manifest" ( - $(.MT) -manifest "$(<[1]).manifest" "-outputresource:$(<[1]);1" - ) - } - - actions link.dll bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE - { - $(.LD) /DLL $(LINKFLAGS) /out:"$(<[1]:W)" /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:"$(DEF_FILE)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - if %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% - } - - actions manifest.dll - { - if exist "$(<[1]).manifest" ( - $(.MT) -manifest "$(<[1]).manifest" "-outputresource:$(<[1]);2" - ) - } -} -else -{ - actions link bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE - { - $(.LD) $(LINKFLAGS) /out:"$(<[1]:W)" /LIBPATH:"$(LINKPATH:W)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - } - - actions manifest - { - if test -e "$(<[1]).manifest"; then - $(.MT) -manifest "$(<[1]:W).manifest" "-outputresource:$(<[1]:W);1" - fi - } - - actions link.dll bind DEF_FILE LIBRARIES_MENTIONED_BY_FILE - { - $(.LD) /DLL $(LINKFLAGS) /out:"$(<[1]:W)" /IMPLIB:"$(<[2]:W)" /LIBPATH:"$(LINKPATH:W)" /def:"$(DEF_FILE)" $(OPTIONS) @"@($(<[1]:W).rsp:E=$(.nl)"$(>)" $(.nl)$(LIBRARIES_MENTIONED_BY_FILE) $(.nl)$(LIBRARIES) $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_ST).lib" $(.nl)"$(LIBRARY_OPTION)$(FINDLIBS_SA).lib")" - } - - actions manifest.dll - { - if test -e "$(<[1]).manifest"; then - $(.MT) -manifest "$(<[1]:W).manifest" "-outputresource:$(<[1]:W);2" - fi - } -} - -# this rule sets up the pdb file that will be used when generating static -# libraries and the debug-store option is database, so that the compiler -# puts all debug info into a single .pdb file named after the library -# -# Poking at source targets this way is probably not clean, but it's the -# easiest approach. -rule archive ( targets + : sources * : properties * ) -{ - PDB_NAME on $(>) = $(<:S=.pdb) ; -} - -################################################################################ -# -# Classes. -# -################################################################################ - -class msvc-pch-generator : pch-generator -{ - import property-set ; - - rule run-pch ( project name ? : property-set : sources * ) - { - # Searching for the header and source file in the sources. - local pch-header ; - local pch-source ; - for local s in $(sources) - { - if [ type.is-derived [ $(s).type ] H ] - { - pch-header = $(s) ; - } - else if - [ type.is-derived [ $(s).type ] CPP ] || - [ type.is-derived [ $(s).type ] C ] - { - pch-source = $(s) ; - } - } - - if ! $(pch-header) - { - errors.user-error "can not build pch without pch-header" ; - } - - # If we do not have the PCH source - that is fine. We will just create a - # temporary .cpp file in the action. - - local generated = [ generator.run $(project) $(name) - : [ property-set.create - # Passing of is a dirty trick, needed because - # non-composing generators with multiple inputs are subtly - # broken. For more detailed information see: - # https://zigzag.cs.msu.su:7813/boost.build/ticket/111 - $(pch-source) - [ $(property-set).raw ] ] - : $(pch-header) ] ; - - local pch-file ; - for local g in $(generated) - { - if [ type.is-derived [ $(g).type ] PCH ] - { - pch-file = $(g) ; - } - } - - return [ property-set.create $(pch-header) - $(pch-file) ] $(generated) ; - } -} - - -################################################################################ -# -# Local rules. -# -################################################################################ - -# Detects versions listed as '.known-versions' by checking registry information, -# environment variables & default paths. Supports both native Windows and -# Cygwin. -# -local rule auto-detect-toolset-versions ( ) -{ - if [ os.name ] in NT CYGWIN - { - # Get installation paths from the registry. - for local i in $(.known-versions) - { - if $(.version-$(i)-reg) - { - local vc-path ; - for local x in "" "Wow6432Node\\" - { - vc-path += [ W32_GETREG - "HKEY_LOCAL_MACHINE\\SOFTWARE\\"$(x)"\\Microsoft\\"$(.version-$(i)-reg) - : "ProductDir" ] ; - } - - if $(vc-path) - { - vc-path = [ path.join [ path.make-NT $(vc-path[1]) ] "bin" ] ; - register-configuration $(i) : [ path.native $(vc-path[1]) ] ; - } - } - } - } - - # Check environment and default installation paths. - for local i in $(.known-versions) - { - if ! $(i) in [ $(.versions).all ] - { - register-configuration $(i) : [ default-path $(i) ] ; - } - } -} - - -# Worker rule for toolset version configuration. Takes an explicit version id or -# nothing in case it should configure the default toolset version (the first -# registered one or a new 'default' one in case no toolset versions have been -# registered yet). -# -local rule configure-really ( version ? : options * ) -{ - local v = $(version) ; - - # Decide what the 'default' version is. - if ! $(v) - { - # Take the first registered (i.e. auto-detected) version. - version = [ $(.versions).all ] ; - version = $(version[1]) ; - v = $(version) ; - - # Note: 'version' can still be empty at this point if no versions have - # been auto-detected. - version ?= "default" ; - } - - # Version alias -> real version number. - if $(.version-alias-$(version)) - { - version = $(.version-alias-$(version)) ; - } - - # Check whether the selected configuration is already in use. - if $(version) in [ $(.versions).used ] - { - # Allow multiple 'toolset.using' calls for the same configuration if the - # identical sets of options are used. - if $(options) && ( $(options) != [ $(.versions).get $(version) : options ] ) - { - errors.error "MSVC toolset configuration: Toolset version" - "'$(version)' already configured." ; - } - } - else - { - # Register a new configuration. - $(.versions).register $(version) ; - - # Add user-supplied to auto-detected options. - options = [ $(.versions).get $(version) : options ] $(options) ; - - # Mark the configuration as 'used'. - $(.versions).use $(version) ; - - # Generate conditions and save them. - local conditions = [ common.check-init-parameters msvc : version $(v) ] - ; - - $(.versions).set $(version) : conditions : $(conditions) ; - - local command = [ feature.get-values : $(options) ] ; - - # If version is specified, we try to search first in default paths, and - # only then in PATH. - command = [ common.get-invocation-command msvc : cl.exe : $(command) : - [ default-paths $(version) ] : $(version) ] ; - - common.handle-options msvc : $(conditions) : $(command) : $(options) ; - - if ! $(version) - { - # Even if version is not explicitly specified, try to detect the - # version from the path. - # FIXME: We currently detect both Microsoft Visual Studio 9.0 and - # 9.0express as 9.0 here. - if [ MATCH "(Microsoft Visual Studio 10)" : $(command) ] - { - version = 10.0 ; - } - else if [ MATCH "(Microsoft Visual Studio 9)" : $(command) ] - { - version = 9.0 ; - } - else if [ MATCH "(Microsoft Visual Studio 8)" : $(command) ] - { - version = 8.0 ; - } - else if [ MATCH "(NET 2003[\/\\]VC7)" : $(command) ] - { - version = 7.1 ; - } - else if [ MATCH "(Microsoft Visual C\\+\\+ Toolkit 2003)" : - $(command) ] - { - version = 7.1toolkit ; - } - else if [ MATCH "(.NET[\/\\]VC7)" : $(command) ] - { - version = 7.0 ; - } - else - { - version = 6.0 ; - } - } - - # Generate and register setup command. - - local below-8.0 = [ MATCH ^([67]\\.) : $(version) ] ; - - local cpu = i386 amd64 ia64 ; - if $(below-8.0) - { - cpu = i386 ; - } - - local setup-amd64 ; - local setup-i386 ; - local setup-ia64 ; - - if $(command) - { - # TODO: Note that if we specify a non-existant toolset version then - # this rule may find and use a corresponding compiler executable - # belonging to an incorrect toolset version. For example, if you - # have only MSVC 7.1 installed, have its executable on the path and - # specify you want Boost Build to use MSVC 9.0, then you want Boost - # Build to report an error but this may cause it to silently use the - # MSVC 7.1 compiler even though it thinks it is using the msvc-9.0 - # toolset version. - command = [ common.get-absolute-tool-path $(command[-1]) ] ; - } - - if $(command) - { - local parent = [ path.make $(command) ] ; - parent = [ path.parent $(parent) ] ; - parent = [ path.native $(parent) ] ; - - # Setup will be used if the command name has been specified. If - # setup is not specified explicitly then a default setup script will - # be used instead. Setup scripts may be global or arhitecture/ - # /platform/cpu specific. Setup options are used only in case of - # global setup scripts. - - # Default setup scripts provided with different VC distributions: - # - # VC 7.1 had only the vcvars32.bat script specific to 32 bit i386 - # builds. It was located in the bin folder for the regular version - # and in the root folder for the free VC 7.1 tools. - # - # Later 8.0 & 9.0 versions introduce separate platform specific - # vcvars*.bat scripts (e.g. 32 bit, 64 bit AMD or 64 bit Itanium) - # located in or under the bin folder. Most also include a global - # vcvarsall.bat helper script located in the root folder which runs - # one of the aforementioned vcvars*.bat scripts based on the options - # passed to it. So far only the version coming with some PlatformSDK - # distributions does not include this top level script but to - # support those we need to fall back to using the worker scripts - # directly in case the top level script can not be found. - - local global-setup = [ feature.get-values : $(options) ] ; - global-setup = $(global-setup[1]) ; - if ! $(below-8.0) - { - global-setup ?= [ locate-default-setup $(command) : $(parent) : - vcvarsall.bat ] ; - } - - local default-setup-amd64 = vcvarsx86_amd64.bat ; - local default-setup-i386 = vcvars32.bat ; - local default-setup-ia64 = vcvarsx86_ia64.bat ; - - # http://msdn2.microsoft.com/en-us/library/x4d2c09s(VS.80).aspx and - # http://msdn2.microsoft.com/en-us/library/x4d2c09s(vs.90).aspx - # mention an x86_IPF option, that seems to be a documentation bug - # and x86_ia64 is the correct option. - local default-global-setup-options-amd64 = x86_amd64 ; - local default-global-setup-options-i386 = x86 ; - local default-global-setup-options-ia64 = x86_ia64 ; - - # When using 64-bit Windows, and targeting 64-bit, it is possible to - # use a native 64-bit compiler, selected by the "amd64" & "ia64" - # parameters to vcvarsall.bat. There are two variables we can use -- - # PROCESSOR_ARCHITECTURE and PROCESSOR_IDENTIFIER. The first is - # 'x86' when running 32-bit Windows, no matter which processor is - # used, and 'AMD64' on 64-bit windows on x86 (either AMD64 or EM64T) - # Windows. - # - if [ MATCH ^(AMD64) : [ os.environ PROCESSOR_ARCHITECTURE ] ] - { - default-global-setup-options-amd64 = amd64 ; - } - # TODO: The same 'native compiler usage' should be implemented for - # the Itanium platform by using the "ia64" parameter. For this - # though we need someone with access to this platform who can find - # out how to correctly detect this case. - else if $(somehow-detect-the-itanium-platform) - { - default-global-setup-options-ia64 = ia64 ; - } - - local setup-prefix = "call " ; - local setup-suffix = " >nul"$(.nl) ; - if ! [ os.name ] in NT - { - setup-prefix = "cmd.exe /S /C call " ; - setup-suffix = " \">nul\" \"&&\" " ; - } - - for local c in $(cpu) - { - local setup-options ; - - setup-$(c) = [ feature.get-values : $(options) ] ; - - if ! $(setup-$(c))-is-not-empty - { - if $(global-setup)-is-not-empty - { - setup-$(c) = $(global-setup) ; - - # If needed we can easily add using configuration flags - # here for overriding which options get passed to the - # global setup command for which target platform: - # setup-options = [ feature.get-values : $(options) ] ; - - setup-options ?= $(default-global-setup-options-$(c)) ; - } - else - { - setup-$(c) = [ locate-default-setup $(command) : $(parent) : $(default-setup-$(c)) ] ; - } - } - - # Cygwin to Windows path translation. - setup-$(c) = "\""$(setup-$(c):W)"\"" ; - - # Append setup options to the setup name and add the final setup - # prefix & suffix. - setup-options ?= "" ; - setup-$(c) = $(setup-prefix)$(setup-$(c):J=" ")" "$(setup-options:J=" ")$(setup-suffix) ; - } - } - - # Get tool names (if any) and finish setup. - - compiler = [ feature.get-values : $(options) ] ; - compiler ?= cl ; - - linker = [ feature.get-values : $(options) ] ; - linker ?= link ; - - resource-compiler = [ feature.get-values : $(options) ] ; - resource-compiler ?= rc ; - - # Turn on some options for i386 assembler - # -coff generate COFF format object file (compatible with cl.exe output) - local default-assembler-amd64 = ml64 ; - local default-assembler-i386 = "ml -coff" ; - local default-assembler-ia64 = ias ; - - assembler = [ feature.get-values : $(options) ] ; - - idl-compiler = [ feature.get-values : $(options) ] ; - idl-compiler ?= midl ; - - mc-compiler = [ feature.get-values : $(options) ] ; - mc-compiler ?= mc ; - - manifest-tool = [ feature.get-values : $(options) ] ; - manifest-tool ?= mt ; - - local cc-filter = [ feature.get-values : $(options) ] ; - - for local c in $(cpu) - { - # Setup script is not required in some configurations. - setup-$(c) ?= "" ; - - local cpu-conditions = $(conditions)/$(.cpu-arch-$(c)) ; - - if $(.debug-configuration) - { - for local cpu-condition in $(cpu-conditions) - { - ECHO "notice: [msvc-cfg] condition: '$(cpu-condition)', setup: '$(setup-$(c))'" ; - } - } - - local cpu-assembler = $(assembler) ; - cpu-assembler ?= $(default-assembler-$(c)) ; - - toolset.flags msvc.compile .CC $(cpu-conditions) : $(setup-$(c))$(compiler) /Zm800 -nologo ; - toolset.flags msvc.compile .RC $(cpu-conditions) : $(setup-$(c))$(resource-compiler) ; - toolset.flags msvc.compile .ASM $(cpu-conditions) : $(setup-$(c))$(cpu-assembler) -nologo ; - toolset.flags msvc.link .LD $(cpu-conditions) : $(setup-$(c))$(linker) /NOLOGO /INCREMENTAL:NO ; - toolset.flags msvc.archive .LD $(cpu-conditions) : $(setup-$(c))$(linker) /lib /NOLOGO ; - toolset.flags msvc.compile .IDL $(cpu-conditions) : $(setup-$(c))$(idl-compiler) ; - toolset.flags msvc.compile .MC $(cpu-conditions) : $(setup-$(c))$(mc-compiler) ; - - toolset.flags msvc.link .MT $(cpu-conditions) : $(setup-$(c))$(manifest-tool) -nologo ; - - if $(cc-filter) - { - toolset.flags msvc .CC.FILTER $(cpu-conditions) : "|" $(cc-filter) ; - } - } - - # Set version-specific flags. - configure-version-specific msvc : $(version) : $(conditions) ; - } -} - - -# Returns the default installation path for the given version. -# -local rule default-path ( version ) -{ - # Use auto-detected path if possible. - local path = [ feature.get-values : [ $(.versions).get $(version) - : options ] ] ; - - if $(path) - { - path = $(path:D) ; - } - else - { - # Check environment. - if $(.version-$(version)-env) - { - local vc-path = [ os.environ $(.version-$(version)-env) ] ; - if $(vc-path) - { - vc-path = [ path.make $(vc-path) ] ; - vc-path = [ path.join $(vc-path) $(.version-$(version)-envpath) ] ; - vc-path = [ path.native $(vc-path) ] ; - - path = $(vc-path) ; - } - } - - # Check default path. - if ! $(path) && $(.version-$(version)-path) - { - path = [ path.native [ path.join $(.ProgramFiles) $(.version-$(version)-path) ] ] ; - } - } - - return $(path) ; -} - - -# Returns either the default installation path (if 'version' is not empty) or -# list of all known default paths (if no version is given) -# -local rule default-paths ( version ? ) -{ - local possible-paths ; - - if $(version) - { - possible-paths += [ default-path $(version) ] ; - } - else - { - for local i in $(.known-versions) - { - possible-paths += [ default-path $(i) ] ; - } - } - - return $(possible-paths) ; -} - - -rule get-rspline ( target : lang-opt ) -{ - CC_RSPLINE on $(target) = [ on $(target) return $(lang-opt) -U$(UNDEFS) - $(CFLAGS) $(C++FLAGS) $(OPTIONS) -c $(.nl)-D$(DEFINES) - $(.nl)\"-I$(INCLUDES:W)\" ] ; -} - -class msvc-linking-generator : linking-generator -{ - # Calls the base version. If necessary, also create a target for the - # manifest file.specifying source's name as the name of the created - # target. As result, the PCH will be named whatever.hpp.gch, and not - # whatever.gch. - rule generated-targets ( sources + : property-set : project name ? ) - { - local result = [ linking-generator.generated-targets $(sources) - : $(property-set) : $(project) $(name) ] ; - - if $(result) - { - local name-main = [ $(result[0]).name ] ; - local action = [ $(result[0]).action ] ; - - if [ $(property-set).get ] = "on" - { - # We force exact name on PDB. The reason is tagging -- the tag rule may - # reasonably special case some target types, like SHARED_LIB. The tag rule - # will not catch PDB, and it cannot even easily figure if PDB is paired with - # SHARED_LIB or EXE or something else. Because PDB always get the - # same name as the main target, with .pdb as extension, just force it. - local target = [ class.new file-target $(name-main:S=.pdb) exact : PDB : $(project) : $(action) ] ; - local registered-target = [ virtual-target.register $(target) ] ; - if $(target) != $(registered-target) - { - $(action).replace-targets $(target) : $(registered-target) ; - } - result += $(registered-target) ; - } - - if [ $(property-set).get ] = "off" - { - # Manifest is evil target. It has .manifest appened to the name of - # main target, including extension. E.g. a.exe.manifest. We use 'exact' - # name because to achieve this effect. - local target = [ class.new file-target $(name-main).manifest exact : MANIFEST : $(project) : $(action) ] ; - local registered-target = [ virtual-target.register $(target) ] ; - if $(target) != $(registered-target) - { - $(action).replace-targets $(target) : $(registered-target) ; - } - result += $(registered-target) ; - } - } - return $(result) ; - } -} - - - -# Unsafe worker rule for the register-toolset() rule. Must not be called -# multiple times. -# -local rule register-toolset-really ( ) -{ - feature.extend toolset : msvc ; - - # Intel and msvc supposedly have link-compatible objects. - feature.subfeature toolset msvc : vendor : intel : propagated optional ; - - # Inherit MIDL flags. - toolset.inherit-flags msvc : midl ; - - # Inherit MC flags. - toolset.inherit-flags msvc : mc ; - - # Dynamic runtime comes only in MT flavour. - toolset.add-requirements - msvc,shared:multi ; - - # Declare msvc toolset specific features. - { - feature.feature debug-store : object database : propagated ; - feature.feature pch-source : : dependency free ; - } - - # Declare generators. - { - # TODO: Is it possible to combine these? Make the generators - # non-composing so that they do not convert each source into a separate - # .rsp file. - generators.register [ new msvc-linking-generator - msvc.link : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : EXE : msvc ] ; - generators.register [ new msvc-linking-generator - msvc.link.dll : OBJ SEARCHED_LIB STATIC_LIB IMPORT_LIB : SHARED_LIB IMPORT_LIB : msvc ] ; - - generators.register-archiver msvc.archive : OBJ : STATIC_LIB : msvc ; - generators.register-c-compiler msvc.compile.c++ : CPP : OBJ : msvc ; - generators.register-c-compiler msvc.compile.c : C : OBJ : msvc ; - generators.register-c-compiler msvc.compile.c++.preprocess : CPP : PREPROCESSED_CPP : msvc ; - generators.register-c-compiler msvc.compile.c.preprocess : C : PREPROCESSED_C : msvc ; - - # Using 'register-c-compiler' adds the build directory to INCLUDES. - generators.register-c-compiler msvc.compile.rc : RC : OBJ(%_res) : msvc ; - generators.override msvc.compile.rc : rc.compile.resource ; - generators.register-standard msvc.compile.asm : ASM : OBJ : msvc ; - - generators.register-c-compiler msvc.compile.idl : IDL : MSTYPELIB H C(%_i) C(%_proxy) C(%_dlldata) : msvc ; - generators.override msvc.compile.idl : midl.compile.idl ; - - generators.register-standard msvc.compile.mc : MC : H RC : msvc ; - generators.override msvc.compile.mc : mc.compile ; - - # Note: the 'H' source type will catch both '.h' and '.hpp' headers as - # the latter have their HPP type derived from H. The type of compilation - # is determined entirely by the destination type. - generators.register [ new msvc-pch-generator msvc.compile.c.pch : H : C_PCH OBJ : on msvc ] ; - generators.register [ new msvc-pch-generator msvc.compile.c++.pch : H : CPP_PCH OBJ : on msvc ] ; - - generators.override msvc.compile.c.pch : pch.default-c-pch-generator ; - generators.override msvc.compile.c++.pch : pch.default-cpp-pch-generator ; - } - - toolset.flags msvc.compile PCH_FILE on : ; - toolset.flags msvc.compile PCH_SOURCE on : ; - toolset.flags msvc.compile PCH_HEADER on : ; - - # - # Declare flags for compilation. - # - - toolset.flags msvc.compile CFLAGS speed : /O2 ; - toolset.flags msvc.compile CFLAGS space : /O1 ; - - toolset.flags msvc.compile CFLAGS $(.cpu-arch-ia64)/$(.cpu-type-itanium) : /G1 ; - toolset.flags msvc.compile CFLAGS $(.cpu-arch-ia64)/$(.cpu-type-itanium2) : /G2 ; - - toolset.flags msvc.compile CFLAGS on/object : /Z7 ; - toolset.flags msvc.compile CFLAGS on/database : /Zi ; - toolset.flags msvc.compile CFLAGS off : /Od ; - toolset.flags msvc.compile CFLAGS off : /Ob0 ; - toolset.flags msvc.compile CFLAGS on : /Ob1 ; - toolset.flags msvc.compile CFLAGS full : /Ob2 ; - - toolset.flags msvc.compile CFLAGS on : /W3 ; - toolset.flags msvc.compile CFLAGS off : /W0 ; - toolset.flags msvc.compile CFLAGS all : /W4 ; - toolset.flags msvc.compile CFLAGS on : /WX ; - - toolset.flags msvc.compile C++FLAGS on/off/off : /EHs ; - toolset.flags msvc.compile C++FLAGS on/off/on : /EHsc ; - toolset.flags msvc.compile C++FLAGS on/on/off : /EHa ; - toolset.flags msvc.compile C++FLAGS on/on/on : /EHac ; - - # By default 8.0 enables rtti support while prior versions disabled it. We - # simply enable or disable it explicitly so we do not have to depend on this - # default behaviour. - toolset.flags msvc.compile CFLAGS on : /GR ; - toolset.flags msvc.compile CFLAGS off : /GR- ; - toolset.flags msvc.compile CFLAGS off/shared : /MD ; - toolset.flags msvc.compile CFLAGS on/shared : /MDd ; - - toolset.flags msvc.compile CFLAGS off/static/multi : /MT ; - toolset.flags msvc.compile CFLAGS on/static/multi : /MTd ; - - toolset.flags msvc.compile OPTIONS : ; - toolset.flags msvc.compile.c++ OPTIONS : ; - - toolset.flags msvc.compile PDB_CFLAG on/database : /Fd ; - - toolset.flags msvc.compile DEFINES ; - toolset.flags msvc.compile UNDEFS ; - toolset.flags msvc.compile INCLUDES ; - - # Declare flags for the assembler. - toolset.flags msvc.compile.asm USER_ASMFLAGS ; - - toolset.flags msvc.compile.asm ASMFLAGS on : "/Zi /Zd" ; - - toolset.flags msvc.compile.asm ASMFLAGS on : /W3 ; - toolset.flags msvc.compile.asm ASMFLAGS off : /W0 ; - toolset.flags msvc.compile.asm ASMFLAGS all : /W4 ; - toolset.flags msvc.compile.asm ASMFLAGS on : /WX ; - - toolset.flags msvc.compile.asm DEFINES ; - - # Declare flags for linking. - { - toolset.flags msvc.link PDB_LINKFLAG on/database : /PDB: ; # not used yet - toolset.flags msvc.link LINKFLAGS on : /DEBUG ; - toolset.flags msvc.link DEF_FILE ; - - # The linker disables the default optimizations when using /DEBUG so we - # have to enable them manually for release builds with debug symbols. - toolset.flags msvc LINKFLAGS on/off : /OPT:REF,ICF ; - - toolset.flags msvc LINKFLAGS console : /subsystem:console ; - toolset.flags msvc LINKFLAGS gui : /subsystem:windows ; - toolset.flags msvc LINKFLAGS wince : /subsystem:windowsce ; - toolset.flags msvc LINKFLAGS native : /subsystem:native ; - toolset.flags msvc LINKFLAGS auto : /subsystem:posix ; - - toolset.flags msvc.link OPTIONS ; - toolset.flags msvc.link LINKPATH ; - - toolset.flags msvc.link FINDLIBS_ST ; - toolset.flags msvc.link FINDLIBS_SA ; - toolset.flags msvc.link LIBRARY_OPTION msvc : "" : unchecked ; - toolset.flags msvc.link LIBRARIES_MENTIONED_BY_FILE : ; - } - - toolset.flags msvc.archive AROPTIONS ; -} - - -# Locates the requested setup script under the given folder and returns its full -# path or nothing in case the script can not be found. In case multiple scripts -# are found only the first one is returned. -# -# TODO: There used to exist a code comment for the msvc.init rule stating that -# we do not correctly detect the location of the vcvars32.bat setup script for -# the free VC7.1 tools in case user explicitly provides a path. This should be -# tested or simply remove this whole comment in case this toolset version is no -# longer important. -# -local rule locate-default-setup ( command : parent : setup-name ) -{ - local result = [ GLOB $(command) $(parent) : $(setup-name) ] ; - if $(result[1]) - { - return $(result[1]) ; - } -} - - -# Validates given path, registers found configuration and prints debug -# information about it. -# -local rule register-configuration ( version : path ? ) -{ - if $(path) - { - local command = [ GLOB $(path) : cl.exe ] ; - - if $(command) - { - if $(.debug-configuration) - { - ECHO "notice: [msvc-cfg] msvc-$(version) detected, command: '$(command)'" ; - } - - $(.versions).register $(version) ; - $(.versions).set $(version) : options : $(command) ; - } - } -} - - -################################################################################ -# -# Startup code executed when loading this module. -# -################################################################################ - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - -# Miscellaneous constants. -.RM = [ common.rm-command ] ; -.nl = " -" ; -.ProgramFiles = [ path.make [ common.get-program-files-dir ] ] ; -.escaped-double-quote = "\"" ; -.TOUCH_FILE = [ common.file-touch-command ] ; - -# List of all registered configurations. -.versions = [ new configurations ] ; - -# Supported CPU architectures. -.cpu-arch-i386 = - / - /32 - x86/ - x86/32 ; - -.cpu-arch-amd64 = - /64 - x86/64 ; - -.cpu-arch-ia64 = - ia64/ - ia64/64 ; - - -# Supported CPU types (only Itanium optimization options are supported from -# VC++ 2005 on). See -# http://msdn2.microsoft.com/en-us/library/h66s5s0e(vs.90).aspx for more -# detailed information. -.cpu-type-g5 = i586 pentium pentium-mmx ; -.cpu-type-g6 = i686 pentiumpro pentium2 pentium3 pentium3m pentium-m k6 - k6-2 k6-3 winchip-c6 winchip2 c3 c3-2 ; -.cpu-type-em64t = prescott nocona conroe conroe-xe conroe-l allendale mermon - mermon-xe kentsfield kentsfield-xe penryn wolfdale - yorksfield nehalem ; -.cpu-type-amd64 = k8 opteron athlon64 athlon-fx ; -.cpu-type-g7 = pentium4 pentium4m athlon athlon-tbird athlon-4 athlon-xp - athlon-mp $(.cpu-type-em64t) $(.cpu-type-amd64) ; -.cpu-type-itanium = itanium itanium1 merced ; -.cpu-type-itanium2 = itanium2 mckinley ; - - -# Known toolset versions, in order of preference. -.known-versions = 10.0 10.0express 9.0 9.0express 8.0 8.0express 7.1 7.1toolkit 7.0 6.0 ; - -# Version aliases. -.version-alias-6 = 6.0 ; -.version-alias-6.5 = 6.0 ; -.version-alias-7 = 7.0 ; -.version-alias-8 = 8.0 ; -.version-alias-9 = 9.0 ; -.version-alias-10 = 10.0 ; - -# Names of registry keys containing the Visual C++ installation path (relative -# to "HKEY_LOCAL_MACHINE\SOFTWARE\\Microsoft"). -.version-6.0-reg = "VisualStudio\\6.0\\Setup\\Microsoft Visual C++" ; -.version-7.0-reg = "VisualStudio\\7.0\\Setup\\VC" ; -.version-7.1-reg = "VisualStudio\\7.1\\Setup\\VC" ; -.version-8.0-reg = "VisualStudio\\8.0\\Setup\\VC" ; -.version-8.0express-reg = "VCExpress\\8.0\\Setup\\VC" ; -.version-9.0-reg = "VisualStudio\\9.0\\Setup\\VC" ; -.version-9.0express-reg = "VCExpress\\9.0\\Setup\\VC" ; -.version-10.0-reg = "VisualStudio\\10.0\\Setup\\VC" ; -.version-10.0express-reg = "VCExpress\\10.0\\Setup\\VC" ; - -# Visual C++ Toolkit 2003 does not store its installation path in the registry. -# The environment variable 'VCToolkitInstallDir' and the default installation -# path will be checked instead. -.version-7.1toolkit-path = "Microsoft Visual C++ Toolkit 2003" "bin" ; -.version-7.1toolkit-env = VCToolkitInstallDir ; - -# Path to the folder containing "cl.exe" relative to the value of the -# corresponding environment variable. -.version-7.1toolkit-envpath = "bin" ; - - -# Auto-detect all the available msvc installations on the system. -auto-detect-toolset-versions ; - - -# And finally trigger the actual Boost Build toolset registration. -register-toolset ; diff --git a/jam-files/boost-build/tools/notfile.jam b/jam-files/boost-build/tools/notfile.jam deleted file mode 100644 index 97a5b0e8..00000000 --- a/jam-files/boost-build/tools/notfile.jam +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2005 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import "class" : new ; -import generators ; -import project ; -import targets ; -import toolset ; -import type ; - - -type.register NOTFILE_MAIN ; - - -class notfile-generator : generator -{ - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * : multiple ? ) - { - local action ; - local action-name = [ $(property-set).get ] ; - - local m = [ MATCH ^@(.*) : $(action-name) ] ; - - if $(m) - { - action = [ new action $(sources) : $(m[1]) - : $(property-set) ] ; - } - else - { - action = [ new action $(sources) : notfile.run - : $(property-set) ] ; - } - return [ virtual-target.register - [ new notfile-target $(name) : $(project) : $(action) ] ] ; - } -} - - -generators.register [ new notfile-generator notfile.main : : NOTFILE_MAIN ] ; - - -toolset.flags notfile.run ACTION : ; - - -actions run -{ - $(ACTION) -} - - -rule notfile ( target-name : action + : sources * : requirements * : default-build * ) -{ - local project = [ project.current ] ; - - requirements += $(action) ; - - targets.main-target-alternative - [ new typed-target $(target-name) : $(project) : NOTFILE_MAIN - : [ targets.main-target-sources $(sources) : $(target-name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; -} - -IMPORT $(__name__) : notfile : : notfile ; diff --git a/jam-files/boost-build/tools/notfile.py b/jam-files/boost-build/tools/notfile.py deleted file mode 100644 index afbf68fb..00000000 --- a/jam-files/boost-build/tools/notfile.py +++ /dev/null @@ -1,51 +0,0 @@ -# Status: ported. -# Base revision: 64429. -# -# Copyright (c) 2005-2010 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - - -import b2.build.type as type -import b2.build.generators as generators -import b2.build.virtual_target as virtual_target -import b2.build.toolset as toolset -import b2.build.targets as targets - -from b2.manager import get_manager -from b2.util import bjam_signature - -type.register("NOTFILE_MAIN") - -class NotfileGenerator(generators.Generator): - - def run(self, project, name, ps, sources): - pass - action_name = ps.get('action')[0] - if action_name[0] == '@': - action = virtual_target.Action(get_manager(), sources, action_name[1:], ps) - else: - action = virtual_target.Action(get_manager(), sources, "notfile.run", ps) - - return [get_manager().virtual_targets().register( - virtual_target.NotFileTarget(name, project, action))] - -generators.register(NotfileGenerator("notfile.main", False, [], ["NOTFILE_MAIN"])) - -toolset.flags("notfile.run", "ACTION", [], [""]) - -get_manager().engine().register_action("notfile.run", "$(ACTION)") - -@bjam_signature((["target_name"], ["action"], ["sources", "*"], ["requirements", "*"], - ["default_build", "*"])) -def notfile(target_name, action, sources, requirements, default_build): - - requirements.append("" + action) - - return targets.create_typed_metatarget(target_name, "NOTFILE_MAIN", sources, requirements, - default_build, []) - - -get_manager().projects().add_rule("notfile", notfile) diff --git a/jam-files/boost-build/tools/package.jam b/jam-files/boost-build/tools/package.jam deleted file mode 100644 index 198c2231..00000000 --- a/jam-files/boost-build/tools/package.jam +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2005 Vladimir Prus. -# Copyright 2006 Rene Rivera. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# Provides mechanism for installing whole packages into a specific directory -# structure. This is opposed to the 'install' rule, that installs a number of -# targets to a single directory, and does not care about directory structure at -# all. - -# Example usage: -# -# package.install boost : -# : -# : -# : -# ; -# -# This will install binaries, libraries and headers to the 'proper' location, -# given by command line options --prefix, --exec-prefix, --bindir, --libdir and -# --includedir. -# -# The rule is just a convenient wrapper, avoiding the need to define several -# 'install' targets. -# -# The only install-related feature is . It will apply to -# headers only and if present, paths of headers relatively to source root will -# be retained after installing. If it is not specified, then "." is assumed, so -# relative paths in headers are always preserved. - -import "class" : new ; -import option ; -import project ; -import feature ; -import property ; -import stage ; -import targets ; -import modules ; - -feature.feature install-default-prefix : : free incidental ; - -rule install ( name package-name ? : requirements * : binaries * : libraries * : headers * ) -{ - package-name ?= $(name) ; - if [ MATCH --prefix=(.*) : [ modules.peek : ARGV ] ] - { - # If --prefix is explicitly specified on the command line, - # then we need wipe away any settings of libdir/includir that - # is specified via options in config files. - option.set bindir : ; - option.set libdir : ; - option.set includedir : ; - } - - # If is not specified, all headers are installed to - # prefix/include, no matter what their relative path is. Sometimes that is - # what is needed. - local install-source-root = [ property.select : - $(requirements) ] ; - install-source-root = $(install-source-root:G=) ; - requirements = [ property.change $(requirements) : ] ; - - local install-header-subdir = [ property.select : - $(requirements) ] ; - install-header-subdir = /$(install-header-subdir:G=) ; - install-header-subdir ?= "" ; - requirements = [ property.change $(requirements) : ] - ; - - # First, figure out all locations. Use the default if no prefix option - # given. - local prefix = [ get-prefix $(name) : $(requirements) ] ; - - # Architecture dependent files. - local exec-locate = [ option.get exec-prefix : $(prefix) ] ; - - # Binaries. - local bin-locate = [ option.get bindir : $(prefix)/bin ] ; - - # Object code libraries. - local lib-locate = [ option.get libdir : $(prefix)/lib ] ; - - # Source header files. - local include-locate = [ option.get includedir : $(prefix)/include ] ; - - stage.install $(name)-bin : $(binaries) : $(requirements) - $(bin-locate) ; - alias $(name)-lib : $(name)-lib-shared $(name)-lib-static ; - - # Since the install location of shared libraries differs on universe - # and cygwin, use target alternatives to make different targets. - # We should have used indirection conditioanl requirements, but it's - # awkward to pass bin-locate and lib-locate from there to another rule. - alias $(name)-lib-shared : $(name)-lib-shared-universe ; - alias $(name)-lib-shared : $(name)-lib-shared-cygwin : cygwin ; - - # For shared libraries, we install both explicitly specified one and the - # shared libraries that the installed executables depend on. - stage.install $(name)-lib-shared-universe : $(binaries) $(libraries) : $(requirements) - $(lib-locate) on SHARED_LIB ; - stage.install $(name)-lib-shared-cygwin : $(binaries) $(libraries) : $(requirements) - $(bin-locate) on SHARED_LIB ; - - # For static libraries, we do not care about executable dependencies, since - # static libraries are already incorporated into them. - stage.install $(name)-lib-static : $(libraries) : $(requirements) - $(lib-locate) on STATIC_LIB ; - stage.install $(name)-headers : $(headers) : $(requirements) - $(include-locate)$(install-header-subdir) - $(install-source-root) ; - alias $(name) : $(name)-bin $(name)-lib $(name)-headers ; - - local c = [ project.current ] ; - local project-module = [ $(c).project-module ] ; - module $(project-module) - { - explicit $(1)-bin $(1)-lib $(1)-headers $(1) $(1)-lib-shared $(1)-lib-static - $(1)-lib-shared-universe $(1)-lib-shared-cygwin ; - } -} - -rule install-data ( target-name : package-name : data * : requirements * ) -{ - package-name ?= target-name ; - if [ MATCH --prefix=(.*) : [ modules.peek : ARGV ] ] - { - # If --prefix is explicitly specified on the command line, - # then we need wipe away any settings of datarootdir - option.set datarootdir : ; - } - - local prefix = [ get-prefix $(package-name) : $(requirements) ] ; - local datadir = [ option.get datarootdir : $(prefix)/share ] ; - - stage.install $(target-name) - : $(data) - : $(requirements) $(datadir)/$(package-name) - ; - - local c = [ project.current ] ; - local project-module = [ $(c).project-module ] ; - module $(project-module) - { - explicit $(1) ; - } -} - -local rule get-prefix ( package-name : requirements * ) -{ - local prefix = [ option.get prefix : [ property.select - : $(requirements) ] ] ; - prefix = $(prefix:G=) ; - requirements = [ property.change $(requirements) : - ] ; - # Or some likely defaults if neither is given. - if ! $(prefix) - { - if [ modules.peek : NT ] { prefix = C:\\$(package-name) ; } - else if [ modules.peek : UNIX ] { prefix = /usr/local ; } - } - return $(prefix) ; -} - diff --git a/jam-files/boost-build/tools/package.py b/jam-files/boost-build/tools/package.py deleted file mode 100644 index aa081b4f..00000000 --- a/jam-files/boost-build/tools/package.py +++ /dev/null @@ -1,168 +0,0 @@ -# Status: ported -# Base revision: 64488 -# -# Copyright (c) 2005, 2010 Vladimir Prus. -# Copyright 2006 Rene Rivera. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# Provides mechanism for installing whole packages into a specific directory -# structure. This is opposed to the 'install' rule, that installs a number of -# targets to a single directory, and does not care about directory structure at -# all. - -# Example usage: -# -# package.install boost : -# : -# : -# : -# ; -# -# This will install binaries, libraries and headers to the 'proper' location, -# given by command line options --prefix, --exec-prefix, --bindir, --libdir and -# --includedir. -# -# The rule is just a convenient wrapper, avoiding the need to define several -# 'install' targets. -# -# The only install-related feature is . It will apply to -# headers only and if present, paths of headers relatively to source root will -# be retained after installing. If it is not specified, then "." is assumed, so -# relative paths in headers are always preserved. - -import b2.build.feature as feature -import b2.build.property as property -import b2.util.option as option -import b2.tools.stage as stage - -from b2.build.alias import alias - -from b2.manager import get_manager - -from b2.util import bjam_signature -from b2.util.utility import ungrist - - -import os - -feature.feature("install-default-prefix", [], ["free", "incidental"]) - -@bjam_signature((["name", "package_name", "?"], ["requirements", "*"], - ["binaries", "*"], ["libraries", "*"], ["headers", "*"])) -def install(name, package_name=None, requirements=[], binaries=[], libraries=[], headers=[]): - - requirements = requirements[:] - binaries = binaries[:] - libraries - - if not package_name: - package_name = name - - if option.get("prefix"): - # If --prefix is explicitly specified on the command line, - # then we need wipe away any settings of libdir/includir that - # is specified via options in config files. - option.set("bindir", None) - option.set("libdir", None) - option.set("includedir", None) - - # If is not specified, all headers are installed to - # prefix/include, no matter what their relative path is. Sometimes that is - # what is needed. - install_source_root = property.select('install-source-root', requirements) - if install_source_root: - requirements = property.change(requirements, 'install-source-root', None) - - install_header_subdir = property.select('install-header-subdir', requirements) - if install_header_subdir: - install_header_subdir = ungrist(install_header_subdir[0]) - requirements = property.change(requirements, 'install-header-subdir', None) - - # First, figure out all locations. Use the default if no prefix option - # given. - prefix = get_prefix(name, requirements) - - # Architecture dependent files. - exec_locate = option.get("exec-prefix", prefix) - - # Binaries. - bin_locate = option.get("bindir", os.path.join(prefix, "bin")) - - # Object code libraries. - lib_locate = option.get("libdir", os.path.join(prefix, "lib")) - - # Source header files. - include_locate = option.get("includedir", os.path.join(prefix, "include")) - - stage.install(name + "-bin", binaries, requirements + ["" + bin_locate]) - - alias(name + "-lib", [name + "-lib-shared", name + "-lib-static"]) - - # Since the install location of shared libraries differs on universe - # and cygwin, use target alternatives to make different targets. - # We should have used indirection conditioanl requirements, but it's - # awkward to pass bin-locate and lib-locate from there to another rule. - alias(name + "-lib-shared", [name + "-lib-shared-universe"]) - alias(name + "-lib-shared", [name + "-lib-shared-cygwin"], ["cygwin"]) - - # For shared libraries, we install both explicitly specified one and the - # shared libraries that the installed executables depend on. - stage.install(name + "-lib-shared-universe", binaries + libraries, - requirements + ["" + lib_locate, "on", - "SHARED_LIB"]) - stage.install(name + "-lib-shared-cygwin", binaries + libraries, - requirements + ["" + bin_locate, "on", - "SHARED_LIB"]) - - # For static libraries, we do not care about executable dependencies, since - # static libraries are already incorporated into them. - stage.install(name + "-lib-static", libraries, requirements + - ["" + lib_locate, "on", "STATIC_LIB"]) - stage.install(name + "-headers", headers, requirements \ - + ["" + os.path.join(include_locate, s) for s in install_header_subdir] - + install_source_root) - - alias(name, [name + "-bin", name + "-lib", name + "-headers"]) - - pt = get_manager().projects().current() - - for subname in ["bin", "lib", "headers", "lib-shared", "lib-static", "lib-shared-universe", "lib-shared-cygwin"]: - pt.mark_targets_as_explicit([name + "-" + subname]) - -@bjam_signature((["target_name"], ["package_name"], ["data", "*"], ["requirements", "*"])) -def install_data(target_name, package_name, data, requirements): - if not package_name: - package_name = target_name - - if option.get("prefix"): - # If --prefix is explicitly specified on the command line, - # then we need wipe away any settings of datarootdir - option.set("datarootdir", None) - - prefix = get_prefix(package_name, requirements) - datadir = option.get("datarootdir", os.path.join(prefix, "share")) - - stage.install(target_name, data, - requirements + ["" + os.path.join(datadir, package_name)]) - - get_manager().projects().current().mark_targets_as_explicit([target_name]) - -def get_prefix(package_name, requirements): - - specified = property.select("install-default-prefix", requirements) - if specified: - specified = ungrist(specified[0]) - prefix = option.get("prefix", specified) - requirements = property.change(requirements, "install-default-prefix", None) - # Or some likely defaults if neither is given. - if not prefix: - if os.name == "nt": - prefix = "C:\\" + package_name - elif os.name == "posix": - prefix = "/usr/local" - - return prefix - diff --git a/jam-files/boost-build/tools/pathscale.jam b/jam-files/boost-build/tools/pathscale.jam deleted file mode 100644 index 454e3454..00000000 --- a/jam-files/boost-build/tools/pathscale.jam +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2006 Noel Belcourt -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import property ; -import generators ; -import toolset : flags ; -import feature ; -import type ; -import common ; -import fortran ; - -feature.extend toolset : pathscale ; -toolset.inherit pathscale : unix ; -generators.override pathscale.prebuilt : builtin.prebuilt ; -generators.override pathscale.searched-lib-generator : searched-lib-generator ; - -# Documentation and toolchain description located -# http://www.pathscale.com/docs.html - -rule init ( version ? : command * : options * ) -{ - command = [ common.get-invocation-command pathscale : pathCC : $(command) - : /opt/ekopath/bin ] ; - - # Determine the version - local command-string = $(command:J=" ") ; - if $(command) - { - version ?= [ MATCH "^([0-9.]+)" - : [ SHELL "$(command-string) -dumpversion" ] ] ; - } - - local condition = [ common.check-init-parameters pathscale - : version $(version) ] ; - - common.handle-options pathscale : $(condition) : $(command) : $(options) ; - - toolset.flags pathscale.compile.fortran90 OPTIONS $(condition) : - [ feature.get-values : $(options) ] : unchecked ; - - command_c = $(command_c[1--2]) $(command[-1]:B=pathcc) ; - - toolset.flags pathscale CONFIG_C_COMMAND $(condition) : $(command_c) ; - - # fortran support - local f-command = [ common.get-invocation-command pathscale : pathf90 : $(command) ] ; - local command_f = $(command_f[1--2]) $(f-command[-1]:B=pathf90) ; - local command_f90 = $(command_f[1--2]) $(f-command[-1]:B=pathf90) ; - - toolset.flags pathscale CONFIG_F_COMMAND $(condition) : $(command_f) ; - toolset.flags pathscale CONFIG_F90_COMMAND $(condition) : $(command_f90) ; - - # always link lib rt to resolve clock_gettime() - flags pathscale.link FINDLIBS-SA : rt : unchecked ; -} - -# Declare generators -generators.register-c-compiler pathscale.compile.c : C : OBJ : pathscale ; -generators.register-c-compiler pathscale.compile.c++ : CPP : OBJ : pathscale ; -generators.register-fortran-compiler pathscale.compile.fortran : FORTRAN : OBJ : pathscale ; -generators.register-fortran90-compiler pathscale.compile.fortran90 : FORTRAN90 : OBJ : pathscale ; - -# Declare flags and actions for compilation -flags pathscale.compile OPTIONS off : -O0 ; -flags pathscale.compile OPTIONS speed : -O3 ; -flags pathscale.compile OPTIONS space : -Os ; - -flags pathscale.compile OPTIONS off : -noinline ; -flags pathscale.compile OPTIONS on : -inline ; -flags pathscale.compile OPTIONS full : -inline ; - -flags pathscale.compile OPTIONS off : -woffall ; -flags pathscale.compile OPTIONS on : -Wall ; -flags pathscale.compile OPTIONS all : -Wall -pedantic ; -flags pathscale.compile OPTIONS on : -Werror ; - -flags pathscale.compile OPTIONS on : -ggdb ; -flags pathscale.compile OPTIONS on : -pg ; -flags pathscale.compile OPTIONS shared : -fPIC ; -flags pathscale.compile OPTIONS 32 : -m32 ; -flags pathscale.compile OPTIONS 64 : -m64 ; - -flags pathscale.compile USER_OPTIONS ; -flags pathscale.compile.c++ USER_OPTIONS ; -flags pathscale.compile DEFINES ; -flags pathscale.compile INCLUDES ; - -flags pathscale.compile.fortran USER_OPTIONS ; -flags pathscale.compile.fortran90 USER_OPTIONS ; - -actions compile.c -{ - "$(CONFIG_C_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.fortran -{ - "$(CONFIG_F_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -rule compile.fortran90 ( targets * : sources * : properties * ) -{ - # the space rule inserts spaces between targets and it's necessary - SPACE on $(targets) = " " ; - # Serialize execution of the compile.fortran90 action - # F90 source must be compiled in a particular order so we - # serialize the build as a parallel F90 compile might fail - JAM_SEMAPHORE on $(targets) = pathscale-f90-semaphore ; -} - -actions compile.fortran90 -{ - "$(CONFIG_F90_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -module $(<[1]:D) -c -o "$(<)" "$(>)" -} - -# Declare flags and actions for linking -flags pathscale.link OPTIONS on : -ggdb -rdynamic ; -# Strip the binary when no debugging is needed -flags pathscale.link OPTIONS off : -g0 ; -flags pathscale.link OPTIONS on : -pg ; -flags pathscale.link USER_OPTIONS ; -flags pathscale.link LINKPATH ; -flags pathscale.link FINDLIBS-ST ; -flags pathscale.link FINDLIBS-SA ; -flags pathscale.link FINDLIBS-SA multi : pthread ; -flags pathscale.link LIBRARIES ; -flags pathscale.link LINK-RUNTIME static : static ; -flags pathscale.link LINK-RUNTIME shared : dynamic ; -flags pathscale.link RPATH ; -# On gcc, there are separate options for dll path at runtime and -# link time. On Solaris, there's only one: -R, so we have to use -# it, even though it's bad idea. -flags pathscale.link RPATH ; - -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) -} - -# Slight mods for dlls -rule link.dll ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,"$(RPATH)" -o "$(<)" -Wl,-soname$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) -} - -# Declare action for creating static libraries -# "$(CONFIG_COMMAND)" -ar -o "$(<)" "$(>)" -actions piecemeal archive -{ - ar $(ARFLAGS) ru "$(<)" "$(>)" -} diff --git a/jam-files/boost-build/tools/pch.jam b/jam-files/boost-build/tools/pch.jam deleted file mode 100644 index 0c6e98fa..00000000 --- a/jam-files/boost-build/tools/pch.jam +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2005 Reece H. Dunn. -# Copyright 2006 Ilya Sokolov -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -##### Using Precompiled Headers (Quick Guide) ##### -# -# Make precompiled mypch.hpp: -# -# import pch ; -# -# cpp-pch mypch -# : # sources -# mypch.hpp -# : # requiremnts -# msvc:mypch.cpp -# ; -# -# Add cpp-pch to sources: -# -# exe hello -# : main.cpp hello.cpp mypch -# ; - -import "class" : new ; -import type ; -import feature ; -import generators ; - -type.register PCH : pch ; - -type.register C_PCH : : PCH ; -type.register CPP_PCH : : PCH ; - -# Control precompiled header (PCH) generation. -feature.feature pch : - on - off - : propagated ; - - -feature.feature pch-header : : free dependency ; -feature.feature pch-file : : free dependency ; - -# Base PCH generator. The 'run' method has the logic to prevent this generator -# from being run unless it's being used for a top-level PCH target. -class pch-generator : generator -{ - import property-set ; - - rule action-class ( ) - { - return compile-action ; - } - - rule run ( project name ? : property-set : sources + ) - { - if ! $(name) - { - # Unless this generator is invoked as the top-most generator for a - # main target, fail. This allows using 'H' type as input type for - # this generator, while preventing Boost.Build to try this generator - # when not explicitly asked for. - # - # One bad example is msvc, where pch generator produces both PCH - # target and OBJ target, so if there's any header generated (like by - # bison, or by msidl), we'd try to use pch generator to get OBJ from - # that H, which is completely wrong. By restricting this generator - # only to pch main target, such problem is solved. - } - else - { - local r = [ run-pch $(project) $(name) - : [ $(property-set).add-raw BOOST_BUILD_PCH_ENABLED ] - : $(sources) ] ; - return [ generators.add-usage-requirements $(r) - : BOOST_BUILD_PCH_ENABLED ] ; - } - } - - # This rule must be overridden by the derived classes. - rule run-pch ( project name ? : property-set : sources + ) - { - } -} - - -# NOTE: requirements are empty, default pch generator can be applied when -# pch=off. -generators.register - [ new dummy-generator pch.default-c-pch-generator : : C_PCH ] ; -generators.register - [ new dummy-generator pch.default-cpp-pch-generator : : CPP_PCH ] ; diff --git a/jam-files/boost-build/tools/pch.py b/jam-files/boost-build/tools/pch.py deleted file mode 100644 index 21d3db09..00000000 --- a/jam-files/boost-build/tools/pch.py +++ /dev/null @@ -1,83 +0,0 @@ -# Status: Being ported by Steven Watanabe -# Base revision: 47077 -# -# Copyright (c) 2005 Reece H. Dunn. -# Copyright 2006 Ilya Sokolov -# Copyright (c) 2008 Steven Watanabe -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -##### Using Precompiled Headers (Quick Guide) ##### -# -# Make precompiled mypch.hpp: -# -# import pch ; -# -# cpp-pch mypch -# : # sources -# mypch.hpp -# : # requiremnts -# msvc:mypch.cpp -# ; -# -# Add cpp-pch to sources: -# -# exe hello -# : main.cpp hello.cpp mypch -# ; - -from b2.build import type, feature, generators - -type.register('PCH', ['pch']) -type.register('C_PCH', [], 'PCH') -type.register('CPP_PCH', [], 'PCH') - -# Control precompiled header (PCH) generation. -feature.feature('pch', - ['on', 'off'], - ['propagated']) - -feature.feature('pch-header', [], ['free', 'dependency']) -feature.feature('pch-file', [], ['free', 'dependency']) - -class PchGenerator(generators.Generator): - """ - Base PCH generator. The 'run' method has the logic to prevent this generator - from being run unless it's being used for a top-level PCH target. - """ - def action_class(self): - return 'compile-action' - - def run(self, project, name, prop_set, sources): - if not name: - # Unless this generator is invoked as the top-most generator for a - # main target, fail. This allows using 'H' type as input type for - # this generator, while preventing Boost.Build to try this generator - # when not explicitly asked for. - # - # One bad example is msvc, where pch generator produces both PCH - # target and OBJ target, so if there's any header generated (like by - # bison, or by msidl), we'd try to use pch generator to get OBJ from - # that H, which is completely wrong. By restricting this generator - # only to pch main target, such problem is solved. - pass - else: - r = self.run_pch(project, name, - prop_set.add_raw('BOOST_BUILD_PCH_ENABLED'), - sources) - return generators.add_usage_requirements( - r, ['BOOST_BUILD_PCH_ENABLED']) - - # This rule must be overridden by the derived classes. - def run_pch(self, project, name, prop_set, sources): - pass - -#FIXME: dummy-generator in builtins.jam needs to be ported. -# NOTE: requirements are empty, default pch generator can be applied when -# pch=off. -###generators.register( -### [ new dummy-generator pch.default-c-pch-generator : : C_PCH ] ; -###generators.register -### [ new dummy-generator pch.default-cpp-pch-generator : : CPP_PCH ] ; diff --git a/jam-files/boost-build/tools/pgi.jam b/jam-files/boost-build/tools/pgi.jam deleted file mode 100644 index 3a35c644..00000000 --- a/jam-files/boost-build/tools/pgi.jam +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright Noel Belcourt 2007. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import property ; -import generators ; -import os ; -import toolset : flags ; -import feature ; -import fortran ; -import type ; -import common ; -import gcc ; - -feature.extend toolset : pgi ; -toolset.inherit pgi : unix ; -generators.override pgi.prebuilt : builtin.lib-generator ; -generators.override pgi.searched-lib-generator : searched-lib-generator ; - -# Documentation and toolchain description located -# http://www.pgroup.com/resources/docs.htm - -rule init ( version ? : command * : options * ) -{ - local condition = [ common.check-init-parameters pgi : version $(version) ] ; - - local l_command = [ common.get-invocation-command pgi : pgCC : $(command) ] ; - - common.handle-options pgi : $(condition) : $(l_command) : $(options) ; - - command_c = $(command_c[1--2]) $(l_command[-1]:B=cc) ; - - toolset.flags pgi CONFIG_C_COMMAND $(condition) : $(command_c) ; - - flags pgi.compile DEFINES $(condition) : - [ feature.get-values : $(options) ] : unchecked ; - - # IOV_MAX support - flags pgi.compile DEFINES $(condition) : __need_IOV_MAX : unchecked ; - - # set link flags - flags pgi.link FINDLIBS-ST : [ - feature.get-values : $(options) ] : unchecked ; - - # always link lib rt to resolve clock_gettime() - flags pgi.link FINDLIBS-SA : rt [ - feature.get-values : $(options) ] : unchecked ; - - gcc.init-link-flags pgi gnu $(condition) ; -} - -# Declare generators -generators.register-c-compiler pgi.compile.c : C : OBJ : pgi ; -generators.register-c-compiler pgi.compile.c++ : CPP : OBJ : pgi ; -generators.register-fortran-compiler pgi.compile.fortran : FORTRAN : OBJ : pgi ; - -# Declare flags and actions for compilation -flags pgi.compile OPTIONS : -Kieee ; -flags pgi.compile OPTIONS shared : -fpic -fPIC ; -flags pgi.compile OPTIONS on : -gopt ; -flags pgi.compile OPTIONS on : -xprofile=tcov ; -flags pgi.compile OPTIONS speed : -fast -Mx,8,0x10000000 ; -flags pgi.compile OPTIONS space : -xO2 -xspace ; -# flags pgi.compile OPTIONS multi : -mt ; - -flags pgi.compile OPTIONS off : -Minform=severe ; -flags pgi.compile OPTIONS on : -Minform=warn ; - -flags pgi.compile.c++ OPTIONS off : -INLINE:none ; - -flags pgi.compile OPTIONS ; -flags pgi.compile.c++ OPTIONS ; -flags pgi.compile DEFINES ; -flags pgi.compile INCLUDES ; - -flags pgi.compile.fortran OPTIONS ; - -actions compile.c -{ - "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.fortran -{ - "$(CONFIG_F_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -# Declare flags and actions for linking -flags pgi.link OPTIONS on : -gopt ; -# Strip the binary when no debugging is needed -flags pgi.link OPTIONS off : -s ; -flags pgi.link OPTIONS on : -xprofile=tcov ; -flags pgi.link OPTIONS ; -flags pgi.link OPTIONS shared : -fpic -fPIC ; -flags pgi.link LINKPATH ; -flags pgi.link FINDLIBS-ST ; -flags pgi.link FINDLIBS-SA ; -flags pgi.link FINDLIBS-SA multi : pthread rt ; -flags pgi.link LIBRARIES ; -flags pgi.link LINK-RUNTIME static : static ; -flags pgi.link LINK-RUNTIME shared : dynamic ; -flags pgi.link RPATH ; - -# On gcc, there are separate options for dll path at runtime and -# link time. On Solaris, there's only one: -R, so we have to use -# it, even though it's bad idea. -flags pgi.link RPATH ; - -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -# reddish can only link statically and, somehow, the presence of -Bdynamic on the link line -# marks the executable as a dynamically linked exec even though no dynamic libraries are supplied. -# Yod on redstorm refuses to load an executable that is dynamically linked. -# removing the dynamic link options should get us where we need to be on redstorm. -# "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bstatic -l$(FINDLIBS-ST) -Bdynamic -l$(FINDLIBS-SA) -B$(LINK-RUNTIME) -} - -# Slight mods for dlls -rule link.dll ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -# "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" -h$(<[1]:D=) -G "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) - -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -shared -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" -Wl,-h -Wl,$(<[1]:D=) "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -} - -actions updated together piecemeal pgi.archive -{ - ar -rc$(ARFLAGS:E=) "$(<)" "$(>)" -} - diff --git a/jam-files/boost-build/tools/python-config.jam b/jam-files/boost-build/tools/python-config.jam deleted file mode 100644 index 40aa825b..00000000 --- a/jam-files/boost-build/tools/python-config.jam +++ /dev/null @@ -1,27 +0,0 @@ -#~ Copyright 2005 Rene Rivera. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Automatic configuration for Python tools and librries. To use, just import this module. - -import os ; -import toolset : using ; - -if [ os.name ] = NT -{ - for local R in 2.4 2.3 2.2 - { - local python-path = [ W32_GETREG - "HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\$(R)\\InstallPath" ] ; - local python-version = $(R) ; - - if $(python-path) - { - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using python ":" $(python-version) ":" $(python-path) ; - } - using python : $(python-version) : $(python-path) ; - } - } -} diff --git a/jam-files/boost-build/tools/python.jam b/jam-files/boost-build/tools/python.jam deleted file mode 100644 index 97a9f9a5..00000000 --- a/jam-files/boost-build/tools/python.jam +++ /dev/null @@ -1,1267 +0,0 @@ -# Copyright 2004 Vladimir Prus. -# Distributed under the Boost Software License, Version 1.0. (See -# accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -# Support for Python and the the Boost.Python library. -# -# This module defines -# -# - a project 'python' with a target 'python' in it, that corresponds to the -# python library -# -# - a main target rule 'python-extension' which can be used to build a python -# extension. -# -# Extensions that use Boost.Python must explicitly link to it. - -import type ; -import testing ; -import generators ; -import project ; -import errors ; -import targets ; -import "class" : new ; -import os ; -import common ; -import toolset ; -import regex ; -import numbers ; -import string ; -import property ; -import sequence ; -import path ; -import feature ; -import set ; -import builtin ; -import version ; - - -# Make this module a project. -project.initialize $(__name__) ; -project python ; - -# Save the project so that if 'init' is called several times we define new -# targets in the python project, not in whatever project we were called by. -.project = [ project.current ] ; - -# Dynamic linker lib. Necessary to specify it explicitly on some platforms. -lib dl ; -# This contains 'openpty' function need by python. Again, on some system need to -# pass this to linker explicitly. -lib util ; -# Python uses pthread symbols. -lib pthread ; -# Extra library needed by phtread on some platforms. -lib rt ; - -# The pythonpath feature specifies additional elements for the PYTHONPATH -# environment variable, set by run-pyd. For example, pythonpath can be used to -# access Python modules that are part of the product being built, but are not -# installed in the development system's default paths. -feature.feature pythonpath : : free optional path ; - -# Initializes the Python toolset. Note that all parameters are optional. -# -# - version -- the version of Python to use. Should be in Major.Minor format, -# for example 2.3. Do not include the subminor version. -# -# - cmd-or-prefix: Preferably, a command that invokes a Python interpreter. -# Alternatively, the installation prefix for Python libraries and includes. If -# empty, will be guessed from the version, the platform's installation -# patterns, and the python executables that can be found in PATH. -# -# - includes: the include path to Python headers. If empty, will be guessed. -# -# - libraries: the path to Python library binaries. If empty, will be guessed. -# On MacOS/Darwin, you can also pass the path of the Python framework. -# -# - condition: if specified, should be a set of properties that are matched -# against the build configuration when Boost.Build selects a Python -# configuration to use. -# -# - extension-suffix: A string to append to the name of extension modules before -# the true filename extension. Ordinarily we would just compute this based on -# the value of the feature. However ubuntu's python-dbg -# package uses the windows convention of appending _d to debug-build extension -# modules. We have no way of detecting ubuntu, or of probing python for the -# "_d" requirement, and if you configure and build python using -# --with-pydebug, you'll be using the standard *nix convention. Defaults to "" -# (or "_d" when targeting windows and is set). -# -# Example usage: -# -# using python : 2.3 ; -# using python : 2.3 : /usr/local/bin/python ; -# -rule init ( version ? : cmd-or-prefix ? : includes * : libraries ? - : condition * : extension-suffix ? ) -{ - project.push-current $(.project) ; - - debug-message Configuring python... ; - for local v in version cmd-or-prefix includes libraries condition - { - if $($(v)) - { - debug-message " user-specified "$(v): \"$($(v))\" ; - } - } - - configure $(version) : $(cmd-or-prefix) : $(includes) : $(libraries) : $(condition) : $(extension-suffix) ; - - project.pop-current ; -} - -# A simpler version of SHELL that grabs stderr as well as stdout, but returns -# nothing if there was an error. -# -local rule shell-cmd ( cmd ) -{ - debug-message running command '$(cmd)" 2>&1"' ; - x = [ SHELL $(cmd)" 2>&1" : exit-status ] ; - if $(x[2]) = 0 - { - return $(x[1]) ; - } - else - { - return ; - } -} - - -# Try to identify Cygwin symlinks. Invoking such a file directly as an NT -# executable from a native Windows build of bjam would be fatal to the bjam -# process. One /can/ invoke them through sh.exe or bash.exe, if you can prove -# that those are not also symlinks. ;-) -# -# If a symlink is found returns non-empty; we try to extract the target of the -# symlink from the file and return that. -# -# Note: 1. only works on NT 2. path is a native path. -local rule is-cygwin-symlink ( path ) -{ - local is-symlink = ; - - # Look for a file with the given path having the S attribute set, as cygwin - # symlinks do. /-C means "do not use thousands separators in file sizes." - local dir-listing = [ shell-cmd "DIR /-C /A:S \""$(path)"\"" ] ; - - if $(dir-listing) - { - # Escape any special regex characters in the base part of the path. - local base-pat = [ regex.escape $(path:D=) : ].[()*+?|\\$^ : \\ ] ; - - # Extract the file's size from the directory listing. - local size-of-system-file = [ MATCH "([0-9]+) "$(base-pat) : $(dir-listing) : 1 ] ; - - # If the file has a reasonably small size, look for the special symlink - # identification text. - if $(size-of-system-file) && [ numbers.less $(size-of-system-file) 1000 ] - { - local link = [ SHELL "FIND /OFF \"!\" \""$(path)"\" 2>&1" ] ; - if $(link[2]) != 0 - { - local nl = " - -" ; - is-symlink = [ MATCH ".*!([^"$(nl)"]*)" : $(link[1]) : 1 ] ; - if $(is-symlink) - { - is-symlink = [ *nix-path-to-native $(is-symlink) ] ; - is-symlink = $(is-symlink:R=$(path:D)) ; - } - - } - } - } - return $(is-symlink) ; -} - - -# Append ext to each member of names that does not contain '.'. -# -local rule default-extension ( names * : ext * ) -{ - local result ; - for local n in $(names) - { - switch $(n) - { - case *.* : result += $(n) ; - case * : result += $(n)$(ext) ; - } - } - return $(result) ; -} - - -# Tries to determine whether invoking "cmd" would actually attempt to launch a -# cygwin symlink. -# -# Note: only works on NT. -# -local rule invokes-cygwin-symlink ( cmd ) -{ - local dirs = $(cmd:D) ; - if ! $(dirs) - { - dirs = . [ os.executable-path ] ; - } - local base = [ default-extension $(cmd:D=) : .exe .cmd .bat ] ; - local paths = [ GLOB $(dirs) : $(base) ] ; - if $(paths) - { - # Make sure we have not run into a Cygwin symlink. Invoking such a file - # as an NT executable would be fatal for the bjam process. - return [ is-cygwin-symlink $(paths[1]) ] ; - } -} - - -local rule debug-message ( message * ) -{ - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO notice: [python-cfg] $(message) ; - } -} - - -# Like W32_GETREG, except prepend HKEY_CURRENT_USER\SOFTWARE and -# HKEY_LOCAL_MACHINE\SOFTWARE to the first argument, returning the first result -# found. Also accounts for the fact that on 64-bit machines, 32-bit software has -# its own area, under SOFTWARE\Wow6432node. -# -local rule software-registry-value ( path : data ? ) -{ - local result ; - for local root in HKEY_CURRENT_USER HKEY_LOCAL_MACHINE - { - for local x64elt in "" Wow6432node\\ # Account for 64-bit windows - { - if ! $(result) - { - result = [ W32_GETREG $(root)\\SOFTWARE\\$(x64elt)$(path) : $(data) ] ; - } - } - - } - return $(result) ; -} - - -.windows-drive-letter-re = ^([A-Za-z]):[\\/](.*) ; -.cygwin-drive-letter-re = ^/cygdrive/([a-z])/(.*) ; - -.working-directory = [ PWD ] ; -.working-drive-letter = [ SUBST $(.working-directory) $(.windows-drive-letter-re) $1 ] ; -.working-drive-letter ?= [ SUBST $(.working-directory) $(.cygwin-drive-letter-re) $1 ] ; - - -local rule windows-to-cygwin-path ( path ) -{ - # If path is rooted with a drive letter, rewrite it using the /cygdrive - # mountpoint. - local p = [ SUBST $(path:T) $(.windows-drive-letter-re) /cygdrive/$1/$2 ] ; - - # Else if path is rooted without a drive letter, use the working directory. - p ?= [ SUBST $(path:T) ^/(.*) /cygdrive/$(.working-drive-letter:L)/$2 ] ; - - # Else return the path unchanged. - return $(p:E=$(path:T)) ; -} - - -# :W only works in Cygwin builds of bjam. This one works on NT builds as well. -# -local rule cygwin-to-windows-path ( path ) -{ - path = $(path:R="") ; # strip any trailing slash - - local drive-letter = [ SUBST $(path) $(.cygwin-drive-letter-re) $1:/$2 ] ; - if $(drive-letter) - { - path = $(drive-letter) ; - } - else if $(path:R=/x) = $(path) # already rooted? - { - # Look for a cygwin mount that includes each head sequence in $(path). - local head = $(path) ; - local tail = "" ; - - while $(head) - { - local root = [ software-registry-value - "Cygnus Solutions\\Cygwin\\mounts v2\\"$(head) : native ] ; - - if $(root) - { - path = $(tail:R=$(root)) ; - head = ; - } - tail = $(tail:R=$(head:D=)) ; - - if $(head) = / - { - head = ; - } - else - { - head = $(head:D) ; - } - } - } - return [ regex.replace $(path:R="") / \\ ] ; -} - - -# Convert a *nix path to native. -# -local rule *nix-path-to-native ( path ) -{ - if [ os.name ] = NT - { - path = [ cygwin-to-windows-path $(path) ] ; - } - return $(path) ; -} - - -# Convert an NT path to native. -# -local rule windows-path-to-native ( path ) -{ - if [ os.name ] = NT - { - return $(path) ; - } - else - { - return [ windows-to-cygwin-path $(path) ] ; - } -} - - -# Return nonempty if path looks like a windows path, i.e. it starts with a drive -# letter or contains backslashes. -# -local rule guess-windows-path ( path ) -{ - return [ SUBST $(path) ($(.windows-drive-letter-re)|.*([\\]).*) $1 ] ; -} - - -local rule path-to-native ( paths * ) -{ - local result ; - - for local p in $(paths) - { - if [ guess-windows-path $(p) ] - { - result += [ windows-path-to-native $(p) ] ; - } - else - { - result += [ *nix-path-to-native $(p:T) ] ; - } - } - return $(result) ; -} - - -# Validate the version string and extract the major/minor part we care about. -# -local rule split-version ( version ) -{ - local major-minor = [ MATCH ^([0-9]+)\.([0-9]+)(.*)$ : $(version) : 1 2 3 ] ; - if ! $(major-minor[2]) || $(major-minor[3]) - { - ECHO "Warning: \"using python\" expects a two part (major, minor) version number; got" $(version) instead ; - - # Add a zero to account for the missing digit if necessary. - major-minor += 0 ; - } - - return $(major-minor[1]) $(major-minor[2]) ; -} - - -# Build a list of versions from 3.0 down to 1.5. Because bjam can not enumerate -# registry sub-keys, we have no way of finding a version with a 2-digit minor -# version, e.g. 2.10 -- let us hope that never happens. -# -.version-countdown = ; -for local v in [ numbers.range 15 30 ] -{ - .version-countdown = [ SUBST $(v) (.)(.*) $1.$2 ] $(.version-countdown) ; -} - - -local rule windows-installed-pythons ( version ? ) -{ - version ?= $(.version-countdown) ; - local interpreters ; - - for local v in $(version) - { - local install-path = [ - software-registry-value "Python\\PythonCore\\"$(v)"\\InstallPath" ] ; - - if $(install-path) - { - install-path = [ windows-path-to-native $(install-path) ] ; - debug-message Registry indicates Python $(v) installed at \"$(install-path)\" ; - } - - interpreters += $(:E=python:R=$(install-path)) ; - } - return $(interpreters) ; -} - - -local rule darwin-installed-pythons ( version ? ) -{ - version ?= $(.version-countdown) ; - - local prefix - = [ GLOB /System/Library/Frameworks /Library/Frameworks - : Python.framework ] ; - - return $(prefix)/Versions/$(version)/bin/python ; -} - - -# Assume "python-cmd" invokes a python interpreter and invoke it to extract all -# the information we care about from its "sys" module. Returns void if -# unsuccessful. -# -local rule probe ( python-cmd ) -{ - # Avoid invoking a Cygwin symlink on NT. - local skip-symlink ; - if [ os.name ] = NT - { - skip-symlink = [ invokes-cygwin-symlink $(python-cmd) ] ; - } - - if $(skip-symlink) - { - debug-message -------------------------------------------------------------------- ; - debug-message \"$(python-cmd)\" would attempt to invoke a Cygwin symlink, ; - debug-message causing a bjam built for Windows to hang. ; - debug-message ; - debug-message If you intend to target a Cygwin build of Python, please ; - debug-message replace the path to the link with the path to a real executable ; - debug-message (guessing: \"$(skip-symlink)\") "in" your 'using python' line ; - debug-message "in" user-config.jam or site-config.jam. Do not forget to escape ; - debug-message backslashes ; - debug-message -------------------------------------------------------------------- ; - } - else - { - # Prepare a List of Python format strings and expressions that can be - # used to print the constants we want from the sys module. - - # We do not really want sys.version since that is a complicated string, - # so get the information from sys.version_info instead. - local format = "version=%d.%d" ; - local exprs = "version_info[0]" "version_info[1]" ; - - for local s in $(sys-elements[2-]) - { - format += $(s)=%s ; - exprs += $(s) ; - } - - # Invoke Python and ask it for all those values. - if [ version.check-jam-version 3 1 17 ] || ( [ os.name ] != NT ) - { - # Prior to version 3.1.17 Boost Jam's SHELL command did not support - # quoted commands correctly on Windows. This means that on that - # platform we do not support using a Python command interpreter - # executable whose path contains a space character. - python-cmd = \"$(python-cmd)\" ; - } - local full-cmd = - $(python-cmd)" -c \"from sys import *; print('"$(format:J=\\n)"' % ("$(exprs:J=,)"))\"" ; - - local output = [ shell-cmd $(full-cmd) ] ; - if $(output) - { - # Parse the output to get all the results. - local nl = " - -" ; - for s in $(sys-elements) - { - # These variables are expected to be declared local in the - # caller, so Jam's dynamic scoping will set their values there. - sys.$(s) = [ SUBST $(output) \\<$(s)=([^$(nl)]+) $1 ] ; - } - } - return $(output) ; - } -} - - -# Make sure the "libraries" and "includes" variables (in an enclosing scope) -# have a value based on the information given. -# -local rule compute-default-paths ( target-os : version ? : prefix ? : - exec-prefix ? ) -{ - exec-prefix ?= $(prefix) ; - - if $(target-os) = windows - { - # The exec_prefix is where you're supposed to look for machine-specific - # libraries. - local default-library-path = $(exec-prefix)\\libs ; - local default-include-path = $(:E=Include:R=$(prefix)) ; - - # If the interpreter was found in a directory called "PCBuild" or - # "PCBuild8," assume we're looking at a Python built from the source - # distro, and go up one additional level to the default root. Otherwise, - # the default root is the directory where the interpreter was found. - - # We ask Python itself what the executable path is in case of - # intermediate symlinks or shell scripts. - local executable-dir = $(sys.executable:D) ; - - if [ MATCH ^(PCBuild) : $(executable-dir:D=) ] - { - debug-message "This Python appears to reside in a source distribution;" ; - debug-message "prepending \""$(executable-dir)"\" to default library search path" ; - - default-library-path = $(executable-dir) $(default-library-path) ; - - default-include-path = $(:E=PC:R=$(executable-dir:D)) $(default-include-path) ; - - debug-message "and \""$(default-include-path[1])"\" to default #include path" ; - } - - libraries ?= $(default-library-path) ; - includes ?= $(default-include-path) ; - } - else - { - includes ?= $(prefix)/include/python$(version) ; - - local lib = $(exec-prefix)/lib ; - libraries ?= $(lib)/python$(version)/config $(lib) ; - } -} - -# The version of the python interpreter to use. -feature.feature python : : propagated ; -feature.feature python.interpreter : : free ; - -toolset.flags python.capture-output PYTHON : ; - -# -# Support for Python configured --with-pydebug -# -feature.feature python-debugging : off on : propagated ; -builtin.variant debug-python : debug : on ; - - -# Return a list of candidate commands to try when looking for a Python -# interpreter. prefix is expected to be a native path. -# -local rule candidate-interpreters ( version ? : prefix ? : target-os ) -{ - local bin-path = bin ; - if $(target-os) = windows - { - # On Windows, look in the root directory itself and, to work with the - # result of a build-from-source, the PCBuild directory. - bin-path = PCBuild8 PCBuild "" ; - } - - bin-path = $(bin-path:R=$(prefix)) ; - - if $(target-os) in windows darwin - { - return # Search: - $(:E=python:R=$(bin-path)) # Relative to the prefix, if any - python # In the PATH - [ $(target-os)-installed-pythons $(version) ] # Standard install locations - ; - } - else - { - # Search relative to the prefix, or if none supplied, in PATH. - local unversioned = $(:E=python:R=$(bin-path:E=)) ; - - # If a version was specified, look for a python with that specific - # version appended before looking for one called, simply, "python" - return $(unversioned)$(version) $(unversioned) ; - } -} - - -# Compute system library dependencies for targets linking with static Python -# libraries. -# -# On many systems, Python uses libraries such as pthreads or libdl. Since static -# libraries carry no library dependency information of their own that the linker -# can extract, these extra dependencies have to be given explicitly on the link -# line of the client. The information about these dependencies is packaged into -# the "python" target below. -# -# Even where Python itself uses pthreads, it never allows extension modules to -# be entered concurrently (unless they explicitly give up the interpreter lock). -# Therefore, extension modules do not need the efficiency overhead of threadsafe -# code as produced by multi, and we handle libpthread along with -# other libraries here. Note: this optimization is based on an assumption that -# the compiler generates link-compatible code in both the single- and -# multi-threaded cases, and that system libraries do not change their ABIs -# either. -# -# Returns a list of usage-requirements that link to the necessary system -# libraries. -# -local rule system-library-dependencies ( target-os ) -{ - switch $(target-os) - { - case s[uo][nl]* : # solaris, sun, sunos - # Add a librt dependency for the gcc toolset on SunOS (the sun - # toolset adds -lrt unconditionally). While this appears to - # duplicate the logic already in gcc.jam, it does not as long as - # we are not forcing multi. - - # On solaris 10, distutils.sysconfig.get_config_var('LIBS') yields - # '-lresolv -lsocket -lnsl -lrt -ldl'. However, that does not seem - # to be the right list for extension modules. For example, on my - # installation, adding -ldl causes at least one test to fail because - # the library can not be found and removing it causes no failures. - - # Apparently, though, we need to add -lrt for gcc. - return gcc:rt ; - - case osf : return pthread gcc:rt ; - - case qnx* : return ; - case darwin : return ; - case windows : return ; - - case hpux : return rt ; - case *bsd : return pthread gcc:util ; - - case aix : return pthread dl ; - - case * : return pthread dl - gcc:util linux:util ; - } -} - - -# Declare a target to represent Python's library. -# -local rule declare-libpython-target ( version ? : requirements * ) -{ - # Compute the representation of Python version in the name of Python's - # library file. - local lib-version = $(version) ; - if windows in $(requirements) - { - local major-minor = [ split-version $(version) ] ; - lib-version = $(major-minor:J="") ; - if on in $(requirements) - { - lib-version = $(lib-version)_d ; - } - } - - if ! $(lib-version) - { - ECHO *** warning: could not determine Python version, which will ; - ECHO *** warning: probably prevent us from linking with the python ; - ECHO *** warning: library. Consider explicitly passing the version ; - ECHO *** warning: to 'using python'. ; - } - - # Declare it. - lib python.lib : : python$(lib-version) $(requirements) ; -} - - -# Implementation of init. -local rule configure ( version ? : cmd-or-prefix ? : includes * : libraries ? : - condition * : extension-suffix ? ) -{ - local prefix ; - local exec-prefix ; - local cmds-to-try ; - local interpreter-cmd ; - - local target-os = [ feature.get-values target-os : $(condition) ] ; - target-os ?= [ feature.defaults target-os ] ; - target-os = $(target-os:G=) ; - - if $(target-os) = windows && on in $(condition) - { - extension-suffix ?= _d ; - } - extension-suffix ?= "" ; - - # Normalize and dissect any version number. - local major-minor ; - if $(version) - { - major-minor = [ split-version $(version) ] ; - version = $(major-minor:J=.) ; - } - - local cmds-to-try ; - - if ! $(cmd-or-prefix) || [ GLOB $(cmd-or-prefix) : * ] - { - # If the user did not pass a command, whatever we got was a prefix. - prefix = $(cmd-or-prefix) ; - cmds-to-try = [ candidate-interpreters $(version) : $(prefix) : $(target-os) ] ; - } - else - { - # Work with the command the user gave us. - cmds-to-try = $(cmd-or-prefix) ; - - # On Windows, do not nail down the interpreter command just yet in case - # the user specified something that turns out to be a cygwin symlink, - # which could bring down bjam if we invoke it. - if $(target-os) != windows - { - interpreter-cmd = $(cmd-or-prefix) ; - } - } - - # Values to use in case we can not really find anything in the system. - local fallback-cmd = $(cmds-to-try[1]) ; - local fallback-version ; - - # Anything left to find or check? - if ! ( $(interpreter-cmd) && $(includes) && $(libraries) ) - { - # Values to be extracted from python's sys module. These will be set by - # the probe rule, above, using Jam's dynamic scoping. - local sys-elements = version platform prefix exec_prefix executable ; - local sys.$(sys-elements) ; - - # Compute the string Python's sys.platform needs to match. If not - # targeting Windows or cygwin we will assume only native builds can - # possibly run, so we will not require a match and we leave sys.platform - # blank. - local platform ; - switch $(target-os) - { - case windows : platform = win32 ; - case cygwin : platform = cygwin ; - } - - while $(cmds-to-try) - { - # Pop top command. - local cmd = $(cmds-to-try[1]) ; - cmds-to-try = $(cmds-to-try[2-]) ; - - debug-message Checking interpreter command \"$(cmd)\"... ; - if [ probe $(cmd) ] - { - fallback-version ?= $(sys.version) ; - - # Check for version/platform validity. - for local x in version platform - { - if $($(x)) && $($(x)) != $(sys.$(x)) - { - debug-message ...$(x) "mismatch (looking for" - $($(x)) but found $(sys.$(x))")" ; - cmd = ; - } - } - - if $(cmd) - { - debug-message ...requested configuration matched! ; - - exec-prefix = $(sys.exec_prefix) ; - - compute-default-paths $(target-os) : $(sys.version) : - $(sys.prefix) : $(sys.exec_prefix) ; - - version = $(sys.version) ; - interpreter-cmd ?= $(cmd) ; - cmds-to-try = ; # All done. - } - } - else - { - debug-message ...does not invoke a working interpreter ; - } - } - } - - # Anything left to compute? - if $(includes) && $(libraries) - { - .configured = true ; - } - else - { - version ?= $(fallback-version) ; - version ?= 2.5 ; - exec-prefix ?= $(prefix) ; - compute-default-paths $(target-os) : $(version) : $(prefix:E=) ; - } - - if ! $(interpreter-cmd) - { - fallback-cmd ?= python ; - debug-message No working Python interpreter found. ; - if [ os.name ] != NT || ! [ invokes-cygwin-symlink $(fallback-cmd) ] - { - interpreter-cmd = $(fallback-cmd) ; - debug-message falling back to \"$(interpreter-cmd)\" ; - } - } - - includes = [ path-to-native $(includes) ] ; - libraries = [ path-to-native $(libraries) ] ; - - debug-message "Details of this Python configuration:" ; - debug-message " interpreter command:" \"$(interpreter-cmd:E=)\" ; - debug-message " include path:" \"$(includes:E=)\" ; - debug-message " library path:" \"$(libraries:E=)\" ; - if $(target-os) = windows - { - debug-message " DLL search path:" \"$(exec-prefix:E=)\" ; - } - - # - # End autoconfiguration sequence. - # - local target-requirements = $(condition) ; - - # Add the version, if any, to the target requirements. - if $(version) - { - if ! $(version) in [ feature.values python ] - { - feature.extend python : $(version) ; - } - target-requirements += $(version:E=default) ; - } - - target-requirements += $(target-os) ; - - # See if we can find a framework directory on darwin. - local framework-directory ; - if $(target-os) = darwin - { - # Search upward for the framework directory. - local framework-directory = $(libraries[-1]) ; - while $(framework-directory:D=) && $(framework-directory:D=) != Python.framework - { - framework-directory = $(framework-directory:D) ; - } - - if $(framework-directory:D=) = Python.framework - { - debug-message framework directory is \"$(framework-directory)\" ; - } - else - { - debug-message "no framework directory found; using library path" ; - framework-directory = ; - } - } - - local dll-path = $(libraries) ; - - # Make sure that we can find the Python DLL on Windows. - if ( $(target-os) = windows ) && $(exec-prefix) - { - dll-path += $(exec-prefix) ; - } - - # - # Prepare usage requirements. - # - local usage-requirements = [ system-library-dependencies $(target-os) ] ; - usage-requirements += $(includes) $(interpreter-cmd) ; - if on in $(condition) - { - if $(target-os) = windows - { - # In pyconfig.h, Py_DEBUG is set if _DEBUG is set. If we define - # Py_DEBUG we will get multiple definition warnings. - usage-requirements += _DEBUG ; - } - else - { - usage-requirements += Py_DEBUG ; - } - } - - # Global, but conditional, requirements to give access to the interpreter - # for general utilities, like other toolsets, that run Python scripts. - toolset.add-requirements - $(target-requirements:J=,):$(interpreter-cmd) ; - - # Register the right suffix for extensions. - register-extension-suffix $(extension-suffix) : $(target-requirements) ; - - # - # Declare the "python" target. This should really be called - # python_for_embedding. - # - - if $(framework-directory) - { - alias python - : - : $(target-requirements) - : - : $(usage-requirements) $(framework-directory) - ; - } - else - { - declare-libpython-target $(version) : $(target-requirements) ; - - # This is an evil hack. On, Windows, when Python is embedded, nothing - # seems to set up sys.path to include Python's standard library - # (http://article.gmane.org/gmane.comp.python.general/544986). The evil - # here, aside from the workaround necessitated by Python's bug, is that: - # - # a. we're guessing the location of the python standard library from the - # location of pythonXX.lib - # - # b. we're hijacking the property to get the - # environment variable set up, and the user may want to use it for - # something else (e.g. launch the debugger). - local set-PYTHONPATH ; - if $(target-os) = windows - { - set-PYTHONPATH = [ common.prepend-path-variable-command PYTHONPATH : - $(libraries:D)/Lib ] ; - } - - alias python - : - : $(target-requirements) - : - # Why python.lib must be listed here instead of along with the - # system libs is a mystery, but if we do not do it, on cygwin, - # -lpythonX.Y never appears in the command line (although it does on - # linux). - : $(usage-requirements) - $(set-PYTHONPATH) - $(libraries) python.lib - ; - } - - # On *nix, we do not want to link either Boost.Python or Python extensions - # to libpython, because the Python interpreter itself provides all those - # symbols. If we linked to libpython, we would get duplicate symbols. So - # declare two targets -- one for building extensions and another for - # embedding. - # - # Unlike most *nix systems, Mac OS X's linker does not permit undefined - # symbols when linking a shared library. So, we still need to link against - # the Python framework, even when building extensions. Note that framework - # builds of Python always use shared libraries, so we do not need to worry - # about duplicate Python symbols. - if $(target-os) in windows cygwin darwin - { - alias python_for_extensions : python : $(target-requirements) ; - } - # On AIX we need Python extensions and Boost.Python to import symbols from - # the Python interpreter. Dynamic libraries opened with dlopen() do not - # inherit the symbols from the Python interpreter. - else if $(target-os) = aix - { - alias python_for_extensions - : - : $(target-requirements) - : - : $(usage-requirements) -Wl,-bI:$(libraries[1])/python.exp - ; - } - else - { - alias python_for_extensions - : - : $(target-requirements) - : - : $(usage-requirements) - ; - } -} - - -rule configured ( ) -{ - return $(.configured) ; -} - - -type.register PYTHON_EXTENSION : : SHARED_LIB ; - - -local rule register-extension-suffix ( root : condition * ) -{ - local suffix ; - - switch [ feature.get-values target-os : $(condition) ] - { - case windows : suffix = pyd ; - case cygwin : suffix = dll ; - case hpux : - { - if [ feature.get-values python : $(condition) ] in 1.5 1.6 2.0 2.1 2.2 2.3 2.4 - { - suffix = sl ; - } - else - { - suffix = so ; - } - } - case * : suffix = so ; - } - - type.set-generated-target-suffix PYTHON_EXTENSION : $(condition) : <$(root).$(suffix)> ; -} - - -# Unset 'lib' prefix for PYTHON_EXTENSION -type.set-generated-target-prefix PYTHON_EXTENSION : : "" ; - - -rule python-extension ( name : sources * : requirements * : default-build * : - usage-requirements * ) -{ - if [ configured ] - { - requirements += /python//python_for_extensions ; - } - requirements += true ; - - local project = [ project.current ] ; - - targets.main-target-alternative - [ new typed-target $(name) : $(project) : PYTHON_EXTENSION - : [ targets.main-target-sources $(sources) : $(name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; -} - -IMPORT python : python-extension : : python-extension ; - -rule py2to3 -{ - common.copy $(>) $(<) ; - 2to3 $(<) ; -} - -actions 2to3 -{ - 2to3 -wn "$(<)" - 2to3 -dwn "$(<)" -} - - -# Support for testing. -type.register PY : py ; -type.register RUN_PYD_OUTPUT ; -type.register RUN_PYD : : TEST ; - - -class python-test-generator : generator -{ - import set ; - - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - self.composing = true ; - } - - rule run ( project name ? : property-set : sources * : multiple ? ) - { - local pyversion = [ $(property-set).get ] ; - local python ; - local other-pythons ; - - # Make new target that converting Python source by 2to3 when running with Python 3. - local rule make-2to3-source ( source ) - { - if $(pyversion) >= 3.0 - { - local a = [ new action $(source) : python.py2to3 : $(property-set) ] ; - local t = [ utility.basename [ $(s).name ] ] ; - local p = [ new file-target $(t) : PY : $(project) : $(a) ] ; - return $(p) ; - } - else - { - return $(source) ; - } - } - - for local s in $(sources) - { - if [ $(s).type ] = PY - { - if ! $(python) - { - # First Python source ends up on command line. - python = [ make-2to3-source $(s) ] ; - - } - else - { - # Other Python sources become dependencies. - other-pythons += [ make-2to3-source $(s) ] ; - } - } - } - - local extensions ; - for local s in $(sources) - { - if [ $(s).type ] = PYTHON_EXTENSION - { - extensions += $(s) ; - } - } - - local libs ; - for local s in $(sources) - { - if [ type.is-derived [ $(s).type ] LIB ] - && ! $(s) in $(extensions) - { - libs += $(s) ; - } - } - - local new-sources ; - for local s in $(sources) - { - if [ type.is-derived [ $(s).type ] CPP ] - { - local name = [ utility.basename [ $(s).name ] ] ; - if $(name) = [ utility.basename [ $(python).name ] ] - { - name = $(name)_ext ; - } - local extension = [ generators.construct $(project) $(name) : - PYTHON_EXTENSION : $(property-set) : $(s) $(libs) ] ; - - # The important part of usage requirements returned from - # PYTHON_EXTENSION generator are xdll-path properties that will - # allow us to find the python extension at runtime. - property-set = [ $(property-set).add $(extension[1]) ] ; - - # Ignore usage requirements. We're a top-level generator and - # nobody is going to use what we generate. - new-sources += $(extension[2-]) ; - } - } - - property-set = [ $(property-set).add-raw $(other-pythons) ] ; - - result = [ construct-result $(python) $(extensions) $(new-sources) : - $(project) $(name) : $(property-set) ] ; - } -} - - -generators.register - [ new python-test-generator python.capture-output : : RUN_PYD_OUTPUT ] ; - -generators.register-standard testing.expect-success - : RUN_PYD_OUTPUT : RUN_PYD ; - - -# There are two different ways of spelling OS names. One is used for [ os.name ] -# and the other is used for the and properties. Until that -# is remedied, this sets up a crude mapping from the latter to the former, that -# will work *for the purposes of cygwin/NT cross-builds only*. Could not think -# of a better name than "translate". -# -.translate-os-windows = NT ; -.translate-os-cygwin = CYGWIN ; -local rule translate-os ( src-os ) -{ - local x = $(.translate-os-$(src-os)) [ os.name ] ; - return $(x[1]) ; -} - - -# Extract the path to a single ".pyd" source. This is used to build the -# PYTHONPATH for running bpl tests. -# -local rule pyd-pythonpath ( source ) -{ - return [ on $(source) return $(LOCATE) $(SEARCH) ] ; -} - - -# The flag settings on testing.capture-output do not apply to python.capture -# output at the moment. Redo this explicitly. -toolset.flags python.capture-output ARGS ; - - -rule capture-output ( target : sources * : properties * ) -{ - # Setup up a proper DLL search path. Here, $(sources[1]) is a python module - # and $(sources[2]) is a DLL. Only $(sources[1]) is passed to - # testing.capture-output, so RUN_PATH variable on $(sources[2]) is not - # consulted. Move it over explicitly. - RUN_PATH on $(sources[1]) = [ on $(sources[2-]) return $(RUN_PATH) ] ; - - PYTHONPATH = [ sequence.transform pyd-pythonpath : $(sources[2-]) ] ; - PYTHONPATH += [ feature.get-values pythonpath : $(properties) ] ; - - # After test is run, we remove the Python module, but not the Python script. - testing.capture-output $(target) : $(sources[1]) : $(properties) : - $(sources[2-]) ; - - # PYTHONPATH is different; it will be interpreted by whichever Python is - # invoked and so must follow path rules for the target os. The only OSes - # where we can run python for other OSes currently are NT and CYGWIN so we - # only need to handle those cases. - local target-os = [ feature.get-values target-os : $(properties) ] ; - # Oddly, host-os is not in properties, so grab the default value. - local host-os = [ feature.defaults host-os ] ; - host-os = $(host-os:G=) ; - if $(target-os) != $(host-os) - { - PYTHONPATH = [ sequence.transform $(host-os)-to-$(target-os)-path : - $(PYTHONPATH) ] ; - } - local path-separator = [ os.path-separator [ translate-os $(target-os) ] ] ; - local set-PYTHONPATH = [ common.variable-setting-command PYTHONPATH : - $(PYTHONPATH:J=$(path-separator)) ] ; - LAUNCHER on $(target) = $(set-PYTHONPATH) [ on $(target) return \"$(PYTHON)\" ] ; -} - - -rule bpl-test ( name : sources * : requirements * ) -{ - local s ; - sources ?= $(name).py $(name).cpp ; - return [ testing.make-test run-pyd : $(sources) /boost/python//boost_python - : $(requirements) : $(name) ] ; -} - - -IMPORT $(__name__) : bpl-test : : bpl-test ; diff --git a/jam-files/boost-build/tools/qcc.jam b/jam-files/boost-build/tools/qcc.jam deleted file mode 100644 index 4f2a4fc1..00000000 --- a/jam-files/boost-build/tools/qcc.jam +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (c) 2001 David Abrahams. -# Copyright (c) 2002-2003 Rene Rivera. -# Copyright (c) 2002-2003 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import "class" : new ; -import common ; -import errors ; -import feature ; -import generators ; -import os ; -import property ; -import set ; -import toolset ; -import type ; -import unix ; - -feature.extend toolset : qcc ; - -toolset.inherit-generators qcc : unix : unix.link unix.link.dll ; -generators.override builtin.lib-generator : qcc.prebuilt ; -toolset.inherit-flags qcc : unix ; -toolset.inherit-rules qcc : unix ; - -# Initializes the qcc toolset for the given version. If necessary, command may -# be used to specify where the compiler is located. The parameter 'options' is a -# space-delimited list of options, each one being specified as -# option-value. Valid option names are: cxxflags, linkflags and -# linker-type. Accepted values for linker-type are gnu and sun, gnu being the -# default. -# -# Example: -# using qcc : 3.4 : : foo bar sun ; -# -rule init ( version ? : command * : options * ) -{ - local condition = [ common.check-init-parameters qcc : version $(version) ] ; - local command = [ common.get-invocation-command qcc : QCC : $(command) ] ; - common.handle-options qcc : $(condition) : $(command) : $(options) ; -} - - -generators.register-c-compiler qcc.compile.c++ : CPP : OBJ : qcc ; -generators.register-c-compiler qcc.compile.c : C : OBJ : qcc ; -generators.register-c-compiler qcc.compile.asm : ASM : OBJ : qcc ; - - -# Declare flags for compilation. -toolset.flags qcc.compile OPTIONS on : -gstabs+ ; - -# Declare flags and action for compilation. -toolset.flags qcc.compile OPTIONS off : -O0 ; -toolset.flags qcc.compile OPTIONS speed : -O3 ; -toolset.flags qcc.compile OPTIONS space : -Os ; - -toolset.flags qcc.compile OPTIONS off : -Wc,-fno-inline ; -toolset.flags qcc.compile OPTIONS on : -Wc,-Wno-inline ; -toolset.flags qcc.compile OPTIONS full : -Wc,-finline-functions -Wc,-Wno-inline ; - -toolset.flags qcc.compile OPTIONS off : -w ; -toolset.flags qcc.compile OPTIONS all : -Wc,-Wall ; -toolset.flags qcc.compile OPTIONS on : -Wc,-Werror ; - -toolset.flags qcc.compile OPTIONS on : -p ; - -toolset.flags qcc.compile OPTIONS ; -toolset.flags qcc.compile.c++ OPTIONS ; -toolset.flags qcc.compile DEFINES ; -toolset.flags qcc.compile INCLUDES ; - -toolset.flags qcc.compile OPTIONS shared : -shared ; - -toolset.flags qcc.compile.c++ TEMPLATE_DEPTH ; - - -rule compile.c++ -{ - # Here we want to raise the template-depth parameter value to something - # higher than the default value of 17. Note that we could do this using the - # feature.set-default rule but we do not want to set the default value for - # all toolsets as well. - # - # TODO: This 'modified default' has been inherited from some 'older Boost - # Build implementation' and has most likely been added to make some Boost - # library parts compile correctly. We should see what exactly prompted this - # and whether we can get around the problem more locally. - local template-depth = [ on $(1) return $(TEMPLATE_DEPTH) ] ; - if ! $(template-depth) - { - TEMPLATE_DEPTH on $(1) = 128 ; - } -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" -Wc,-ftemplate-depth-$(TEMPLATE_DEPTH) $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.asm -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - - -# The class checking that we do not try to use the static property -# while creating or using a shared library, since it is not supported by qcc/ -# /libc. -# -class qcc-linking-generator : unix-linking-generator -{ - rule generated-targets ( sources + : property-set : project name ? ) - { - if static in [ $(property-set).raw ] - { - local m ; - if [ id ] = "qcc.link.dll" - { - m = "on qcc, DLL can't be build with static" ; - } - if ! $(m) - { - for local s in $(sources) - { - local type = [ $(s).type ] ; - if $(type) && [ type.is-derived $(type) SHARED_LIB ] - { - m = "on qcc, using DLLS together with the static options is not possible " ; - } - } - } - if $(m) - { - errors.user-error $(m) : "It is suggested to use" - "static together with static." ; - } - } - - return [ unix-linking-generator.generated-targets - $(sources) : $(property-set) : $(project) $(name) ] ; - } -} - -generators.register [ new qcc-linking-generator qcc.link : LIB OBJ : EXE - : qcc ] ; - -generators.register [ new qcc-linking-generator qcc.link.dll : LIB OBJ - : SHARED_LIB : qcc ] ; - -generators.override qcc.prebuilt : builtin.prebuilt ; -generators.override qcc.searched-lib-generator : searched-lib-generator ; - - -# Declare flags for linking. -# First, the common flags. -toolset.flags qcc.link OPTIONS on : -gstabs+ ; -toolset.flags qcc.link OPTIONS on : -p ; -toolset.flags qcc.link OPTIONS ; -toolset.flags qcc.link LINKPATH ; -toolset.flags qcc.link FINDLIBS-ST ; -toolset.flags qcc.link FINDLIBS-SA ; -toolset.flags qcc.link LIBRARIES ; - -toolset.flags qcc.link FINDLIBS-SA : m ; - -# For static we made sure there are no dynamic libraries in the -# link. -toolset.flags qcc.link OPTIONS static : -static ; - -# Assuming this is just like with gcc. -toolset.flags qcc.link RPATH : : unchecked ; -toolset.flags qcc.link RPATH_LINK : : unchecked ; - - -# Declare actions for linking. -# -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; - # Serialize execution of the 'link' action, since running N links in - # parallel is just slower. For now, serialize only qcc links while it might - # be a good idea to serialize all links. - JAM_SEMAPHORE on $(targets) = qcc-link-semaphore ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) $(OPTIONS) -} - - -# Always remove archive and start again. Here is the rationale from Andre Hentz: -# I had a file, say a1.c, that was included into liba.a. I moved a1.c to a2.c, -# updated my Jamfiles and rebuilt. My program was crashing with absurd errors. -# After some debugging I traced it back to the fact that a1.o was *still* in -# liba.a -RM = [ common.rm-command ] ; -if [ os.name ] = NT -{ - RM = "if exist \"$(<[1])\" DEL \"$(<[1])\"" ; -} - - -# Declare action for creating static libraries. The 'r' letter means to add -# files to the archive with replacement. Since we remove the archive, we do not -# care about replacement, but there is no option to "add without replacement". -# The 'c' letter suppresses warnings in case the archive does not exists yet. -# That warning is produced only on some platforms, for whatever reasons. -# -actions piecemeal archive -{ - $(RM) "$(<)" - ar rc "$(<)" "$(>)" -} - - -rule link.dll ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; - JAM_SEMAPHORE on $(targets) = qcc-link-semaphore ; -} - - -# Differ from 'link' above only by -shared. -# -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -o "$(<)" $(HAVE_SONAME)-Wl,-h$(SPACE)-Wl,$(<[1]:D=) -shared "$(>)" "$(LIBRARIES)" -l$(FINDLIBS-ST) -l$(FINDLIBS-SA) $(OPTIONS) -} diff --git a/jam-files/boost-build/tools/qt.jam b/jam-files/boost-build/tools/qt.jam deleted file mode 100644 index 8aa7ca26..00000000 --- a/jam-files/boost-build/tools/qt.jam +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2006 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# Forwarning toolset file to Qt GUI library. Forwards to the toolset file -# for the current version of Qt. - -import qt4 ; - -rule init ( prefix : full_bin ? : full_inc ? : full_lib ? : version ? : condition * ) -{ - qt4.init $(prefix) : $(full_bin) : $(full_inc) : $(full_lib) : $(version) : $(condition) ; -} - - diff --git a/jam-files/boost-build/tools/qt3.jam b/jam-files/boost-build/tools/qt3.jam deleted file mode 100644 index f82cf0ac..00000000 --- a/jam-files/boost-build/tools/qt3.jam +++ /dev/null @@ -1,209 +0,0 @@ -# Copyright 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Support for the Qt GUI library version 3 -# (http://www.trolltech.com/products/qt3/index.html). -# For new developments, it is recommended to use Qt4 via the qt4 Boost.Build -# module. - -import modules ; -import feature ; -import errors ; -import type ; -import "class" : new ; -import generators ; -import project ; -import toolset : flags ; - -# Convert this module into a project, so that we can declare targets here. -project.initialize $(__name__) ; -project qt3 ; - - -# Initialized the QT support module. The 'prefix' parameter tells where QT is -# installed. When not given, environmental variable QTDIR should be set. -# -rule init ( prefix ? ) -{ - if ! $(prefix) - { - prefix = [ modules.peek : QTDIR ] ; - if ! $(prefix) - { - errors.error - "QT installation prefix not given and QTDIR variable is empty" ; - } - } - - if $(.initialized) - { - if $(prefix) != $(.prefix) - { - errors.error - "Attempt the reinitialize QT with different installation prefix" ; - } - } - else - { - .initialized = true ; - .prefix = $(prefix) ; - - generators.register-standard qt3.moc : H : CPP(moc_%) : qt3 ; - # Note: the OBJ target type here is fake, take a look at - # qt4.jam/uic-h-generator for explanations that apply in this case as - # well. - generators.register [ new moc-h-generator-qt3 - qt3.moc.cpp : MOCCABLE_CPP : OBJ : qt3 ] ; - - # The UI type is defined in types/qt.jam, and UIC_H is only used in - # qt.jam, but not in qt4.jam, so define it here. - type.register UIC_H : : H ; - - generators.register-standard qt3.uic-h : UI : UIC_H : qt3 ; - - # The following generator is used to convert UI files to CPP. It creates - # UIC_H from UI, and constructs CPP from UI/UIC_H. In addition, it also - # returns UIC_H target, so that it can be mocced. - class qt::uic-cpp-generator : generator - { - rule __init__ ( ) - { - generator.__init__ qt3.uic-cpp : UI UIC_H : CPP : qt3 ; - } - - rule run ( project name ? : properties * : sources + ) - { - # Consider this: - # obj test : test_a.cpp : off ; - # - # This generator will somehow be called in this case, and, - # will fail -- which is okay. However, if there are - # properties they will be converted to sources, so the size of - # 'sources' will be more than 1. In this case, the base generator - # will just crash -- and that's not good. Just use a quick test - # here. - - local result ; - if ! $(sources[2]) - { - # Construct CPP as usual - result = [ generator.run $(project) $(name) - : $(properties) : $(sources) ] ; - - # If OK, process UIC_H with moc. It's pretty clear that - # the object generated with UIC will have Q_OBJECT macro. - if $(result) - { - local action = [ $(result[1]).action ] ; - local sources = [ $(action).sources ] ; - local mocced = [ generators.construct $(project) $(name) - : CPP : $(properties) : $(sources[2]) ] ; - result += $(mocced[2-]) ; - } - } - - return $(result) ; - } - } - - generators.register [ new qt::uic-cpp-generator ] ; - - # Finally, declare prebuilt target for QT library. - local usage-requirements = - $(.prefix)/include - $(.prefix)/lib - $(.prefix)/lib - qt3 - ; - lib qt : : qt-mt multi : : $(usage-requirements) ; - lib qt : : qt single : : $(usage-requirements) ; - } -} - -class moc-h-generator-qt3 : generator -{ - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * ) - { - if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_CPP - { - name = [ $(sources[1]).name ] ; - name = $(name:B) ; - - local a = [ new action $(sources[1]) : qt3.moc.cpp : - $(property-set) ] ; - - local target = [ - new file-target $(name) : MOC : $(project) : $(a) ] ; - - local r = [ virtual-target.register $(target) ] ; - - # Since this generator will return a H target, the linking generator - # won't use it at all, and won't set any dependency on it. However, - # we need the target to be seen by bjam, so that the dependency from - # sources to this generated header is detected -- if Jam does not - # know about this target, it won't do anything. - DEPENDS all : [ $(r).actualize ] ; - - return $(r) ; - } - } -} - - -# Query the installation directory. This is needed in at least two scenarios. -# First, when re-using sources from the Qt-Tree. Second, to "install" custom Qt -# plugins to the Qt-Tree. -# -rule directory -{ - return $(.prefix) ; -} - -# -f forces moc to include the processed source file. Without it, it would think -# that .qpp is not a header and would not include it from the generated file. -# -actions moc -{ - $(.prefix)/bin/moc -f $(>) -o $(<) -} - -# When moccing .cpp files, we don't need -f, otherwise generated code will -# include .cpp and we'll get duplicated symbols. -# -actions moc.cpp -{ - $(.prefix)/bin/moc $(>) -o $(<) -} - - -space = " " ; - -# Sometimes it's required to make 'plugins' available during uic invocation. To -# help with this we add paths to all dependency libraries to uic commane line. -# The intention is that it's possible to write -# -# exe a : ... a.ui ... : some_plugin ; -# -# and have everything work. We'd add quite a bunch of unrelated paths but it -# won't hurt. -# -flags qt3.uic-h LIBRARY_PATH ; -actions uic-h -{ - $(.prefix)/bin/uic $(>) -o $(<) -L$(space)$(LIBRARY_PATH) -} - - -flags qt3.uic-cpp LIBRARY_PATH ; -# The second target is uic-generated header name. It's placed in build dir, but -# we want to include it using only basename. -actions uic-cpp -{ - $(.prefix)/bin/uic $(>[1]) -i $(>[2]:D=) -o $(<) -L$(space)$(LIBRARY_PATH) -} diff --git a/jam-files/boost-build/tools/qt4.jam b/jam-files/boost-build/tools/qt4.jam deleted file mode 100644 index 71d1b762..00000000 --- a/jam-files/boost-build/tools/qt4.jam +++ /dev/null @@ -1,724 +0,0 @@ -# Copyright 2002-2006 Vladimir Prus -# Copyright 2005 Alo Sarv -# Copyright 2005-2009 Juergen Hunold -# -# Distributed under the Boost Software License, Version 1.0. (See -# accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -# Qt4 library support module -# -# The module attempts to auto-detect QT installation location from QTDIR -# environment variable; failing that, installation location can be passed as -# argument: -# -# toolset.using qt4 : /usr/local/Trolltech/Qt-4.0.0 ; -# -# The module supports code generation from .ui and .qrc files, as well as -# running the moc preprocessor on headers. Note that you must list all your -# moc-able headers in sources. -# -# Example: -# -# exe myapp : myapp.cpp myapp.h myapp.ui myapp.qrc -# /qt4//QtGui /qt4//QtNetwork ; -# -# It's also possible to run moc on cpp sources: -# -# import cast ; -# -# exe myapp : myapp.cpp [ cast _ moccable-cpp : myapp.cpp ] /qt4//QtGui ; -# -# When moccing source file myapp.cpp you need to include "myapp.moc" from -# myapp.cpp. When moccing .h files, the output of moc will be automatically -# compiled and linked in, you don't need any includes. -# -# This is consistent with Qt guidelines: -# http://doc.trolltech.com/4.0/moc.html - -import modules ; -import feature ; -import errors ; -import type ; -import "class" : new ; -import generators ; -import project ; -import toolset : flags ; -import os ; -import virtual-target ; -import scanner ; - -# Qt3Support control feature -# -# Qt4 configure defaults to build Qt4 libraries with Qt3Support. -# The autodetection is missing, so we default to disable Qt3Support. -# This prevents the user from inadvertedly using a deprecated API. -# -# The Qt3Support library can be activated by adding -# "on" to requirements -# -# Use "on:QT3_SUPPORT_WARNINGS" -# to get warnings about deprecated Qt3 support funtions and classes. -# Files ported by the "qt3to4" conversion tool contain _tons_ of -# warnings, so this define is not set as default. -# -# Todo: Detect Qt3Support from Qt's configure data. -# Or add more auto-configuration (like python). -feature.feature qt3support : off on : propagated link-incompatible ; - -# The Qt version used for requirements -# Valid are 4.4 or 4.5.0 -# Auto-detection via qmake sets 'major.minor.patch' -feature.feature qt : : propagated ; - -project.initialize $(__name__) ; -project qt ; - -# Save the project so that we tolerate 'import + using' combo. -.project = [ project.current ] ; - -# Helper utils for easy debug output -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = TRUE ; -} - -local rule debug-message ( message * ) -{ - if $(.debug-configuration) = TRUE - { - ECHO notice: [qt4-cfg] $(message) ; - } -} - -# Capture qmake output line by line -local rule read-output ( content ) -{ - local lines ; - local nl = " -" ; - local << = "([^$(nl)]*)[$(nl)](.*)" ; - local line+ = [ MATCH "$(<<)" : "$(content)" ] ; - while $(line+) - { - lines += $(line+[1]) ; - line+ = [ MATCH "$(<<)" : "$(line+[2])" ] ; - } - return $(lines) ; -} - -# Capture Qt version from qmake -local rule check-version ( bin_prefix ) -{ - full-cmd = $(bin_prefix)"/qmake -v" ; - debug-message Running '$(full-cmd)' ; - local output = [ SHELL $(full-cmd) ] ; - for line in [ read-output $(output) ] - { - # Parse the output to get all the results. - if [ MATCH "QMake" : $(line) ] - { - # Skip first line of output - } - else - { - temp = [ MATCH "([0-9]*)\\.([0-9]*)\\.([0-9]*)" : $(line) ] ; - } - } - return $(temp) ; -} - -# Validate the version string and extract the major/minor part we care about. -# -local rule split-version ( version ) -{ - local major-minor = [ MATCH ^([0-9]+)\.([0-9]+)(.*)$ : $(version) : 1 2 3 ] ; - if ! $(major-minor[2]) || $(major-minor[3]) - { - ECHO "Warning: 'using qt' expects a two part (major, minor) version number; got" $(version) instead ; - - # Add a zero to account for the missing digit if necessary. - major-minor += 0 ; - } - - return $(major-minor[1]) $(major-minor[2]) ; -} - -# Initialize the QT support module. -# Parameters: -# - 'prefix' parameter tells where Qt is installed. -# - 'full_bin' optional full path to Qt binaries (qmake,moc,uic,rcc) -# - 'full_inc' optional full path to Qt top-level include directory -# - 'full_lib' optional full path to Qt library directory -# - 'version' optional version of Qt, else autodetected via 'qmake -v' -# - 'condition' optional requirements -rule init ( prefix : full_bin ? : full_inc ? : full_lib ? : version ? : condition * ) -{ - project.push-current $(.project) ; - - debug-message "==== Configuring Qt ... ====" ; - for local v in version cmd-or-prefix includes libraries condition - { - if $($(v)) - { - debug-message " user-specified "$(v): '$($(v))' ; - } - } - - # Needed as default value - .prefix = $(prefix) ; - - # pre-build paths to detect reinitializations changes - local inc_prefix lib_prefix bin_prefix ; - if $(full_inc) - { - inc_prefix = $(full_inc) ; - } - else - { - inc_prefix = $(prefix)/include ; - } - if $(full_lib) - { - lib_prefix = $(full_lib) ; - } - else - { - lib_prefix = $(prefix)/lib ; - } - if $(full_bin) - { - bin_prefix = $(full_bin) ; - } - else - { - bin_prefix = $(prefix)/bin ; - } - - # Globally needed variables - .incprefix = $(inc_prefix) ; - .libprefix = $(lib_prefix) ; - .binprefix = $(bin_prefix) ; - - if ! $(.initialized) - { - # Make sure this is initialised only once - .initialized = true ; - - # Generates cpp files from header files using "moc" tool - generators.register-standard qt4.moc : H : CPP(moc_%) : qt4 ; - - # The OBJ result type is a fake, 'H' will be really produced. See - # comments on the generator class, defined below the 'init' function. - generators.register [ new uic-generator qt4.uic : UI : OBJ : - qt4 ] ; - - # The OBJ result type is a fake here too. - generators.register [ new moc-h-generator - qt4.moc.inc : MOCCABLE_CPP : OBJ : qt4 ] ; - - generators.register [ new moc-inc-generator - qt4.moc.inc : MOCCABLE_H : OBJ : qt4 ] ; - - # Generates .cpp files from .qrc files. - generators.register-standard qt4.rcc : QRC : CPP(qrc_%) ; - - # dependency scanner for wrapped files. - type.set-scanner QRC : qrc-scanner ; - - # Save value of first occuring prefix - .PREFIX = $(prefix) ; - } - - if $(version) - { - major-minor = [ split-version $(version) ] ; - version = $(major-minor:J=.) ; - } - else - { - version = [ check-version $(bin_prefix) ] ; - if $(version) - { - version = $(version:J=.) ; - } - debug-message Detected version '$(version)' ; - } - - local target-requirements = $(condition) ; - - # Add the version, if any, to the target requirements. - if $(version) - { - if ! $(version) in [ feature.values qt ] - { - feature.extend qt : $(version) ; - } - target-requirements += $(version:E=default) ; - } - - local target-os = [ feature.get-values target-os : $(condition) ] ; - if ! $(target-os) - { - target-os ?= [ feature.defaults target-os ] ; - target-os = $(target-os:G=) ; - target-requirements += $(target-os) ; - } - - # Build exact requirements for the tools - local tools-requirements = $(target-requirements:J=/) ; - - debug-message "Details of this Qt configuration:" ; - debug-message " prefix: " '$(prefix:E=)' ; - debug-message " binary path: " '$(bin_prefix:E=)' ; - debug-message " include path:" '$(inc_prefix:E=)' ; - debug-message " library path:" '$(lib_prefix:E=)' ; - debug-message " target requirements:" '$(target-requirements)' ; - debug-message " tool requirements: " '$(tools-requirements)' ; - - # setup the paths for the tools - toolset.flags qt4.moc .BINPREFIX $(tools-requirements) : $(bin_prefix) ; - toolset.flags qt4.rcc .BINPREFIX $(tools-requirements) : $(bin_prefix) ; - toolset.flags qt4.uic .BINPREFIX $(tools-requirements) : $(bin_prefix) ; - - # TODO: 2009-02-12: Better support for directories - # Most likely needed are separate getters for: include,libraries,binaries and sources. - toolset.flags qt4.directory .PREFIX $(tools-requirements) : $(prefix) ; - - # Test for a buildable Qt. - if [ glob $(.prefix)/Jamroot ] - { - .bjam-qt = true - - # this will declare QtCore (and qtmain on windows) - add-shared-library QtCore ; - } - else - # Setup common pre-built Qt. - # Special setup for QtCore on which everything depends - { - local usage-requirements = - $(.incprefix) - $(.libprefix) - $(.libprefix) - multi - qt4 ; - - local suffix ; - - # Since Qt-4.2, debug versions on unix have to be built - # separately and therefore have no suffix. - .suffix_version = "" ; - .suffix_debug = "" ; - - # Control flag for auto-configuration of the debug libraries. - # This setup requires Qt 'configure -debug-and-release'. - # Only available on some platforms. - # ToDo: 2009-02-12: Maybe throw this away and - # require separate setup with debug as condition. - .have_separate_debug = FALSE ; - - # Setup other platforms - if $(target-os) in windows cygwin - { - .have_separate_debug = TRUE ; - - # On NT, the libs have "4" suffix, and "d" suffix in debug builds. - .suffix_version = "4" ; - .suffix_debug = "d" ; - - # On Windows we must link against the qtmain library - lib qtmain - : # sources - : # requirements - qtmain$(.suffix_debug) - debug - $(target-requirements) - ; - - lib qtmain - : # sources - : # requirements - qtmain - $(target-requirements) - ; - } - else if $(target-os) = darwin - { - # On MacOS X, both debug and release libraries are available. - .suffix_debug = "_debug" ; - - .have_separate_debug = TRUE ; - - alias qtmain ; - } - else - { - alias qtmain : : $(target-requirements) ; - } - - lib QtCore : qtmain - : # requirements - QtCore$(.suffix_version) - $(target-requirements) - : # default-build - : # usage-requirements - QT_CORE_LIB - QT_NO_DEBUG - $(.incprefix)/QtCore - $(usage-requirements) - ; - - if $(.have_separate_debug) = TRUE - { - debug-message Configure debug libraries with suffix '$(.suffix_debug)' ; - - lib QtCore : $(main) - : # requirements - QtCore$(.suffix_debug)$(.suffix_version) - debug - $(target-requirements) - : # default-build - : # usage-requirements - QT_CORE_LIB - $(.incprefix)/QtCore - $(usage-requirements) - ; - } - } - - # Initialising the remaining libraries is canonical - # parameters 'module' : 'depends-on' : 'usage-define' : 'requirements' : 'include' - # 'include' only for non-canonical include paths. - add-shared-library QtGui : QtCore : QT_GUI_LIB : $(target-requirements) ; - add-shared-library QtNetwork : QtCore : QT_NETWORK_LIB : $(target-requirements) ; - add-shared-library QtSql : QtCore : QT_SQL_LIB : $(target-requirements) ; - add-shared-library QtXml : QtCore : QT_XML_LIB : $(target-requirements) ; - - add-shared-library Qt3Support : QtGui QtNetwork QtXml QtSql - : QT_QT3SUPPORT_LIB QT3_SUPPORT - : on $(target-requirements) ; - - # Dummy target to enable "off" and - # "/qt//Qt3Support" at the same time. This enables quick - # switching from one to the other for test/porting purposes. - alias Qt3Support : : off $(target-requirements) ; - - # OpenGl Support - add-shared-library QtOpenGL : QtGui : QT_OPENGL_LIB : $(target-requirements) ; - - # SVG-Support (Qt 4.1) - add-shared-library QtSvg : QtXml QtOpenGL : QT_SVG_LIB : $(target-requirements) ; - - # Test-Support (Qt 4.1) - add-shared-library QtTest : QtCore : : $(target-requirements) ; - - # Qt designer library - add-shared-library QtDesigner : QtGui QtXml : : $(target-requirements) ; - add-shared-library QtDesignerComponents : QtGui QtXml : : $(target-requirements) ; - - # Support for dynamic Widgets (Qt 4.1) - add-static-library QtUiTools : QtGui QtXml : $(target-requirements) ; - - # DBus-Support (Qt 4.2) - add-shared-library QtDBus : QtXml : : $(target-requirements) ; - - # Script-Engine (Qt 4.3) - add-shared-library QtScript : QtGui QtXml : QT_SCRIPT_LIB : $(target-requirements) ; - - # Tools for the Script-Engine (Qt 4.5) - add-shared-library QtScriptTools : QtScript : QT_SCRIPTTOOLS_LIB : $(target-requirements) ; - - # WebKit (Qt 4.4) - add-shared-library QtWebKit : QtGui : QT_WEBKIT_LIB : $(target-requirements) ; - - # Phonon Multimedia (Qt 4.4) - add-shared-library phonon : QtGui QtXml : QT_PHONON_LIB : $(target-requirements) ; - - # Multimedia engine (Qt 4.6) - add-shared-library QtMultimedia : QtGui : QT_MULTIMEDIA_LIB : $(target-requirements) ; - - # XmlPatterns-Engine (Qt 4.4) - add-shared-library QtXmlPatterns : QtNetwork : QT_XMLPATTERNS_LIB : $(target-requirements) ; - - # Help-Engine (Qt 4.4) - add-shared-library QtHelp : QtGui QtSql QtXml : : $(target-requirements) ; - add-shared-library QtCLucene : QCore QtSql QtXml : : $(target-requirements) ; - - # QML-Engine (Qt 4.7) - add-shared-library QtDeclarative : QtGui QtXml : : $(target-requirements) ; - - # AssistantClient Support - # Compat library removed in 4.7.0 - # Pre-4.4 help system, use QtHelp for new programs - if $(version) < "4.7" - { - add-shared-library QtAssistantClient : QtGui : : $(target-requirements) : QtAssistant ; - } - debug-message "==== Configured Qt-$(version) ====" ; - - project.pop-current ; -} - -rule initialized ( ) -{ - return $(.initialized) ; -} - - - -# This custom generator is needed because in QT4, UI files are translated only -# into H files, and no C++ files are created. Further, the H files need not be -# passed via MOC. The header is used only via inclusion. If we define a standard -# UI -> H generator, Boost.Build will run MOC on H, and then compile the -# resulting cpp. It will give a warning, since output from moc will be empty. -# -# This generator is declared with a UI -> OBJ signature, so it gets invoked when -# linking generator tries to convert sources to OBJ, but it produces target of -# type H. This is non-standard, but allowed. That header won't be mocced. -# -class uic-generator : generator -{ - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * ) - { - if ! $(name) - { - name = [ $(sources[0]).name ] ; - name = $(name:B) ; - } - - local a = [ new action $(sources[1]) : qt4.uic : $(property-set) ] ; - - # The 'ui_' prefix is to match qmake's default behavior. - local target = [ new file-target ui_$(name) : H : $(project) : $(a) ] ; - - local r = [ virtual-target.register $(target) ] ; - - # Since this generator will return a H target, the linking generator - # won't use it at all, and won't set any dependency on it. However, we - # need the target to be seen by bjam, so that dependency from sources to - # this generated header is detected -- if jam does not know about this - # target, it won't do anything. - DEPENDS all : [ $(r).actualize ] ; - - return $(r) ; - } -} - - -class moc-h-generator : generator -{ - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * ) - { - if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_CPP - { - name = [ $(sources[0]).name ] ; - name = $(name:B) ; - - local a = [ new action $(sources[1]) : qt4.moc.inc : - $(property-set) ] ; - - local target = [ new file-target $(name) : MOC : $(project) : $(a) - ] ; - - local r = [ virtual-target.register $(target) ] ; - - # Since this generator will return a H target, the linking generator - # won't use it at all, and won't set any dependency on it. However, - # we need the target to be seen by bjam, so that dependency from - # sources to this generated header is detected -- if jam does not - # know about this target, it won't do anything. - DEPENDS all : [ $(r).actualize ] ; - - return $(r) ; - } - } -} - - -class moc-inc-generator : generator -{ - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * ) - { - if ! $(sources[2]) && [ $(sources[1]).type ] = MOCCABLE_H - { - name = [ $(sources[0]).name ] ; - name = $(name:B) ; - - local a = [ new action $(sources[1]) : qt4.moc.inc : - $(property-set) ] ; - - local target = [ new file-target moc_$(name) : CPP : $(project) : - $(a) ] ; - - # Since this generator will return a H target, the linking generator - # won't use it at all, and won't set any dependency on it. However, - # we need the target to be seen by bjam, so that dependency from - # sources to this generated header is detected -- if jam does not - # know about this target, it won't do anything. - DEPENDS all : [ $(target).actualize ] ; - - return [ virtual-target.register $(target) ] ; - } - } -} - - -# Query the installation directory. This is needed in at least two scenarios. -# First, when re-using sources from the Qt-Tree. Second, to "install" custom Qt -# plugins to the Qt-Tree. -# -rule directory -{ - return $(.PREFIX) ; -} - -# Add a shared Qt library. -rule add-shared-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? ) -{ - add-library $(lib-name) : $(.suffix_version) : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ; -} - -# Add a static Qt library. -rule add-static-library ( lib-name : depends-on * : usage-defines * : requirements * : include ? ) -{ - add-library $(lib-name) : : $(depends-on) : $(usage-defines) : $(requirements) : $(include) ; -} - -# Add a Qt library. -# Static libs are unversioned, whereas shared libs have the major number as suffix. -# Creates both release and debug versions on platforms where both are enabled by Qt configure. -# Flags: -# - lib-name Qt library Name -# - version Qt major number used as shared library suffix (QtCore4.so) -# - depends-on other Qt libraries -# - usage-defines those are set by qmake, so set them when using this library -# - requirements addional requirements -# - include non-canonical include path. The canonical path is $(.incprefix)/$(lib-name). -rule add-library ( lib-name : version ? : depends-on * : usage-defines * : requirements * : include ? ) -{ - if $(.bjam-qt) - { - # Import Qt module - # Eveything will be setup there - alias $(lib-name) - : $(.prefix)//$(lib-name) - : - : - : qt4 ; - } - else - { - local real_include ; - real_include ?= $(include) ; - real_include ?= $(lib-name) ; - - lib $(lib-name) - : # sources - $(depends-on) - : # requirements - $(lib-name)$(version) - $(requirements) - : # default-build - : # usage-requirements - $(usage-defines) - $(.incprefix)/$(real_include) - ; - - if $(.have_separate_debug) = TRUE - { - lib $(lib-name) - : # sources - $(depends-on) - : # requirements - $(lib-name)$(.suffix_debug)$(version) - $(requirements) - debug - : # default-build - : # usage-requirements - $(usage-defines) - $(.incprefix)/$(real_include) - ; - } - } - - # Make library explicit so that a simple qt4 will not bring in everything. - # And some components like QtDBus/Phonon may not be available on all platforms. - explicit $(lib-name) ; -} - -# Use $(.BINPREFIX[-1]) for the paths as several tools-requirements can match. -# The exact match is the last one. - -# Get and from current toolset. -flags qt4.moc INCLUDES ; -flags qt4.moc DEFINES ; - -# need a newline for expansion of DEFINES and INCLUDES in the response file. -.nl = " -" ; - -# Processes headers to create Qt MetaObject information. Qt4-moc has its -# c++-parser, so pass INCLUDES and DEFINES. -# We use response file with one INCLUDE/DEFINE per line -# -actions moc -{ - $(.BINPREFIX[-1])/moc -f $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))" -} - -# When moccing files for include only, we don't need -f, otherwise the generated -# code will include the .cpp and we'll get duplicated symbols. -# -actions moc.inc -{ - $(.BINPREFIX[-1])/moc $(>) -o $(<) @"@($(<).rsp:E=-D$(DEFINES)$(.nl) -I$(INCLUDES:T)$(.nl))" -} - - -# Generates source files from resource files. -# -actions rcc -{ - $(.BINPREFIX[-1])/rcc $(>) -name $(>:B) -o $(<) -} - - -# Generates user-interface source from .ui files. -# -actions uic -{ - $(.BINPREFIX[-1])/uic $(>) -o $(<) -} - - -# Scanner for .qrc files. Look for the CDATA section of the tag. Ignore -# the "alias" attribute. See http://doc.trolltech.com/qt/resources.html for -# detailed documentation of the Qt Resource System. -# -class qrc-scanner : common-scanner -{ - rule pattern ( ) - { - return "(.*)" ; - } -} - - -# Wrapped files are "included". -scanner.register qrc-scanner : include ; diff --git a/jam-files/boost-build/tools/quickbook-config.jam b/jam-files/boost-build/tools/quickbook-config.jam deleted file mode 100644 index e983a78a..00000000 --- a/jam-files/boost-build/tools/quickbook-config.jam +++ /dev/null @@ -1,44 +0,0 @@ -#~ Copyright 2005 Rene Rivera. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Automatic configuration for BoostBook tools. To use, just import this module. - -import os ; -import toolset : using ; - -if [ os.name ] = NT -{ - local boost-dir = ; - for local R in snapshot cvs 1.33.0 - { - boost-dir += [ W32_GETREG - "HKEY_LOCAL_MACHINE\\SOFTWARE\\Boost.org\\$(R)" - : "InstallRoot" ] ; - } - local quickbook-path = [ GLOB "$(boost-dir)\\bin" "\\Boost\\bin" : quickbook.exe ] ; - quickbook-path = $(quickbook-path[1]) ; - - if $(quickbook-path) - { - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using quickbook ":" $(quickbook-path) ; - } - using quickbook : $(quickbook-path) ; - } -} -else -{ - local quickbook-path = [ GLOB "/usr/local/bin" "/usr/bin" "/opt/bin" : quickbook ] ; - quickbook-path = $(quickbook-path[1]) ; - - if $(quickbook-path) - { - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using quickbook ":" $(quickbook-path) ; - } - using quickbook : $(quickbook-path) ; - } -} diff --git a/jam-files/boost-build/tools/quickbook.jam b/jam-files/boost-build/tools/quickbook.jam deleted file mode 100644 index 6de2d42f..00000000 --- a/jam-files/boost-build/tools/quickbook.jam +++ /dev/null @@ -1,361 +0,0 @@ -# -# Copyright (c) 2005 João Abecasis -# Copyright (c) 2005 Vladimir Prus -# Copyright (c) 2006 Rene Rivera -# -# Distributed under the Boost Software License, Version 1.0. (See -# accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) -# - -# This toolset defines a generator to translate QuickBook to BoostBook. It can -# be used to generate nice (!) user documentation in different formats -# (pdf/html/...), from a single text file with simple markup. -# -# The toolset defines the QUICKBOOK type (file extension 'qbk') and -# a QUICKBOOK to XML (BOOSTBOOK) generator. -# -# -# =========================================================================== -# Q & A -# =========================================================================== -# -# If you don't know what this is all about, some Q & A will hopefully get you -# up to speed with QuickBook and this toolset. -# -# -# What is QuickBook ? -# -# QuickBook is a WikiWiki style documentation tool geared towards C++ -# documentation using simple rules and markup for simple formatting tasks. -# QuickBook extends the WikiWiki concept. Like the WikiWiki, QuickBook -# documents are simple text files. A single QuickBook document can -# generate a fully linked set of nice HTML and PostScript/PDF documents -# complete with images and syntax-colorized source code. -# -# -# Where can I get QuickBook ? -# -# Quickbook can be found in Boost's repository, under the tools/quickbook -# directory it was added there on Jan 2005, some time after the release of -# Boost v1.32.0 and has been an integral part of the Boost distribution -# since v1.33. -# -# Here's a link to the SVN repository: -# https://svn.boost.org/svn/boost/trunk/tools/quickbook -# -# And to QuickBook's QuickBook-generated docs: -# http://www.boost.org/doc/libs/release/tools/quickbook/index.html -# -# -# How do I use QuickBook and this toolset in my projects ? -# -# The minimal example is: -# -# using boostbook ; -# import quickbook ; -# -# boostbook my_docs : my_docs_source.qbk ; -# -# where my_docs is a target name and my_docs_source.qbk is a QuickBook -# file. The documentation format to be generated is determined by the -# boostbook toolset. By default html documentation should be generated, -# but you should check BoostBook's docs to be sure. -# -# -# What do I need ? -# -# You should start by setting up the BoostBook toolset. Please refer to -# boostbook.jam and the BoostBook documentation for information on how to -# do this. -# -# A QuickBook executable is also needed. The toolset will generate this -# executable if it can find the QuickBook sources. The following -# directories will be searched: -# -# BOOST_ROOT/tools/quickbook/ -# BOOST_BUILD_PATH/../../quickbook/ -# -# (BOOST_ROOT and BOOST_BUILD_PATH are environment variables) -# -# If QuickBook sources are not found the toolset will then try to use -# the shell command 'quickbook'. -# -# -# How do I provide a custom QuickBook executable ? -# -# You may put the following in your user-config.jam or site-config.jam: -# -# using quickbook : /path/to/quickbook ; -# -# or, if 'quickbook' can be found in your PATH, -# -# using quickbook : quickbook ; -# -# -# For convenience three alternatives are tried to get a QuickBook executable: -# -# 1. If the user points us to the a QuickBook executable, that is used. -# -# 2. Otherwise, we search for the QuickBook sources and compile QuickBook -# using the default toolset. -# -# 3. As a last resort, we rely on the shell for finding 'quickbook'. -# - -import boostbook ; -import "class" : new ; -import feature ; -import generators ; -import toolset ; -import type ; -import scanner ; -import project ; -import targets ; -import build-system ; -import path ; -import common ; -import errors ; - -# The one and only QUICKBOOK type! -type.register QUICKBOOK : qbk ; - -# shell command to run QuickBook -# targets to build QuickBook from sources. -feature.feature : : free ; -feature.feature : : free dependency ; -feature.feature : : free ; -feature.feature : : free ; -feature.feature : : free ; - - -# quickbook-binary-generator handles generation of the QuickBook executable, by -# marking it as a dependency for QuickBook docs. -# -# If the user supplied the QuickBook command that will be used. -# -# Otherwise we search some sensible places for the QuickBook sources and compile -# from scratch using the default toolset. -# -# As a last resort we rely on the shell to find 'quickbook'. -# -class quickbook-binary-generator : generator -{ - import modules path targets quickbook ; - - rule run ( project name ? : property-set : sources * : multiple ? ) - { - quickbook.freeze-config ; - # QuickBook invocation command and dependencies. - local quickbook-binary = [ modules.peek quickbook : .quickbook-binary ] ; - local quickbook-binary-dependencies ; - - if ! $(quickbook-binary) - { - # If the QuickBook source directory was found, mark its main target - # as a dependency for the current project. Otherwise, try to find - # 'quickbook' in user's PATH - local quickbook-dir = [ modules.peek quickbook : .quickbook-dir ] ; - if $(quickbook-dir) - { - # Get the main-target in QuickBook directory. - local quickbook-main-target = [ targets.resolve-reference $(quickbook-dir) : $(project) ] ; - - # The first element are actual targets, the second are - # properties found in target-id. We do not care about these - # since we have passed the id ourselves. - quickbook-main-target = - [ $(quickbook-main-target[1]).main-target quickbook ] ; - - quickbook-binary-dependencies = - [ $(quickbook-main-target).generate [ $(property-set).propagated ] ] ; - - # Ignore usage-requirements returned as first element. - quickbook-binary-dependencies = $(quickbook-binary-dependencies[2-]) ; - - # Some toolsets generate extra targets (e.g. RSP). We must mark - # all targets as dependencies for the project, but we will only - # use the EXE target for quickbook-to-boostbook translation. - for local target in $(quickbook-binary-dependencies) - { - if [ $(target).type ] = EXE - { - quickbook-binary = - [ path.native - [ path.join - [ $(target).path ] - [ $(target).name ] - ] - ] ; - } - } - } - } - - # Add $(quickbook-binary-dependencies) as a dependency of the current - # project and set it as the feature for the - # quickbook-to-boostbook rule, below. - property-set = [ $(property-set).add-raw - $(quickbook-binary-dependencies) - $(quickbook-binary) - $(quickbook-binary-dependencies) - ] ; - - return [ generator.run $(project) $(name) : $(property-set) : $(sources) : $(multiple) ] ; - } -} - - -# Define a scanner for tracking QBK include dependencies. -# -class qbk-scanner : common-scanner -{ - rule pattern ( ) - { - return "\\[[ ]*include[ ]+([^]]+)\\]" - "\\[[ ]*include:[a-zA-Z0-9_]+[ ]+([^]]+)\\]" - "\\[[ ]*import[ ]+([^]]+)\\]" ; - } -} - - -scanner.register qbk-scanner : include ; - -type.set-scanner QUICKBOOK : qbk-scanner ; - - -# Initialization of toolset. -# -# Parameters: -# command ? -> path to QuickBook executable. -# -# When command is not supplied toolset will search for QuickBook directory and -# compile the executable from source. If that fails we still search the path for -# 'quickbook'. -# -rule init ( - command ? # path to the QuickBook executable. - ) -{ - if $(command) - { - if $(.config-frozen) - { - errors.user-error "quickbook: configuration cannot be changed after it has been used." ; - } - .command = $(command) ; - } -} - -rule freeze-config ( ) -{ - if ! $(.config-frozen) - { - .config-frozen = true ; - - # QuickBook invocation command and dependencies. - - .quickbook-binary = $(.command) ; - - if $(.quickbook-binary) - { - # Use user-supplied command. - .quickbook-binary = [ common.get-invocation-command quickbook : quickbook : $(.quickbook-binary) ] ; - } - else - { - # Search for QuickBook sources in sensible places, like - # $(BOOST_ROOT)/tools/quickbook - # $(BOOST_BUILD_PATH)/../../quickbook - - # And build quickbook executable from sources. - - local boost-root = [ modules.peek : BOOST_ROOT ] ; - local boost-build-path = [ build-system.location ] ; - - if $(boost-root) - { - .quickbook-dir += [ path.join $(boost-root) tools ] ; - } - - if $(boost-build-path) - { - .quickbook-dir += $(boost-build-path)/../.. ; - } - - .quickbook-dir = [ path.glob $(.quickbook-dir) : quickbook ] ; - - # If the QuickBook source directory was found, mark its main target - # as a dependency for the current project. Otherwise, try to find - # 'quickbook' in user's PATH - if $(.quickbook-dir) - { - .quickbook-dir = [ path.make $(.quickbook-dir[1]) ] ; - } - else - { - ECHO "QuickBook warning: The path to the quickbook executable was" ; - ECHO " not provided. Additionally, couldn't find QuickBook" ; - ECHO " sources searching in" ; - ECHO " * BOOST_ROOT/tools/quickbook" ; - ECHO " * BOOST_BUILD_PATH/../../quickbook" ; - ECHO " Will now try to find a precompiled executable by searching" ; - ECHO " the PATH for 'quickbook'." ; - ECHO " To disable this warning in the future, or to completely" ; - ECHO " avoid compilation of quickbook, you can explicitly set the" ; - ECHO " path to a quickbook executable command in user-config.jam" ; - ECHO " or site-config.jam with the call" ; - ECHO " using quickbook : /path/to/quickbook ;" ; - - # As a last resort, search for 'quickbook' command in path. Note - # that even if the 'quickbook' command is not found, - # get-invocation-command will still return 'quickbook' and might - # generate an error while generating the virtual-target. - - .quickbook-binary = [ common.get-invocation-command quickbook : quickbook ] ; - } - } - } -} - - -generators.register [ new quickbook-binary-generator quickbook.quickbook-to-boostbook : QUICKBOOK : XML ] ; - - -# shell command to run QuickBook -# targets to build QuickBook from sources. -toolset.flags quickbook.quickbook-to-boostbook QB-COMMAND ; -toolset.flags quickbook.quickbook-to-boostbook QB-DEPENDENCIES ; -toolset.flags quickbook.quickbook-to-boostbook INCLUDES ; -toolset.flags quickbook.quickbook-to-boostbook QB-DEFINES ; -toolset.flags quickbook.quickbook-to-boostbook QB-INDENT ; -toolset.flags quickbook.quickbook-to-boostbook QB-LINE-WIDTH ; - - -rule quickbook-to-boostbook ( target : source : properties * ) -{ - # Signal dependency of quickbook sources on - # upon invocation of quickbook-to-boostbook. - DEPENDS $(target) : [ on $(target) return $(QB-DEPENDENCIES) ] ; -} - - -actions quickbook-to-boostbook -{ - "$(QB-COMMAND)" -I"$(INCLUDES)" -D"$(QB-DEFINES)" --indent="$(QB-INDENT)" --linewidth="$(QB-LINE-WIDTH)" --output-file="$(1)" "$(2)" -} - - -# Declare a main target to convert a quickbook source into a boostbook XML file. -# -rule to-boostbook ( target-name : sources * : requirements * : default-build * ) -{ - local project = [ project.current ] ; - - targets.main-target-alternative - [ new typed-target $(target-name) : $(project) : XML - : [ targets.main-target-sources $(sources) : $(target-name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; -} diff --git a/jam-files/boost-build/tools/rc.jam b/jam-files/boost-build/tools/rc.jam deleted file mode 100644 index 9964d339..00000000 --- a/jam-files/boost-build/tools/rc.jam +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright (C) Andre Hentz 2003. Permission to copy, use, modify, sell and -# distribute this software is granted provided this copyright notice appears in -# all copies. This software is provided "as is" without express or implied -# warranty, and with no claim as to its suitability for any purpose. -# -# Copyright (c) 2006 Rene Rivera. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -import type ; -import generators ; -import feature ; -import errors ; -import scanner ; -import toolset : flags ; - -if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ] -{ - .debug-configuration = true ; -} - -type.register RC : rc ; - -rule init ( ) -{ -} - -# Configures a new resource compilation command specific to a condition, -# usually a toolset selection condition. The possible options are: -# -# * (rc|windres) - Indicates the type of options the command -# accepts. -# -# Even though the arguments are all optional, only when a command, condition, -# and at minimum the rc-type option are given will the command be configured. -# This is so that callers don't have to check auto-configuration values -# before calling this. And still get the functionality of build failures when -# the resource compiler can't be found. -# -rule configure ( command ? : condition ? : options * ) -{ - local rc-type = [ feature.get-values : $(options) ] ; - - if $(command) && $(condition) && $(rc-type) - { - flags rc.compile.resource .RC $(condition) : $(command) ; - flags rc.compile.resource .RC_TYPE $(condition) : $(rc-type:L) ; - flags rc.compile.resource DEFINES ; - flags rc.compile.resource INCLUDES ; - if $(.debug-configuration) - { - ECHO notice: using rc compiler :: $(condition) :: $(command) ; - } - } -} - -rule compile.resource ( target : sources * : properties * ) -{ - local rc-type = [ on $(target) return $(.RC_TYPE) ] ; - rc-type ?= null ; - compile.resource.$(rc-type) $(target) : $(sources[1]) ; -} - -actions compile.resource.rc -{ - "$(.RC)" -l 0x409 "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -fo "$(<)" "$(>)" -} - -actions compile.resource.windres -{ - "$(.RC)" "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -o "$(<)" -i "$(>)" -} - -actions quietly compile.resource.null -{ - as /dev/null -o "$(<)" -} - -# Since it's a common practice to write -# exe hello : hello.cpp hello.rc -# we change the name of object created from RC file, to -# avoid conflict with hello.cpp. -# The reason we generate OBJ and not RES, is that gcc does not -# seem to like RES files, but works OK with OBJ. -# See http://article.gmane.org/gmane.comp.lib.boost.build/5643/ -# -# Using 'register-c-compiler' adds the build directory to INCLUDES -generators.register-c-compiler rc.compile.resource : RC : OBJ(%_res) ; - -# Register scanner for resources -class res-scanner : scanner -{ - import regex virtual-target path scanner ; - - rule __init__ ( includes * ) - { - scanner.__init__ ; - - self.includes = $(includes) ; - } - - rule pattern ( ) - { - return "(([^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)[ ]+([^ \"]+|\"[^\"]+\"))|(#include[ ]*(<[^<]+>|\"[^\"]+\")))" ; - } - - rule process ( target : matches * : binding ) - { - local angle = [ regex.transform $(matches) : "#include[ ]*<([^<]+)>" ] ; - local quoted = [ regex.transform $(matches) : "#include[ ]*\"([^\"]+)\"" ] ; - local res = [ regex.transform $(matches) : "[^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)[ ]+(([^ \"]+)|\"([^\"]+)\")" : 3 4 ] ; - - # Icons and other includes may referenced as - # - # IDR_MAINFRAME ICON "res\\icon.ico" - # - # so we have to replace double backslashes to single ones. - res = [ regex.replace-list $(res) : "\\\\\\\\" : "/" ] ; - - # CONSIDER: the new scoping rule seem to defeat "on target" variables. - local g = [ on $(target) return $(HDRGRIST) ] ; - local b = [ NORMALIZE_PATH $(binding:D) ] ; - - # Attach binding of including file to included targets. - # When target is directly created from virtual target - # this extra information is unnecessary. But in other - # cases, it allows to distinguish between two headers of the - # same name included from different places. - # We don't need this extra information for angle includes, - # since they should not depend on including file (we can't - # get literal "." in include path). - local g2 = $(g)"#"$(b) ; - - angle = $(angle:G=$(g)) ; - quoted = $(quoted:G=$(g2)) ; - res = $(res:G=$(g2)) ; - - local all = $(angle) $(quoted) ; - - INCLUDES $(target) : $(all) ; - DEPENDS $(target) : $(res) ; - NOCARE $(all) $(res) ; - SEARCH on $(angle) = $(self.includes:G=) ; - SEARCH on $(quoted) = $(b) $(self.includes:G=) ; - SEARCH on $(res) = $(b) $(self.includes:G=) ; - - # Just propagate current scanner to includes, in a hope - # that includes do not change scanners. - scanner.propagate $(__name__) : $(angle) $(quoted) : $(target) ; - } -} - -scanner.register res-scanner : include ; -type.set-scanner RC : res-scanner ; diff --git a/jam-files/boost-build/tools/rc.py b/jam-files/boost-build/tools/rc.py deleted file mode 100644 index 0b82d231..00000000 --- a/jam-files/boost-build/tools/rc.py +++ /dev/null @@ -1,189 +0,0 @@ -# Status: being ported by Steven Watanabe -# Base revision: 47077 -# -# Copyright (C) Andre Hentz 2003. Permission to copy, use, modify, sell and -# distribute this software is granted provided this copyright notice appears in -# all copies. This software is provided "as is" without express or implied -# warranty, and with no claim as to its suitability for any purpose. -# -# Copyright (c) 2006 Rene Rivera. -# -# Copyright (c) 2008 Steven Watanabe -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -##import type ; -##import generators ; -##import feature ; -##import errors ; -##import scanner ; -##import toolset : flags ; - -from b2.build import type, toolset, generators, scanner, feature -from b2.tools import builtin -from b2.util import regex -from b2.build.toolset import flags -from b2.manager import get_manager - -__debug = None - -def debug(): - global __debug - if __debug is None: - __debug = "--debug-configuration" in bjam.variable("ARGV") - return __debug - -type.register('RC', ['rc']) - -def init(): - pass - -def configure (command = None, condition = None, options = None): - """ - Configures a new resource compilation command specific to a condition, - usually a toolset selection condition. The possible options are: - - * (rc|windres) - Indicates the type of options the command - accepts. - - Even though the arguments are all optional, only when a command, condition, - and at minimum the rc-type option are given will the command be configured. - This is so that callers don't have to check auto-configuration values - before calling this. And still get the functionality of build failures when - the resource compiler can't be found. - """ - rc_type = feature.get_values('', options) - if rc_type: - assert(len(rc_type) == 1) - rc_type = rc_type[0] - - if command and condition and rc_type: - flags('rc.compile.resource', '.RC', condition, command) - flags('rc.compile.resource', '.RC_TYPE', condition, rc_type.lower()) - flags('rc.compile.resource', 'DEFINES', [], ['']) - flags('rc.compile.resource', 'INCLUDES', [], ['']) - if debug(): - print 'notice: using rc compiler ::', condition, '::', command - -engine = get_manager().engine() - -class RCAction: - """Class representing bjam action defined from Python. - The function must register the action to execute.""" - - def __init__(self, action_name, function): - self.action_name = action_name - self.function = function - - def __call__(self, targets, sources, property_set): - if self.function: - self.function(targets, sources, property_set) - -# FIXME: What is the proper way to dispatch actions? -def rc_register_action(action_name, function = None): - global engine - if engine.actions.has_key(action_name): - raise "Bjam action %s is already defined" % action_name - engine.actions[action_name] = RCAction(action_name, function) - -def rc_compile_resource(targets, sources, properties): - rc_type = bjam.call('get-target-variable', targets, '.RC_TYPE') - global engine - engine.set_update_action('rc.compile.resource.' + rc_type, targets, sources, properties) - -rc_register_action('rc.compile.resource', rc_compile_resource) - - -engine.register_action( - 'rc.compile.resource.rc', - '"$(.RC)" -l 0x409 "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -fo "$(<)" "$(>)"') - -engine.register_action( - 'rc.compile.resource.windres', - '"$(.RC)" "-U$(UNDEFS)" "-D$(DEFINES)" -I"$(>:D)" -I"$(<:D)" -I"$(INCLUDES)" -o "$(<)" -i "$(>)"') - -# FIXME: this was originally declared quietly -engine.register_action( - 'compile.resource.null', - 'as /dev/null -o "$(<)"') - -# Since it's a common practice to write -# exe hello : hello.cpp hello.rc -# we change the name of object created from RC file, to -# avoid conflict with hello.cpp. -# The reason we generate OBJ and not RES, is that gcc does not -# seem to like RES files, but works OK with OBJ. -# See http://article.gmane.org/gmane.comp.lib.boost.build/5643/ -# -# Using 'register-c-compiler' adds the build directory to INCLUDES -# FIXME: switch to generators -builtin.register_c_compiler('rc.compile.resource', ['RC'], ['OBJ(%_res)'], []) - -__angle_include_re = "#include[ ]*<([^<]+)>" - -# Register scanner for resources -class ResScanner(scanner.Scanner): - - def __init__(self, includes): - scanner.__init__ ; - self.includes = includes - - def pattern(self): - return "(([^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)" +\ - "[ ]+([^ \"]+|\"[^\"]+\"))|(#include[ ]*(<[^<]+>|\"[^\"]+\")))" ; - - def process(self, target, matches, binding): - - angle = regex.transform(matches, "#include[ ]*<([^<]+)>") - quoted = regex.transform(matches, "#include[ ]*\"([^\"]+)\"") - res = regex.transform(matches, - "[^ ]+[ ]+(BITMAP|CURSOR|FONT|ICON|MESSAGETABLE|RT_MANIFEST)" +\ - "[ ]+(([^ \"]+)|\"([^\"]+)\")", [3, 4]) - - # Icons and other includes may referenced as - # - # IDR_MAINFRAME ICON "res\\icon.ico" - # - # so we have to replace double backslashes to single ones. - res = [ re.sub(r'\\\\', '/', match) for match in res ] - - # CONSIDER: the new scoping rule seem to defeat "on target" variables. - g = bjam.call('get-target-variable', target, 'HDRGRIST') - b = os.path.normalize_path(os.path.dirname(binding)) - - # Attach binding of including file to included targets. - # When target is directly created from virtual target - # this extra information is unnecessary. But in other - # cases, it allows to distinguish between two headers of the - # same name included from different places. - # We don't need this extra information for angle includes, - # since they should not depend on including file (we can't - # get literal "." in include path). - g2 = g + "#" + b - - g = "<" + g + ">" - g2 = "<" + g2 + ">" - angle = [g + x for x in angle] - quoted = [g2 + x for x in quoted] - res = [g2 + x for x in res] - - all = angle + quoted - - bjam.call('mark-included', target, all) - - engine = get_manager().engine() - - engine.add_dependency(target, res) - bjam.call('NOCARE', all + res) - engine.set_target_variable(angle, 'SEARCH', ungrist(self.includes)) - engine.set_target_variable(quoted, 'SEARCH', b + ungrist(self.includes)) - engine.set_target_variable(res, 'SEARCH', b + ungrist(self.includes)) ; - - # Just propagate current scanner to includes, in a hope - # that includes do not change scanners. - get_manager().scanners().propagate(self, angle + quoted) - -scanner.register(ResScanner, 'include') -type.set_scanner('RC', ResScanner) diff --git a/jam-files/boost-build/tools/stage.jam b/jam-files/boost-build/tools/stage.jam deleted file mode 100644 index 296e7558..00000000 --- a/jam-files/boost-build/tools/stage.jam +++ /dev/null @@ -1,524 +0,0 @@ -# Copyright 2003 Dave Abrahams -# Copyright 2005, 2006 Rene Rivera -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module defines the 'install' rule, used to copy a set of targets to a -# single location. - -import targets ; -import "class" : new ; -import errors ; -import type ; -import generators ; -import feature ; -import project ; -import virtual-target ; -import path ; -import types/register ; - - -feature.feature : off on : incidental ; -feature.feature : : free incidental ; -feature.feature : : free path ; -feature.feature : : free incidental ; - -# If 'on', version symlinks for shared libraries will not be created. Affects -# Unix builds only. -feature.feature : on : optional incidental ; - - -class install-target-class : basic-target -{ - import feature ; - import project ; - import type ; - import errors ; - import generators ; - import path ; - import stage ; - import "class" : new ; - import property ; - import property-set ; - - rule __init__ ( name-and-dir : project : sources * : requirements * : default-build * ) - { - basic-target.__init__ $(name-and-dir) : $(project) : $(sources) : - $(requirements) : $(default-build) ; - } - - # If is not set, sets it based on the project data. - # - rule update-location ( property-set ) - { - local loc = [ $(property-set).get ] ; - if ! $(loc) - { - loc = [ path.root $(self.name) [ $(self.project).get location ] ] ; - property-set = [ $(property-set).add-raw $(loc:G=) ] ; - } - - return $(property-set) ; - } - - # Takes a target that is installed and a property set which is used when - # installing. - # - rule adjust-properties ( target : build-property-set ) - { - local ps-raw ; - local a = [ $(target).action ] ; - if $(a) - { - local ps = [ $(a).properties ] ; - ps-raw = [ $(ps).raw ] ; - - # Unless true is in properties, which can happen - # only if the user has explicitly requested it, nuke all - # properties. - if [ $(build-property-set).get ] != true - { - ps-raw = [ property.change $(ps-raw) : ] ; - } - - # If any properties were specified for installing, add - # them. - local l = [ $(build-property-set).get ] ; - ps-raw += $(l:G=) ; - - # Also copy feature from current build set, to be used - # for relinking. - local l = [ $(build-property-set).get ] ; - ps-raw += $(l:G=) ; - - # Remove the feature on original targets. - ps-raw = [ property.change $(ps-raw) : ] ; - - # And . If stage target has another stage target in - # sources, then we shall get virtual targets with the - # property set. - ps-raw = [ property.change $(ps-raw) : ] ; - } - - local d = [ $(build-property-set).get ] ; - ps-raw += $(d:G=) ; - - local d = [ $(build-property-set).get ] ; - ps-raw += $(d:G=) ; - - local ns = [ $(build-property-set).get ] ; - ps-raw += $(ns:G=) ; - - local d = [ $(build-property-set).get ] ; - # Make the path absolute: we shall use it to compute relative paths and - # making the path absolute will help. - if $(d) - { - d = [ path.root $(d) [ path.pwd ] ] ; - ps-raw += $(d:G=) ; - } - - if $(ps-raw) - { - return [ property-set.create $(ps-raw) ] ; - } - else - { - return [ property-set.empty ] ; - } - } - - rule construct ( name : source-targets * : property-set ) - { - source-targets = [ targets-to-stage $(source-targets) : - $(property-set) ] ; - - property-set = [ update-location $(property-set) ] ; - - local ename = [ $(property-set).get ] ; - - if $(ename) && $(source-targets[2]) - { - errors.error "When property is used in 'install', only one" - "source is allowed" ; - } - - local result ; - for local i in $(source-targets) - { - local staged-targets ; - - local new-properties = [ adjust-properties $(i) : - $(property-set) ] ; - - # See if something special should be done when staging this type. It - # is indicated by the presence of a special "INSTALLED_" type. - local t = [ $(i).type ] ; - if $(t) && [ type.registered INSTALLED_$(t) ] - { - if $(ename) - { - errors.error "In 'install': property specified with target that requires relinking." ; - } - else - { - local targets = [ generators.construct $(self.project) - $(name) : INSTALLED_$(t) : $(new-properties) : $(i) ] ; - staged-targets += $(targets[2-]) ; - } - } - else - { - staged-targets = [ stage.copy-file $(self.project) $(ename) : - $(i) : $(new-properties) ] ; - } - - if ! $(staged-targets) - { - errors.error "Unable to generate staged version of " [ $(source).str ] ; - } - - for t in $(staged-targets) - { - result += [ virtual-target.register $(t) ] ; - } - } - - return [ property-set.empty ] $(result) ; - } - - # Given the list of source targets explicitly passed to 'stage', returns the - # list of targets which must be staged. - # - rule targets-to-stage ( source-targets * : property-set ) - { - local result ; - - # Traverse the dependencies, if needed. - if [ $(property-set).get ] = "on" - { - source-targets = [ collect-targets $(source-targets) ] ; - } - - # Filter the target types, if needed. - local included-types = [ $(property-set).get ] ; - for local r in $(source-targets) - { - local ty = [ $(r).type ] ; - if $(ty) - { - # Do not stage searched libs. - if $(ty) != SEARCHED_LIB - { - if $(included-types) - { - if [ include-type $(ty) : $(included-types) ] - { - result += $(r) ; - } - } - else - { - result += $(r) ; - } - } - } - else if ! $(included-types) - { - # Don't install typeless target if there is an explicit list of - # allowed types. - result += $(r) ; - } - } - - return $(result) ; - } - - # CONSIDER: figure out why we can not use virtual-target.traverse here. - # - rule collect-targets ( targets * ) - { - # Find subvariants - local s ; - for local t in $(targets) - { - s += [ $(t).creating-subvariant ] ; - } - s = [ sequence.unique $(s) ] ; - - local result = [ new set ] ; - $(result).add $(targets) ; - - for local i in $(s) - { - $(i).all-referenced-targets $(result) ; - } - local result2 ; - for local r in [ $(result).list ] - { - if $(r:G) != - { - result2 += $(r:G=) ; - } - } - DELETE_MODULE $(result) ; - result = [ sequence.unique $(result2) ] ; - } - - # Returns true iff 'type' is subtype of some element of 'types-to-include'. - # - local rule include-type ( type : types-to-include * ) - { - local found ; - while $(types-to-include) && ! $(found) - { - if [ type.is-subtype $(type) $(types-to-include[1]) ] - { - found = true ; - } - types-to-include = $(types-to-include[2-]) ; - } - - return $(found) ; - } -} - - -# Creates a copy of target 'source'. The 'properties' object should have a -# property which specifies where the target must be placed. -# -rule copy-file ( project name ? : source : properties ) -{ - name ?= [ $(source).name ] ; - local relative ; - - local new-a = [ new non-scanning-action $(source) : common.copy : - $(properties) ] ; - local source-root = [ $(properties).get ] ; - if $(source-root) - { - # Get the real path of the target. We probably need to strip relative - # path from the target name at construction. - local path = [ $(source).path ] ; - path = [ path.root $(name:D) $(path) ] ; - # Make the path absolute. Otherwise, it would be hard to compute the - # relative path. The 'source-root' is already absolute, see the - # 'adjust-properties' method above. - path = [ path.root $(path) [ path.pwd ] ] ; - - relative = [ path.relative-to $(source-root) $(path) ] ; - } - - # Note: Using $(name:D=$(relative)) might be faster here, but then we would - # need to explicitly check that relative is not ".", otherwise we might get - # paths like '/boost/.', try to create it and mkdir would obviously - # fail. - name = [ path.join $(relative) $(name:D=) ] ; - - return [ new file-target $(name) exact : [ $(source).type ] : $(project) : - $(new-a) ] ; -} - - -rule symlink ( name : project : source : properties ) -{ - local a = [ new action $(source) : symlink.ln : $(properties) ] ; - return [ new file-target $(name) exact : [ $(source).type ] : $(project) : - $(a) ] ; -} - - -rule relink-file ( project : source : property-set ) -{ - local action = [ $(source).action ] ; - local cloned-action = [ virtual-target.clone-action $(action) : $(project) : - "" : $(property-set) ] ; - return [ $(cloned-action).targets ] ; -} - - -# Declare installed version of the EXE type. Generator for this type will cause -# relinking to the new location. -type.register INSTALLED_EXE : : EXE ; - - -class installed-exe-generator : generator -{ - import type ; - import property-set ; - import modules ; - import stage ; - - rule __init__ ( ) - { - generator.__init__ install-exe : EXE : INSTALLED_EXE ; - } - - rule run ( project name ? : property-set : source : multiple ? ) - { - local need-relink ; - - if [ $(property-set).get ] in NT CYGWIN || - [ $(property-set).get ] in windows cygwin - { - } - else - { - # See if the dll-path properties are not changed during - # install. If so, copy, don't relink. - local a = [ $(source).action ] ; - local p = [ $(a).properties ] ; - local original = [ $(p).get ] ; - local current = [ $(property-set).get ] ; - - if $(current) != $(original) - { - need-relink = true ; - } - } - - - if $(need-relink) - { - return [ stage.relink-file $(project) - : $(source) : $(property-set) ] ; - } - else - { - return [ stage.copy-file $(project) - : $(source) : $(property-set) ] ; - } - } -} - - -generators.register [ new installed-exe-generator ] ; - - -# Installing a shared link on Unix might cause a creation of versioned symbolic -# links. -type.register INSTALLED_SHARED_LIB : : SHARED_LIB ; - - -class installed-shared-lib-generator : generator -{ - import type ; - import property-set ; - import modules ; - import stage ; - - rule __init__ ( ) - { - generator.__init__ install-shared-lib : SHARED_LIB - : INSTALLED_SHARED_LIB ; - } - - rule run ( project name ? : property-set : source : multiple ? ) - { - if [ $(property-set).get ] in NT CYGWIN || - [ $(property-set).get ] in windows cygwin - { - local copied = [ stage.copy-file $(project) : $(source) : - $(property-set) ] ; - return [ virtual-target.register $(copied) ] ; - } - else - { - local a = [ $(source).action ] ; - local copied ; - if ! $(a) - { - # Non-derived file, just copy. - copied = [ stage.copy-file $(project) : $(source) : - $(property-set) ] ; - } - else - { - local cp = [ $(a).properties ] ; - local current-dll-path = [ $(cp).get ] ; - local new-dll-path = [ $(property-set).get ] ; - - if $(current-dll-path) != $(new-dll-path) - { - # Rpath changed, need to relink. - copied = [ stage.relink-file $(project) : $(source) : - $(property-set) ] ; - } - else - { - copied = [ stage.copy-file $(project) : $(source) : - $(property-set) ] ; - } - } - - copied = [ virtual-target.register $(copied) ] ; - - local result = $(copied) ; - # If the name is in the form NNN.XXX.YYY.ZZZ, where all 'X', 'Y' and - # 'Z' are numbers, we need to create NNN.XXX and NNN.XXX.YYY - # symbolic links. - local m = [ MATCH (.*)\\.([0123456789]+)\\.([0123456789]+)\\.([0123456789]+)$ - : [ $(copied).name ] ] ; - if $(m) - { - # Symlink without version at all is used to make - # -lsome_library work. - result += [ stage.symlink $(m[1]) : $(project) : $(copied) : - $(property-set) ] ; - - # Symlinks of some libfoo.N and libfoo.N.M are used so that - # library can found at runtime, if libfoo.N.M.X has soname of - # libfoo.N. That happens when the library makes some binary - # compatibility guarantees. If not, it is possible to skip those - # symlinks. - local suppress = - [ $(property-set).get ] ; - - if $(suppress) != "on" - { - result += [ stage.symlink $(m[1]).$(m[2]) : $(project) - : $(copied) : $(property-set) ] ; - result += [ stage.symlink $(m[1]).$(m[2]).$(m[3]) : $(project) - : $(copied) : $(property-set) ] ; - } - } - - return $(result) ; - } - } -} - -generators.register [ new installed-shared-lib-generator ] ; - - -# Main target rule for 'install'. -# -rule install ( name : sources * : requirements * : default-build * ) -{ - local project = [ project.current ] ; - - # Unless the user has explicitly asked us to hardcode dll paths, add - # false in requirements, to override default value. - if ! true in $(requirements) - { - requirements += false ; - } - - if in $(requirements:G) - { - errors.user-error - "The property is not allowed for the 'install' rule" ; - } - - targets.main-target-alternative - [ new install-target-class $(name) : $(project) - : [ targets.main-target-sources $(sources) : $(name) ] - : [ targets.main-target-requirements $(requirements) : $(project) ] - : [ targets.main-target-default-build $(default-build) : $(project) ] - ] ; -} - - -IMPORT $(__name__) : install : : install ; -IMPORT $(__name__) : install : : stage ; diff --git a/jam-files/boost-build/tools/stage.py b/jam-files/boost-build/tools/stage.py deleted file mode 100644 index 25eccbe5..00000000 --- a/jam-files/boost-build/tools/stage.py +++ /dev/null @@ -1,350 +0,0 @@ -# Status: ported. -# Base revision 64444. -# -# Copyright 2003 Dave Abrahams -# Copyright 2005, 2006 Rene Rivera -# Copyright 2002, 2003, 2004, 2005, 2006, 2010 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module defines the 'install' rule, used to copy a set of targets to a -# single location. - -import b2.build.feature as feature -import b2.build.targets as targets -import b2.build.property as property -import b2.build.property_set as property_set -import b2.build.generators as generators -import b2.build.virtual_target as virtual_target - -from b2.manager import get_manager -from b2.util.sequence import unique -from b2.util import bjam_signature - -import b2.build.type - -import os.path -import re -import types - -feature.feature('install-dependencies', ['off', 'on'], ['incidental']) -feature.feature('install-type', [], ['free', 'incidental']) -feature.feature('install-source-root', [], ['free', 'path']) -feature.feature('so-version', [], ['free', 'incidental']) - -# If 'on', version symlinks for shared libraries will not be created. Affects -# Unix builds only. -feature.feature('install-no-version-symlinks', ['on'], ['optional', 'incidental']) - -class InstallTargetClass(targets.BasicTarget): - - def update_location(self, ps): - """If is not set, sets it based on the project data.""" - - loc = ps.get('location') - if not loc: - loc = os.path.join(self.project().get('location'), self.name()) - ps = ps.add_raw(["" + loc]) - - return ps - - def adjust_properties(self, target, build_ps): - a = target.action() - properties = [] - if a: - ps = a.properties() - properties = ps.all() - - # Unless true is in properties, which can happen - # only if the user has explicitly requested it, nuke all - # properties. - - if build_ps.get('hardcode-dll-paths') != ['true']: - properties = [p for p in properties if p.feature().name() != 'dll-path'] - - # If any properties were specified for installing, add - # them. - properties.extend(build_ps.get_properties('dll-path')) - - # Also copy feature from current build set, to be used - # for relinking. - properties.extend(build_ps.get_properties('linkflags')) - - # Remove the feature on original targets. - # And . If stage target has another stage target in - # sources, then we shall get virtual targets with the - # property set. - properties = [p for p in properties - if not p.feature().name() in ['tag', 'location']] - - properties.extend(build_ps.get_properties('dependency')) - - properties.extend(build_ps.get_properties('location')) - - - properties.extend(build_ps.get_properties('install-no-version-symlinks')) - - d = build_ps.get_properties('install-source-root') - - # Make the path absolute: we shall use it to compute relative paths and - # making the path absolute will help. - if d: - p = d[0] - properties.append(property.Property(p.feature(), os.path.abspath(p.value()))) - - return property_set.create(properties) - - - def construct(self, name, source_targets, ps): - - source_targets = self.targets_to_stage(source_targets, ps) - ps = self.update_location(ps) - - ename = ps.get('name') - if ename: - ename = ename[0] - if ename and len(source_targets) > 1: - get_manager().errors()("When property is used in 'install', only one source is allowed") - - result = [] - - for i in source_targets: - - staged_targets = [] - new_ps = self.adjust_properties(i, ps) - - # See if something special should be done when staging this type. It - # is indicated by the presence of a special "INSTALLED_" type. - t = i.type() - if t and b2.build.type.registered("INSTALLED_" + t): - - if ename: - get_manager().errors()("In 'install': property specified with target that requires relinking.") - else: - (r, targets) = generators.construct(self.project(), name, "INSTALLED_" + t, - new_ps, [i]) - assert isinstance(r, property_set.PropertySet) - staged_targets.extend(targets) - - else: - staged_targets.append(copy_file(self.project(), ename, i, new_ps)) - - if not staged_targets: - get_manager().errors()("Unable to generate staged version of " + i) - - result.extend(get_manager().virtual_targets().register(t) for t in staged_targets) - - return (property_set.empty(), result) - - def targets_to_stage(self, source_targets, ps): - """Given the list of source targets explicitly passed to 'stage', returns the - list of targets which must be staged.""" - - result = [] - - # Traverse the dependencies, if needed. - if ps.get('install-dependencies') == ['on']: - source_targets = self.collect_targets(source_targets) - - # Filter the target types, if needed. - included_types = ps.get('install-type') - for r in source_targets: - ty = r.type() - if ty: - # Do not stage searched libs. - if ty != "SEARCHED_LIB": - if included_types: - if self.include_type(ty, included_types): - result.append(r) - else: - result.append(r) - elif not included_types: - # Don't install typeless target if there is an explicit list of - # allowed types. - result.append(r) - - return result - - # CONSIDER: figure out why we can not use virtual-target.traverse here. - # - def collect_targets(self, targets): - - s = [t.creating_subvariant() for t in targets] - s = unique(s) - - result = set(targets) - for i in s: - i.all_referenced_targets(result) - - result2 = [] - for r in result: - if isinstance(r, property.Property): - - if r.feature().name() != 'use': - result2.append(r.value()) - else: - result2.append(r) - result2 = unique(result2) - return result2 - - # Returns true iff 'type' is subtype of some element of 'types-to-include'. - # - def include_type(self, type, types_to_include): - return any(b2.build.type.is_subtype(type, ti) for ti in types_to_include) - -# Creates a copy of target 'source'. The 'properties' object should have a -# property which specifies where the target must be placed. -# -def copy_file(project, name, source, ps): - - if not name: - name = source.name() - - relative = "" - - new_a = virtual_target.NonScanningAction([source], "common.copy", ps) - source_root = ps.get('install-source-root') - if source_root: - source_root = source_root[0] - # Get the real path of the target. We probably need to strip relative - # path from the target name at construction. - path = os.path.join(source.path(), os.path.dirname(name)) - # Make the path absolute. Otherwise, it would be hard to compute the - # relative path. The 'source-root' is already absolute, see the - # 'adjust-properties' method above. - path = os.path.abspath(path) - - relative = os.path.relpath(path, source_root) - - name = os.path.join(relative, os.path.basename(name)) - return virtual_target.FileTarget(name, source.type(), project, new_a, exact=True) - -def symlink(name, project, source, ps): - a = virtual_target.Action([source], "symlink.ln", ps) - return virtual_target.FileTarget(name, source.type(), project, a, exact=True) - -def relink_file(project, source, ps): - action = source.action() - cloned_action = virtual_target.clone_action(action, project, "", ps) - targets = cloned_action.targets() - # We relink only on Unix, where exe or shared lib is always a single file. - assert len(targets) == 1 - return targets[0] - - -# Declare installed version of the EXE type. Generator for this type will cause -# relinking to the new location. -b2.build.type.register('INSTALLED_EXE', [], 'EXE') - -class InstalledExeGenerator(generators.Generator): - - def __init__(self): - generators.Generator.__init__(self, "install-exe", False, ['EXE'], ['INSTALLED_EXE']) - - def run(self, project, name, ps, source): - - need_relink = False; - - if ps.get('os') in ['NT', 'CYGWIN'] or ps.get('target-os') in ['windows', 'cygwin']: - # Never relink - pass - else: - # See if the dll-path properties are not changed during - # install. If so, copy, don't relink. - need_relink = ps.get('dll-path') != source[0].action().properties().get('dll-path') - - if need_relink: - return [relink_file(project, source, ps)] - else: - return [copy_file(project, None, source[0], ps)] - -generators.register(InstalledExeGenerator()) - - -# Installing a shared link on Unix might cause a creation of versioned symbolic -# links. -b2.build.type.register('INSTALLED_SHARED_LIB', [], 'SHARED_LIB') - -class InstalledSharedLibGenerator(generators.Generator): - - def __init__(self): - generators.Generator.__init__(self, 'install-shared-lib', False, ['SHARED_LIB'], ['INSTALLED_SHARED_LIB']) - - def run(self, project, name, ps, source): - - source = source[0] - if ps.get('os') in ['NT', 'CYGWIN'] or ps.get('target-os') in ['windows', 'cygwin']: - copied = copy_file(project, None, source, ps) - return [get_manager().virtual_targets().register(copied)] - else: - a = source.action() - if not a: - # Non-derived file, just copy. - copied = copy_file(project, source, ps) - else: - - need_relink = ps.get('dll-path') != source.action().properties().get('dll-path') - - if need_relink: - # Rpath changed, need to relink. - copied = relink_file(project, source, ps) - else: - copied = copy_file(project, None, source, ps) - - result = [get_manager().virtual_targets().register(copied)] - # If the name is in the form NNN.XXX.YYY.ZZZ, where all 'X', 'Y' and - # 'Z' are numbers, we need to create NNN.XXX and NNN.XXX.YYY - # symbolic links. - m = re.match("(.*)\\.([0123456789]+)\\.([0123456789]+)\\.([0123456789]+)$", - copied.name()); - if m: - # Symlink without version at all is used to make - # -lsome_library work. - result.append(symlink(m.group(1), project, copied, ps)) - - # Symlinks of some libfoo.N and libfoo.N.M are used so that - # library can found at runtime, if libfoo.N.M.X has soname of - # libfoo.N. That happens when the library makes some binary - # compatibility guarantees. If not, it is possible to skip those - # symlinks. - if ps.get('install-no-version-symlinks') != ['on']: - - result.append(symlink(m.group(1) + '.' + m.group(2), project, copied, ps)) - result.append(symlink(m.group(1) + '.' + m.group(2) + '.' + m.group(3), - project, copied, ps)) - - return result - -generators.register(InstalledSharedLibGenerator()) - - -# Main target rule for 'install'. -# -@bjam_signature((["name"], ["sources", "*"], ["requirements", "*"], - ["default_build", "*"], ["usage_requirements", "*"])) -def install(name, sources, requirements=[], default_build=[], usage_requirements=[]): - - requirements = requirements[:] - # Unless the user has explicitly asked us to hardcode dll paths, add - # false in requirements, to override default value. - if not 'true' in requirements: - requirements.append('false') - - if any(r.startswith('') for r in requirements): - get_manager().errors()("The property is not allowed for the 'install' rule") - - from b2.manager import get_manager - t = get_manager().targets() - - project = get_manager().projects().current() - - return t.main_target_alternative( - InstallTargetClass(name, project, - t.main_target_sources(sources, name), - t.main_target_requirements(requirements, project), - t.main_target_default_build(default_build, project), - t.main_target_usage_requirements(usage_requirements, project))) - -get_manager().projects().add_rule("install", install) -get_manager().projects().add_rule("stage", install) - diff --git a/jam-files/boost-build/tools/stlport.jam b/jam-files/boost-build/tools/stlport.jam deleted file mode 100644 index 62eebda5..00000000 --- a/jam-files/boost-build/tools/stlport.jam +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright Gennadiy Rozental -# Copyright 2006 Rene Rivera -# Copyright 2003, 2004, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# The STLPort is usable by means of 'stdlib' feature. When -# stdlib=stlport is specified, default version of STLPort will be used, -# while stdlib=stlport-4.5 will use specific version. -# The subfeature value 'hostios' means to use host compiler's iostreams. -# -# The specific version of stlport is selected by features: -# The feature selects between static and shared library -# The on selects STLPort with debug symbols -# and stl debugging. -# There's no way to use STLPort with debug symbols but without -# stl debugging. - -# TODO: must implement selection of different STLPort installations based -# on used toolset. -# Also, finish various flags: -# -# This is copied from V1 toolset, "+" means "implemented" -#+flags $(CURR_TOOLSET) DEFINES off : _STLP_NO_OWN_IOSTREAMS=1 _STLP_HAS_NO_NEW_IOSTREAMS=1 ; -#+flags $(CURR_TOOLSET) DEFINES off : _STLP_NO_EXTENSIONS=1 ; -# flags $(CURR_TOOLSET) DEFINES off : _STLP_NO_ANACHRONISMS=1 ; -# flags $(CURR_TOOLSET) DEFINES global : _STLP_VENDOR_GLOBAL_CSTD=1 ; -# flags $(CURR_TOOLSET) DEFINES off : _STLP_NO_EXCEPTIONS=1 ; -# flags $(CURR_TOOLSET) DEFINES on : _STLP_DEBUG_ALLOC=1 ; -#+flags $(CURR_TOOLSET) DEFINES debug : _STLP_DEBUG=1 _STLP_DEBUG_UNINITIALIZED=1 ; -#+flags $(CURR_TOOLSET) DEFINES dynamic : _STLP_USE_DYNAMIC_LIB=1 ; - - -import feature : feature subfeature ; -import project ; -import "class" : new ; -import targets ; -import property-set ; -import common ; -import type ; - -# Make this module into a project. -project.initialize $(__name__) ; -project stlport ; - -# The problem: how to request to use host compiler's iostreams? -# -# Solution 1: Global 'stlport-iostream' feature. -# That's ugly. Subfeature make more sense for stlport-specific thing. -# Solution 2: Use subfeature with two values, one of which ("use STLPort iostream") -# is default. -# The problem is that such subfeature will appear in target paths, and that's ugly -# Solution 3: Use optional subfeature with only one value. - -feature.extend stdlib : stlport ; -feature.compose stlport : /stlport//stlport ; - -# STLport iostreams or native iostreams -subfeature stdlib stlport : iostream : hostios : optional propagated ; - -# STLport extensions -subfeature stdlib stlport : extensions : noext : optional propagated ; - -# STLport anachronisms -- NOT YET SUPPORTED -# subfeature stdlib stlport : anachronisms : on off ; - -# STLport debug allocation -- NOT YET SUPPORTED -#subfeature stdlib stlport : debug-alloc : off on ; - -# Declare a special target class to handle the creation of search-lib-target -# instances for STLport. We need a special class, because otherwise we'll have -# - declare prebuilt targets for all possible toolsets. And by the time 'init' -# is called we don't even know the list of toolsets that are registered -# - when host iostreams are used, we really should produce nothing. It would -# be hard/impossible to achieve this using prebuilt targets. - -class stlport-target-class : basic-target -{ - import feature project type errors generators ; - import set : difference ; - - rule __init__ ( project : headers ? : libraries * : version ? ) - { - basic-target.__init__ stlport : $(project) ; - self.headers = $(headers) ; - self.libraries = $(libraries) ; - self.version = $(version) ; - self.version.5 = [ MATCH "^(5[.][0123456789]+).*" : $(version) ] ; - - local requirements ; - requirements += $(self.version) ; - self.requirements = [ property-set.create $(requirements) ] ; - } - - rule generate ( property-set ) - { - # Since this target is built with stlport, it will also - # have /stlport//stlport in requirements, which will - # cause a loop in main target references. Remove that property - # manually. - - property-set = [ property-set.create - [ difference - [ $(property-set).raw ] : - /stlport//stlport - stlport - ] - ] ; - return [ basic-target.generate $(property-set) ] ; - } - - rule construct ( name : source-targets * : property-set ) - { - # Deduce the name of stlport library, based on toolset and - # debug setting. - local raw = [ $(property-set).raw ] ; - local hostios = [ feature.get-values : $(raw) ] ; - local toolset = [ feature.get-values : $(raw) ] ; - - if $(self.version.5) - { - # Version 5.x - - # STLport host IO streams no longer supported. So we always - # need libraries. - - # name: stlport(stl)?[dg]?(_static)?.M.R - local name = stlport ; - if [ feature.get-values : $(raw) ] = "on" - { - name += stl ; - switch $(toolset) - { - case gcc* : name += g ; - case darwin* : name += g ; - case * : name += d ; - } - } - - if [ feature.get-values : $(raw) ] = "static" - { - name += _static ; - } - - # Starting with version 5.2.0, the STLport static libraries no longer - # include a version number in their name - local version.pre.5.2 = [ MATCH "^(5[.][01]+).*" : $(version) ] ; - if $(version.pre.5.2) || [ feature.get-values : $(raw) ] != "static" - { - name += .$(self.version.5) ; - } - - name = $(name:J=) ; - - if [ feature.get-values : $(raw) ] = "on" - { - #~ Allow explicitly asking to install the STLport lib by - #~ refering to it directly: /stlport//stlport/on - #~ This allows for install packaging of all libs one might need for - #~ a standalone distribution. - import path : make : path-make ; - local runtime-link - = [ feature.get-values : $(raw) ] ; - local lib-file.props - = [ property-set.create $(raw) $(runtime-link) ] ; - local lib-file.prefix - = [ type.generated-target-prefix $(runtime-link:U)_LIB : $(lib-file.props) ] ; - local lib-file.suffix - = [ type.generated-target-suffix $(runtime-link:U)_LIB : $(lib-file.props) ] ; - lib-file.prefix - ?= "" "lib" ; - lib-file.suffix - ?= "" ; - local lib-file - = [ GLOB $(self.libraries) [ modules.peek : PATH ] : - $(lib-file.prefix)$(name).$(lib-file.suffix) ] ; - lib-file - = [ new file-reference [ path-make $(lib-file[1]) ] : $(self.project) ] ; - lib-file - = [ $(lib-file).generate "" ] ; - local lib-file.requirements - = [ targets.main-target-requirements - [ $(lib-file.props).raw ] $(lib-file[-1]) - : $(self.project) ] ; - return [ generators.construct $(self.project) $(name) : LIB : $(lib-file.requirements) ] ; - } - else - { - #~ Otherwise, it's just a regular usage of the library. - return [ generators.construct - $(self.project) $(name) : SEARCHED_LIB : $(property-set) ] ; - } - } - else if ! $(hostios) && $(toolset) != msvc - { - # We don't need libraries if host istreams are used. For - # msvc, automatic library selection will be used. - - # name: stlport_(_stldebug)? - local name = stlport ; - name = $(name)_$(toolset) ; - if [ feature.get-values : $(raw) ] = "on" - { - name = $(name)_stldebug ; - } - - return [ generators.construct - $(self.project) $(name) : SEARCHED_LIB : $(property-set) ] ; - } - else - { - return [ property-set.empty ] ; - } - } - - rule compute-usage-requirements ( subvariant ) - { - local usage-requirements = - $(self.headers) - $(self.libraries) - $(self.libraries) - ; - - local rproperties = [ $(subvariant).build-properties ] ; - # CONSIDER: should this "if" sequence be replaced with - # some use of 'property-map' class? - if [ $(rproperties).get ] = "on" - { - usage-requirements += - _STLP_DEBUG=1 - _STLP_DEBUG_UNINITIALIZED=1 ; - } - if [ $(rproperties).get ] = "shared" - { - usage-requirements += - _STLP_USE_DYNAMIC_LIB=1 ; - } - if [ $(rproperties).get ] = noext - { - usage-requirements += - _STLP_NO_EXTENSIONS=1 ; - } - if [ $(rproperties).get ] = hostios - { - usage-requirements += - _STLP_NO_OWN_IOSTREAMS=1 - _STLP_HAS_NO_NEW_IOSTREAMS=1 ; - } - if $(self.version.5) - { - # Version 5.x - if [ $(rproperties).get ] = "single" - { - # Since STLport5 doesn't normally support single-thread - # we force STLport5 into the multi-thread mode. Hence - # getting what other libs provide of single-thread code - # linking against a multi-thread lib. - usage-requirements += - _STLP_THREADS=1 ; - } - } - - return [ property-set.create $(usage-requirements) ] ; - } -} - -rule stlport-target ( headers ? : libraries * : version ? ) -{ - local project = [ project.current ] ; - - targets.main-target-alternative - [ new stlport-target-class $(project) : $(headers) : $(libraries) - : $(version) - ] ; -} - -local .version-subfeature-defined ; - -# Initialize stlport support. -rule init ( - version ? : - headers : # Location of header files - libraries * # Location of libraries, lib and bin subdirs of STLport. - ) -{ - # FIXME: need to use common.check-init-parameters here. - # At the moment, that rule always tries to define subfeature - # of the 'toolset' feature, while we need to define subfeature - # of stlport, so tweaks to check-init-parameters are needed. - if $(version) - { - if ! $(.version-subfeature-defined) - { - feature.subfeature stdlib stlport : version : : propagated ; - .version-subfeature-defined = true ; - } - feature.extend-subfeature stdlib stlport : version : $(version) ; - } - - # Declare the main target for this STLPort version. - stlport-target $(headers) : $(libraries) : $(version) ; -} - diff --git a/jam-files/boost-build/tools/sun.jam b/jam-files/boost-build/tools/sun.jam deleted file mode 100644 index 0ca927d3..00000000 --- a/jam-files/boost-build/tools/sun.jam +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) Christopher Currie 2003. Permission to copy, use, -# modify, sell and distribute this software is granted provided this -# copyright notice appears in all copies. This software is provided -# "as is" without express or implied warranty, and with no claim as -# to its suitability for any purpose. - -import property ; -import generators ; -import os ; -import toolset : flags ; -import feature ; -import type ; -import common ; - -feature.extend toolset : sun ; -toolset.inherit sun : unix ; -generators.override sun.prebuilt : builtin.lib-generator ; -generators.override sun.prebuilt : builtin.prebuilt ; -generators.override sun.searched-lib-generator : searched-lib-generator ; - -feature.extend stdlib : sun-stlport ; -feature.compose sun-stlport - : -library=stlport4 -library=stlport4 - ; - -rule init ( version ? : command * : options * ) -{ - local condition = [ - common.check-init-parameters sun : version $(version) ] ; - - command = [ common.get-invocation-command sun : CC - : $(command) : "/opt/SUNWspro/bin" ] ; - - # Even if the real compiler is not found, put CC to - # command line so that user see command line that would have being executed. - command ?= CC ; - - common.handle-options sun : $(condition) : $(command) : $(options) ; - - command_c = $(command[1--2]) $(command[-1]:B=cc) ; - - toolset.flags sun CONFIG_C_COMMAND $(condition) : $(command_c) ; -} - -# Declare generators -generators.register-c-compiler sun.compile.c : C : OBJ : sun ; -generators.register-c-compiler sun.compile.c++ : CPP : OBJ : sun ; - -# Declare flags and actions for compilation -flags sun.compile OPTIONS on : -g ; -flags sun.compile OPTIONS on : -xprofile=tcov ; -flags sun.compile OPTIONS speed : -xO4 ; -flags sun.compile OPTIONS space : -xO2 -xspace ; -flags sun.compile OPTIONS multi : -mt ; -flags sun.compile OPTIONS off : -erroff ; -flags sun.compile OPTIONS on : -erroff=%none ; -flags sun.compile OPTIONS all : -erroff=%none ; -flags sun.compile OPTIONS on : -errwarn ; - -flags sun.compile.c++ OPTIONS off : +d ; - -# The -m32 and -m64 options are supported starting -# with Sun Studio 12. On earlier compilers, the -# 'address-model' feature is not supported and should not -# be used. Instead, use -xarch=generic64 command line -# option. -# See http://svn.boost.org/trac/boost/ticket/1186 -# for details. -flags sun OPTIONS 32 : -m32 ; -flags sun OPTIONS 64 : -m64 ; -# On sparc, there's a difference between -Kpic -# and -KPIC. The first is slightly more efficient, -# but has the limits on the size of GOT table. -# For minimal fuss on user side, we use -KPIC here. -# See http://svn.boost.org/trac/boost/ticket/1186#comment:6 -# for detailed explanation. -flags sun OPTIONS shared : -KPIC ; - -flags sun.compile OPTIONS ; -flags sun.compile.c++ OPTIONS ; -flags sun.compile DEFINES ; -flags sun.compile INCLUDES ; - -actions compile.c -{ - "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -actions compile.c++ -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)" -} - -# Declare flags and actions for linking -flags sun.link OPTIONS on : -g ; -# Strip the binary when no debugging is needed -flags sun.link OPTIONS off : -s ; -flags sun.link OPTIONS on : -xprofile=tcov ; -flags sun.link OPTIONS multi : -mt ; -flags sun.link OPTIONS ; -flags sun.link LINKPATH ; -flags sun.link FINDLIBS-ST ; -flags sun.link FINDLIBS-SA ; -flags sun.link LIBRARIES ; -flags sun.link LINK-RUNTIME static : static ; -flags sun.link LINK-RUNTIME shared : dynamic ; -flags sun.link RPATH ; -# On gcc, there are separate options for dll path at runtime and -# link time. On Solaris, there's only one: -R, so we have to use -# it, even though it's bad idea. -flags sun.link RPATH ; - -# The POSIX real-time library is always needed (nanosleep, clock_gettime etc.) -flags sun.link FINDLIBS-SA : rt ; - -rule link ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -} - -# Slight mods for dlls -rule link.dll ( targets * : sources * : properties * ) -{ - SPACE on $(targets) = " " ; -} - -actions link.dll bind LIBRARIES -{ - "$(CONFIG_COMMAND)" $(OPTIONS) -L"$(LINKPATH)" -R"$(RPATH)" -o "$(<)" -h$(<[1]:D=) -G "$(>)" "$(LIBRARIES)" -Bdynamic -l$(FINDLIBS-SA) -Bstatic -l$(FINDLIBS-ST) -B$(LINK-RUNTIME) -} - -# Declare action for creating static libraries -actions piecemeal archive -{ - "$(CONFIG_COMMAND)" -xar -o "$(<)" "$(>)" -} - diff --git a/jam-files/boost-build/tools/symlink.jam b/jam-files/boost-build/tools/symlink.jam deleted file mode 100644 index b33e8260..00000000 --- a/jam-files/boost-build/tools/symlink.jam +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2003 Dave Abrahams -# Copyright 2002, 2003 Rene Rivera -# Copyright 2002, 2003, 2004, 2005 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Defines the "symlink" special target. 'symlink' targets make symbolic links -# to the sources. - -import targets modules path class os feature project property-set ; - -.count = 0 ; - -feature.feature symlink-location : project-relative build-relative : incidental ; - -# The class representing "symlink" targets. -# -class symlink-targets : basic-target -{ - import numbers modules class property project path ; - - rule __init__ ( - project - : targets * - : sources * - ) - { - # Generate a fake name for now. Need unnamed targets eventually. - local c = [ modules.peek symlink : .count ] ; - modules.poke symlink : .count : [ numbers.increment $(c) ] ; - local fake-name = symlink#$(c) ; - - basic-target.__init__ $(fake-name) : $(project) : $(sources) ; - - # Remember the targets to map the sources onto. Pad or truncate - # to fit the sources given. - self.targets = ; - for local source in $(sources) - { - if $(targets) - { - self.targets += $(targets[1]) ; - targets = $(targets[2-]) ; - } - else - { - self.targets += $(source) ; - } - } - - # The virtual targets corresponding to the given targets. - self.virtual-targets = ; - } - - rule construct ( name : source-targets * : property-set ) - { - local i = 1 ; - for local t in $(source-targets) - { - local s = $(self.targets[$(i)]) ; - local a = [ class.new action $(t) : symlink.ln : $(property-set) ] ; - local vt = [ class.new file-target $(s:D=) - : [ $(t).type ] : $(self.project) : $(a) ] ; - - # Place the symlink in the directory relative to the project - # location, instead of placing it in the build directory. - if [ property.select : [ $(property-set).raw ] ] = project-relative - { - $(vt).set-path [ path.root $(s:D) [ $(self.project).get location ] ] ; - } - - self.virtual-targets += $(vt) ; - i = [ numbers.increment $(i) ] ; - } - return [ property-set.empty ] $(self.virtual-targets) ; - } -} - -# Creates a symbolic link from a set of targets to a set of sources. -# The targets and sources map one to one. The symlinks generated are -# limited to be the ones given as the sources. That is, the targets -# are either padded or trimmed to equate to the sources. The padding -# is done with the name of the corresponding source. For example:: -# -# symlink : one two ; -# -# Is equal to:: -# -# symlink one two : one two ; -# -# Names for symlink are relative to the project location. They cannot -# include ".." path components. -rule symlink ( - targets * - : sources * - ) -{ - local project = [ project.current ] ; - - return [ targets.main-target-alternative - [ class.new symlink-targets $(project) : $(targets) : - # Note: inline targets are not supported for symlink, intentionally, - # since it's used to linking existing non-local targets. - $(sources) ] ] ; -} - -rule ln -{ - local os ; - if [ modules.peek : UNIX ] { os = UNIX ; } - else { os ?= [ os.name ] ; } - # Remember the path to make the link relative to where the symlink is located. - local path-to-source = [ path.relative-to - [ path.make [ on $(<) return $(LOCATE) ] ] - [ path.make [ on $(>) return $(LOCATE) ] ] ] ; - if $(path-to-source) = . - { - PATH_TO_SOURCE on $(<) = "" ; - } - else - { - PATH_TO_SOURCE on $(<) = [ path.native $(path-to-source) ] ; - } - ln-$(os) $(<) : $(>) ; -} - -actions ln-UNIX -{ - ln -f -s '$(>:D=:R=$(PATH_TO_SOURCE))' '$(<)' -} - -# there is a way to do this; we fall back to a copy for now -actions ln-NT -{ - echo "NT symlinks not supported yet, making copy" - del /f /q "$(<)" 2>nul >nul - copy "$(>)" "$(<)" $(NULL_OUT) -} - -IMPORT $(__name__) : symlink : : symlink ; diff --git a/jam-files/boost-build/tools/symlink.py b/jam-files/boost-build/tools/symlink.py deleted file mode 100644 index 6345ded6..00000000 --- a/jam-files/boost-build/tools/symlink.py +++ /dev/null @@ -1,112 +0,0 @@ -# Status: ported. -# Base revision: 64488. - -# Copyright 2003 Dave Abrahams -# Copyright 2002, 2003 Rene Rivera -# Copyright 2002, 2003, 2004, 2005 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Defines the "symlink" special target. 'symlink' targets make symbolic links -# to the sources. - -import b2.build.feature as feature -import b2.build.targets as targets -import b2.build.property_set as property_set -import b2.build.virtual_target as virtual_target -import b2.build.targets - -from b2.manager import get_manager - -import bjam - -import os - - -feature.feature("symlink-location", ["project-relative", "build-relative"], ["incidental"]) - -class SymlinkTarget(targets.BasicTarget): - - _count = 0 - - def __init__(self, project, targets, sources): - - # Generate a fake name for now. Need unnamed targets eventually. - fake_name = "symlink#%s" % SymlinkTarget._count - SymlinkTarget._count = SymlinkTarget._count + 1 - - b2.build.targets.BasicTarget.__init__(self, fake_name, project, sources) - - # Remember the targets to map the sources onto. Pad or truncate - # to fit the sources given. - assert len(targets) <= len(sources) - self.targets = targets[:] + sources[len(targets):] - - # The virtual targets corresponding to the given targets. - self.virtual_targets = [] - - def construct(self, name, source_targets, ps): - i = 0 - for t in source_targets: - s = self.targets[i] - a = virtual_target.Action(self.manager(), [t], "symlink.ln", ps) - vt = virtual_target.FileTarget(os.path.basename(s), t.type(), self.project(), a) - - # Place the symlink in the directory relative to the project - # location, instead of placing it in the build directory. - if not ps.get('symlink-location') == "project-relative": - vt.set_path(os.path.join(self.project().get('location'), os.path.dirname(s))) - - vt = get_manager().virtual_targets().register(vt) - self.virtual_targets.append(vt) - i = i + 1 - - return (property_set.empty(), self.virtual_targets) - -# Creates a symbolic link from a set of targets to a set of sources. -# The targets and sources map one to one. The symlinks generated are -# limited to be the ones given as the sources. That is, the targets -# are either padded or trimmed to equate to the sources. The padding -# is done with the name of the corresponding source. For example:: -# -# symlink : one two ; -# -# Is equal to:: -# -# symlink one two : one two ; -# -# Names for symlink are relative to the project location. They cannot -# include ".." path components. -def symlink(targets, sources): - - from b2.manager import get_manager - t = get_manager().targets() - p = get_manager().projects().current() - - return t.main_target_alternative( - SymlinkTarget(p, targets, - # Note: inline targets are not supported for symlink, intentionally, - # since it's used to linking existing non-local targets. - sources)) - - -def setup_ln(targets, sources, ps): - - source_path = bjam.call("get-target-variable", sources[0], "LOCATE")[0] - target_path = bjam.call("get-target-variable", targets[0], "LOCATE")[0] - rel = os.path.relpath(source_path, target_path) - if rel == ".": - bjam.call("set-target-variable", targets, "PATH_TO_SOURCE", "") - else: - bjam.call("set-target-variable", targets, "PATH_TO_SOURCE", rel) - -if os.name == 'nt': - ln_action = """echo "NT symlinks not supported yet, making copy" -del /f /q "$(<)" 2>nul >nul -copy "$(>)" "$(<)" $(NULL_OUT)""" -else: - ln_action = "ln -f -s '$(>:D=:R=$(PATH_TO_SOURCE))' '$(<)'" - -get_manager().engine().register_action("symlink.ln", ln_action, function=setup_ln) - -get_manager().projects().add_rule("symlink", symlink) diff --git a/jam-files/boost-build/tools/testing-aux.jam b/jam-files/boost-build/tools/testing-aux.jam deleted file mode 100644 index 525dafd0..00000000 --- a/jam-files/boost-build/tools/testing-aux.jam +++ /dev/null @@ -1,210 +0,0 @@ -# This module is imported by testing.py. The definitions here are -# too tricky to do in Python - -# Causes the 'target' to exist after bjam invocation if and only if all the -# dependencies were successfully built. -# -rule expect-success ( target : dependency + : requirements * ) -{ - **passed** $(target) : $(sources) ; -} -IMPORT testing : expect-success : : testing.expect-success ; - -# Causes the 'target' to exist after bjam invocation if and only if all some of -# the dependencies were not successfully built. -# -rule expect-failure ( target : dependency + : properties * ) -{ - local grist = [ MATCH ^<(.*)> : $(dependency:G) ] ; - local marker = $(dependency:G=$(grist)*fail) ; - (failed-as-expected) $(marker) ; - FAIL_EXPECTED $(dependency) ; - LOCATE on $(marker) = [ on $(dependency) return $(LOCATE) ] ; - RMOLD $(marker) ; - DEPENDS $(marker) : $(dependency) ; - DEPENDS $(target) : $(marker) ; - **passed** $(target) : $(marker) ; -} -IMPORT testing : expect-failure : : testing.expect-failure ; - -# The rule/action combination used to report successful passing of a test. -# -rule **passed** -{ - # Force deletion of the target, in case any dependencies failed to build. - RMOLD $(<) ; -} - - -# Used to create test files signifying passed tests. -# -actions **passed** -{ - echo passed > "$(<)" -} - - -# Used to create replacement object files that do not get created during tests -# that are expected to fail. -# -actions (failed-as-expected) -{ - echo failed as expected > "$(<)" -} - -# Runs executable 'sources' and stores stdout in file 'target'. Unless -# --preserve-test-targets command line option has been specified, removes the -# executable. The 'target-to-remove' parameter controls what should be removed: -# - if 'none', does not remove anything, ever -# - if empty, removes 'source' -# - if non-empty and not 'none', contains a list of sources to remove. -# -rule capture-output ( target : source : properties * : targets-to-remove * ) -{ - output-file on $(target) = $(target:S=.output) ; - LOCATE on $(target:S=.output) = [ on $(target) return $(LOCATE) ] ; - - # The INCLUDES kill a warning about independent target... - INCLUDES $(target) : $(target:S=.output) ; - # but it also puts .output into dependency graph, so we must tell jam it is - # OK if it cannot find the target or updating rule. - NOCARE $(target:S=.output) ; - - # This has two-fold effect. First it adds input files to the dependendency - # graph, preventing a warning. Second, it causes input files to be bound - # before target is created. Therefore, they are bound using SEARCH setting - # on them and not LOCATE setting of $(target), as in other case (due to jam - # bug). - DEPENDS $(target) : [ on $(target) return $(INPUT_FILES) ] ; - - if $(targets-to-remove) = none - { - targets-to-remove = ; - } - else if ! $(targets-to-remove) - { - targets-to-remove = $(source) ; - } - - if [ on $(target) return $(REMOVE_TEST_TARGETS) ] - { - TEMPORARY $(targets-to-remove) ; - # Set a second action on target that will be executed after capture - # output action. The 'RmTemps' rule has the 'ignore' modifier so it is - # always considered succeeded. This is needed for 'run-fail' test. For - # that test the target will be marked with FAIL_EXPECTED, and without - # 'ignore' successful execution will be negated and be reported as - # failure. With 'ignore' we do not detect a case where removing files - # fails, but it is not likely to happen. - RmTemps $(target) : $(targets-to-remove) ; - } -} - - -if [ os.name ] = NT -{ - .STATUS = %status% ; - .SET_STATUS = "set status=%ERRORLEVEL%" ; - .RUN_OUTPUT_NL = "echo." ; - .STATUS_0 = "%status% EQU 0 (" ; - .STATUS_NOT_0 = "%status% NEQ 0 (" ; - .VERBOSE = "%verbose% EQU 1 (" ; - .ENDIF = ")" ; - .SHELL_SET = "set " ; - .CATENATE = type ; - .CP = copy ; -} -else -{ - .STATUS = "$status" ; - .SET_STATUS = "status=$?" ; - .RUN_OUTPUT_NL = "echo" ; - .STATUS_0 = "test $status -eq 0 ; then" ; - .STATUS_NOT_0 = "test $status -ne 0 ; then" ; - .VERBOSE = "test $verbose -eq 1 ; then" ; - .ENDIF = "fi" ; - .SHELL_SET = "" ; - .CATENATE = cat ; - .CP = cp ; -} - - -.VERBOSE_TEST = 0 ; -if --verbose-test in [ modules.peek : ARGV ] -{ - .VERBOSE_TEST = 1 ; -} - - -.RM = [ common.rm-command ] ; - - -actions capture-output bind INPUT_FILES output-file -{ - $(PATH_SETUP) - $(LAUNCHER) "$(>)" $(ARGS) "$(INPUT_FILES)" > "$(output-file)" 2>&1 - $(.SET_STATUS) - $(.RUN_OUTPUT_NL) >> "$(output-file)" - echo EXIT STATUS: $(.STATUS) >> "$(output-file)" - if $(.STATUS_0) - $(.CP) "$(output-file)" "$(<)" - $(.ENDIF) - $(.SHELL_SET)verbose=$(.VERBOSE_TEST) - if $(.STATUS_NOT_0) - $(.SHELL_SET)verbose=1 - $(.ENDIF) - if $(.VERBOSE) - echo ====== BEGIN OUTPUT ====== - $(.CATENATE) "$(output-file)" - echo ====== END OUTPUT ====== - $(.ENDIF) - exit $(.STATUS) -} - -IMPORT testing : capture-output : : testing.capture-output ; - - -actions quietly updated ignore piecemeal together RmTemps -{ - $(.RM) "$(>)" -} - - -.MAKE_FILE = [ common.file-creation-command ] ; - -actions unit-test -{ - $(PATH_SETUP) - $(LAUNCHER) $(>) $(ARGS) && $(.MAKE_FILE) $(<) -} - -rule record-time ( target : source : start end user system ) -{ - local src-string = [$(source:G=:J=",")"] " ; - USER_TIME on $(target) += $(src-string)$(user) ; - SYSTEM_TIME on $(target) += $(src-string)$(system) ; -} - -# Calling this rule requests that Boost Build time how long it taks to build the -# 'source' target and display the results both on the standard output and in the -# 'target' file. -# -rule time ( target : source : properties * ) -{ - # Set up rule for recording timing information. - __TIMING_RULE__ on $(source) = testing.record-time $(target) ; - - # Make sure that the source is rebuilt any time we need to retrieve that - # information. - REBUILDS $(target) : $(source) ; -} - - -actions time -{ - echo user: $(USER_TIME) - echo system: $(SYSTEM_TIME) - - echo user: $(USER_TIME)" seconds" > "$(<)" - echo system: $(SYSTEM_TIME)" seconds" >> "$(<)" -} diff --git a/jam-files/boost-build/tools/testing.jam b/jam-files/boost-build/tools/testing.jam deleted file mode 100644 index c42075b7..00000000 --- a/jam-files/boost-build/tools/testing.jam +++ /dev/null @@ -1,581 +0,0 @@ -# Copyright 2005 Dave Abrahams -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module implements regression testing framework. It declares a number of -# main target rules which perform some action and, if the results are OK, -# creates an output file. -# -# The exact list of rules is: -# 'compile' -- creates .test file if compilation of sources was -# successful. -# 'compile-fail' -- creates .test file if compilation of sources failed. -# 'run' -- creates .test file is running of executable produced from -# sources was successful. Also leaves behind .output file -# with the output from program run. -# 'run-fail' -- same as above, but .test file is created if running fails. -# -# In all cases, presence of .test file is an indication that the test passed. -# For more convenient reporting, you might want to use C++ Boost regression -# testing utilities (see http://www.boost.org/more/regression.html). -# -# For historical reason, a 'unit-test' rule is available which has the same -# syntax as 'exe' and behaves just like 'run'. - -# Things to do: -# - Teach compiler_status handle Jamfile.v2. -# Notes: -# - is not implemented, since it is Como-specific, and it is not -# clear how to implement it -# - std::locale-support is not implemented (it is used in one test). - - -import alias ; -import "class" ; -import common ; -import errors ; -import feature ; -import generators ; -import os ; -import path ; -import project ; -import property ; -import property-set ; -import regex ; -import sequence ; -import targets ; -import toolset ; -import type ; -import virtual-target ; - - -rule init ( ) -{ -} - - -# Feature controling the command used to lanch test programs. -feature.feature testing.launcher : : free optional ; - -feature.feature test-info : : free incidental ; -feature.feature testing.arg : : free incidental ; -feature.feature testing.input-file : : free dependency ; - -feature.feature preserve-test-targets : on off : incidental propagated ; - -# Register target types. -type.register TEST : test ; -type.register COMPILE : : TEST ; -type.register COMPILE_FAIL : : TEST ; -type.register RUN_OUTPUT : run ; -type.register RUN : : TEST ; -type.register RUN_FAIL : : TEST ; -type.register LINK_FAIL : : TEST ; -type.register LINK : : TEST ; -type.register UNIT_TEST : passed : TEST ; - - -# Declare the rules which create main targets. While the 'type' module already -# creates rules with the same names for us, we need extra convenience: default -# name of main target, so write our own versions. - -# Helper rule. Create a test target, using basename of first source if no target -# name is explicitly passed. Remembers the created target in a global variable. -# -rule make-test ( target-type : sources + : requirements * : target-name ? ) -{ - target-name ?= $(sources[1]:D=:S=) ; - - # Having periods (".") in the target name is problematic because the typed - # generator will strip the suffix and use the bare name for the file - # targets. Even though the location-prefix averts problems most times it - # does not prevent ambiguity issues when referring to the test targets. For - # example when using the XML log output. So we rename the target to remove - # the periods, and provide an alias for users. - local real-name = [ regex.replace $(target-name) "[.]" "~" ] ; - - local project = [ project.current ] ; - # The forces the build system for generate paths in the - # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow - # post-processing tools to work. - local t = [ targets.create-typed-target [ type.type-from-rule-name - $(target-type) ] : $(project) : $(real-name) : $(sources) : - $(requirements) $(real-name).test ] ; - - # The alias to the real target, per period replacement above. - if $(real-name) != $(target-name) - { - alias $(target-name) : $(t) ; - } - - # Remember the test (for --dump-tests). A good way would be to collect all - # given a project. This has some technical problems: e.g. we can not call - # this dump from a Jamfile since projects referred by 'build-project' are - # not available until the whole Jamfile has been loaded. - .all-tests += $(t) ; - return $(t) ; -} - - -# Note: passing more that one cpp file here is known to fail. Passing a cpp file -# and a library target works. -# -rule compile ( sources + : requirements * : target-name ? ) -{ - return [ make-test compile : $(sources) : $(requirements) : $(target-name) ] - ; -} - - -rule compile-fail ( sources + : requirements * : target-name ? ) -{ - return [ make-test compile-fail : $(sources) : $(requirements) : - $(target-name) ] ; -} - - -rule link ( sources + : requirements * : target-name ? ) -{ - return [ make-test link : $(sources) : $(requirements) : $(target-name) ] ; -} - - -rule link-fail ( sources + : requirements * : target-name ? ) -{ - return [ make-test link-fail : $(sources) : $(requirements) : $(target-name) - ] ; -} - - -rule handle-input-files ( input-files * ) -{ - if $(input-files[2]) - { - # Check that sorting made when creating property-set instance will not - # change the ordering. - if [ sequence.insertion-sort $(input-files) ] != $(input-files) - { - errors.user-error "Names of input files must be sorted alphabetically" - : "due to internal limitations" ; - } - } - return $(input-files) ; -} - - -rule run ( sources + : args * : input-files * : requirements * : target-name ? : - default-build * ) -{ - requirements += $(args:J=" ") ; - requirements += [ handle-input-files $(input-files) ] ; - return [ make-test run : $(sources) : $(requirements) : $(target-name) ] ; -} - - -rule run-fail ( sources + : args * : input-files * : requirements * : - target-name ? : default-build * ) -{ - requirements += $(args:J=" ") ; - requirements += [ handle-input-files $(input-files) ] ; - return [ make-test run-fail : $(sources) : $(requirements) : $(target-name) - ] ; -} - - -# Use 'test-suite' as a synonym for 'alias', for backward compatibility. -IMPORT : alias : : test-suite ; - - -# For all main targets in 'project-module', which are typed targets with type -# derived from 'TEST', produce some interesting information. -# -rule dump-tests -{ - for local t in $(.all-tests) - { - dump-test $(t) ; - } -} - - -# Given a project location in normalized form (slashes are forward), compute the -# name of the Boost library. -# -local rule get-library-name ( path ) -{ - # Path is in normalized form, so all slashes are forward. - local match1 = [ MATCH /(tools|libs)/(.*)/(test|example) : $(path) ] ; - local match2 = [ MATCH /(tools|libs)/(.*)$ : $(path) ] ; - local match3 = [ MATCH (/status$) : $(path) ] ; - - if $(match1) { return $(match1[2]) ; } - else if $(match2) { return $(match2[2]) ; } - else if $(match3) { return "" ; } - else if --dump-tests in [ modules.peek : ARGV ] - { - # The 'run' rule and others might be used outside boost. In that case, - # just return the path, since the 'library name' makes no sense. - return $(path) ; - } -} - - -# Was an XML dump requested? -.out-xml = [ MATCH --out-xml=(.*) : [ modules.peek : ARGV ] ] ; - - -# Takes a target (instance of 'basic-target') and prints -# - its type -# - its name -# - comments specified via the property -# - relative location of all source from the project root. -# -rule dump-test ( target ) -{ - local type = [ $(target).type ] ; - local name = [ $(target).name ] ; - local project = [ $(target).project ] ; - - local project-root = [ $(project).get project-root ] ; - local library = [ get-library-name [ path.root [ $(project).get location ] - [ path.pwd ] ] ] ; - if $(library) - { - name = $(library)/$(name) ; - } - - local sources = [ $(target).sources ] ; - local source-files ; - for local s in $(sources) - { - if [ class.is-a $(s) : file-reference ] - { - local location = [ path.root [ path.root [ $(s).name ] - [ $(s).location ] ] [ path.pwd ] ] ; - - source-files += [ path.relative-to [ path.root $(project-root) - [ path.pwd ] ] $(location) ] ; - } - } - - local target-name = [ $(project).get location ] // [ $(target).name ] .test - ; - target-name = $(target-name:J=) ; - - local r = [ $(target).requirements ] ; - # Extract values of the feature. - local test-info = [ $(r).get ] ; - - # If the user requested XML output on the command-line, add the test info to - # that XML file rather than dumping them to stdout. - if $(.out-xml) - { - local nl = " -" ; - .contents on $(.out-xml) += - "$(nl) " - "$(nl) " - "$(nl) " - "$(nl) " - "$(nl) " - ; - } - else - { - # Format them into a single string of quoted strings. - test-info = \"$(test-info:J=\"\ \")\" ; - - ECHO boost-test($(type)) \"$(name)\" [$(test-info)] ":" - \"$(source-files)\" ; - } -} - - -# Register generators. Depending on target type, either 'expect-success' or -# 'expect-failure' rule will be used. -generators.register-standard testing.expect-success : OBJ : COMPILE ; -generators.register-standard testing.expect-failure : OBJ : COMPILE_FAIL ; -generators.register-standard testing.expect-success : RUN_OUTPUT : RUN ; -generators.register-standard testing.expect-failure : RUN_OUTPUT : RUN_FAIL ; -generators.register-standard testing.expect-failure : EXE : LINK_FAIL ; -generators.register-standard testing.expect-success : EXE : LINK ; - -# Generator which runs an EXE and captures output. -generators.register-standard testing.capture-output : EXE : RUN_OUTPUT ; - -# Generator which creates a target if sources run successfully. Differs from RUN -# in that run output is not captured. The reason why it exists is that the 'run' -# rule is much better for automated testing, but is not user-friendly (see -# http://article.gmane.org/gmane.comp.lib.boost.build/6353). -generators.register-standard testing.unit-test : EXE : UNIT_TEST ; - - -# The action rules called by generators. - -# Causes the 'target' to exist after bjam invocation if and only if all the -# dependencies were successfully built. -# -rule expect-success ( target : dependency + : requirements * ) -{ - **passed** $(target) : $(sources) ; -} - - -# Causes the 'target' to exist after bjam invocation if and only if all some of -# the dependencies were not successfully built. -# -rule expect-failure ( target : dependency + : properties * ) -{ - local grist = [ MATCH ^<(.*)> : $(dependency:G) ] ; - local marker = $(dependency:G=$(grist)*fail) ; - (failed-as-expected) $(marker) ; - FAIL_EXPECTED $(dependency) ; - LOCATE on $(marker) = [ on $(dependency) return $(LOCATE) ] ; - RMOLD $(marker) ; - DEPENDS $(marker) : $(dependency) ; - DEPENDS $(target) : $(marker) ; - **passed** $(target) : $(marker) ; -} - - -# The rule/action combination used to report successful passing of a test. -# -rule **passed** -{ - # Dump all the tests, if needed. We do it here, since dump should happen - # only after all Jamfiles have been read, and there is no such place - # currently defined (but there should be). - if ! $(.dumped-tests) && ( --dump-tests in [ modules.peek : ARGV ] ) - { - .dumped-tests = true ; - dump-tests ; - } - - # Force deletion of the target, in case any dependencies failed to build. - RMOLD $(<) ; -} - - -# Used to create test files signifying passed tests. -# -actions **passed** -{ - echo passed > "$(<)" -} - - -# Used to create replacement object files that do not get created during tests -# that are expected to fail. -# -actions (failed-as-expected) -{ - echo failed as expected > "$(<)" -} - - -rule run-path-setup ( target : source : properties * ) -{ - # For testing, we need to make sure that all dynamic libraries needed by the - # test are found. So, we collect all paths from dependency libraries (via - # xdll-path property) and add whatever explicit dll-path user has specified. - # The resulting paths are added to the environment on each test invocation. - local dll-paths = [ feature.get-values : $(properties) ] ; - dll-paths += [ feature.get-values : $(properties) ] ; - dll-paths += [ on $(source) return $(RUN_PATH) ] ; - dll-paths = [ sequence.unique $(dll-paths) ] ; - if $(dll-paths) - { - dll-paths = [ sequence.transform path.native : $(dll-paths) ] ; - PATH_SETUP on $(target) = [ common.prepend-path-variable-command - [ os.shared-library-path-variable ] : $(dll-paths) ] ; - } -} - - -local argv = [ modules.peek : ARGV ] ; - -toolset.flags testing.capture-output ARGS ; -toolset.flags testing.capture-output INPUT_FILES ; -toolset.flags testing.capture-output LAUNCHER ; - - -# Runs executable 'sources' and stores stdout in file 'target'. Unless -# --preserve-test-targets command line option has been specified, removes the -# executable. The 'target-to-remove' parameter controls what should be removed: -# - if 'none', does not remove anything, ever -# - if empty, removes 'source' -# - if non-empty and not 'none', contains a list of sources to remove. -# -rule capture-output ( target : source : properties * : targets-to-remove * ) -{ - output-file on $(target) = $(target:S=.output) ; - LOCATE on $(target:S=.output) = [ on $(target) return $(LOCATE) ] ; - - # The INCLUDES kill a warning about independent target... - INCLUDES $(target) : $(target:S=.output) ; - # but it also puts .output into dependency graph, so we must tell jam it is - # OK if it cannot find the target or updating rule. - NOCARE $(target:S=.output) ; - - # This has two-fold effect. First it adds input files to the dependendency - # graph, preventing a warning. Second, it causes input files to be bound - # before target is created. Therefore, they are bound using SEARCH setting - # on them and not LOCATE setting of $(target), as in other case (due to jam - # bug). - DEPENDS $(target) : [ on $(target) return $(INPUT_FILES) ] ; - - if $(targets-to-remove) = none - { - targets-to-remove = ; - } - else if ! $(targets-to-remove) - { - targets-to-remove = $(source) ; - } - - run-path-setup $(target) : $(source) : $(properties) ; - - if [ feature.get-values preserve-test-targets : $(properties) ] = off - { - TEMPORARY $(targets-to-remove) ; - # Set a second action on target that will be executed after capture - # output action. The 'RmTemps' rule has the 'ignore' modifier so it is - # always considered succeeded. This is needed for 'run-fail' test. For - # that test the target will be marked with FAIL_EXPECTED, and without - # 'ignore' successful execution will be negated and be reported as - # failure. With 'ignore' we do not detect a case where removing files - # fails, but it is not likely to happen. - RmTemps $(target) : $(targets-to-remove) ; - } -} - - -if [ os.name ] = NT -{ - .STATUS = %status% ; - .SET_STATUS = "set status=%ERRORLEVEL%" ; - .RUN_OUTPUT_NL = "echo." ; - .STATUS_0 = "%status% EQU 0 (" ; - .STATUS_NOT_0 = "%status% NEQ 0 (" ; - .VERBOSE = "%verbose% EQU 1 (" ; - .ENDIF = ")" ; - .SHELL_SET = "set " ; - .CATENATE = type ; - .CP = copy ; -} -else -{ - .STATUS = "$status" ; - .SET_STATUS = "status=$?" ; - .RUN_OUTPUT_NL = "echo" ; - .STATUS_0 = "test $status -eq 0 ; then" ; - .STATUS_NOT_0 = "test $status -ne 0 ; then" ; - .VERBOSE = "test $verbose -eq 1 ; then" ; - .ENDIF = "fi" ; - .SHELL_SET = "" ; - .CATENATE = cat ; - .CP = cp ; -} - - -.VERBOSE_TEST = 0 ; -if --verbose-test in [ modules.peek : ARGV ] -{ - .VERBOSE_TEST = 1 ; -} - - -.RM = [ common.rm-command ] ; - - -actions capture-output bind INPUT_FILES output-file -{ - $(PATH_SETUP) - $(LAUNCHER) "$(>)" $(ARGS) "$(INPUT_FILES)" > "$(output-file)" 2>&1 - $(.SET_STATUS) - $(.RUN_OUTPUT_NL) >> "$(output-file)" - echo EXIT STATUS: $(.STATUS) >> "$(output-file)" - if $(.STATUS_0) - $(.CP) "$(output-file)" "$(<)" - $(.ENDIF) - $(.SHELL_SET)verbose=$(.VERBOSE_TEST) - if $(.STATUS_NOT_0) - $(.SHELL_SET)verbose=1 - $(.ENDIF) - if $(.VERBOSE) - echo ====== BEGIN OUTPUT ====== - $(.CATENATE) "$(output-file)" - echo ====== END OUTPUT ====== - $(.ENDIF) - exit $(.STATUS) -} - - -actions quietly updated ignore piecemeal together RmTemps -{ - $(.RM) "$(>)" -} - - -.MAKE_FILE = [ common.file-creation-command ] ; - -toolset.flags testing.unit-test LAUNCHER ; -toolset.flags testing.unit-test ARGS ; - - -rule unit-test ( target : source : properties * ) -{ - run-path-setup $(target) : $(source) : $(properties) ; -} - - -actions unit-test -{ - $(PATH_SETUP) - $(LAUNCHER) $(>) $(ARGS) && $(.MAKE_FILE) $(<) -} - - -IMPORT $(__name__) : compile compile-fail run run-fail link link-fail - : : compile compile-fail run run-fail link link-fail ; - - -type.register TIME : time ; -generators.register-standard testing.time : : TIME ; - - -rule record-time ( target : source : start end user system ) -{ - local src-string = [$(source:G=:J=",")"] " ; - USER_TIME on $(target) += $(src-string)$(user) ; - SYSTEM_TIME on $(target) += $(src-string)$(system) ; -} - - -IMPORT testing : record-time : : testing.record-time ; - - -# Calling this rule requests that Boost Build time how long it taks to build the -# 'source' target and display the results both on the standard output and in the -# 'target' file. -# -rule time ( target : source : properties * ) -{ - # Set up rule for recording timing information. - __TIMING_RULE__ on $(source) = testing.record-time $(target) ; - - # Make sure that the source is rebuilt any time we need to retrieve that - # information. - REBUILDS $(target) : $(source) ; -} - - -actions time -{ - echo user: $(USER_TIME) - echo system: $(SYSTEM_TIME) - - echo user: $(USER_TIME)" seconds" > "$(<)" - echo system: $(SYSTEM_TIME)" seconds" >> "$(<)" -} diff --git a/jam-files/boost-build/tools/testing.py b/jam-files/boost-build/tools/testing.py deleted file mode 100644 index 3b53500c..00000000 --- a/jam-files/boost-build/tools/testing.py +++ /dev/null @@ -1,342 +0,0 @@ -# Status: ported, except for --out-xml -# Base revision: 64488 -# -# Copyright 2005 Dave Abrahams -# Copyright 2002, 2003, 2004, 2005, 2010 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This module implements regression testing framework. It declares a number of -# main target rules which perform some action and, if the results are OK, -# creates an output file. -# -# The exact list of rules is: -# 'compile' -- creates .test file if compilation of sources was -# successful. -# 'compile-fail' -- creates .test file if compilation of sources failed. -# 'run' -- creates .test file is running of executable produced from -# sources was successful. Also leaves behind .output file -# with the output from program run. -# 'run-fail' -- same as above, but .test file is created if running fails. -# -# In all cases, presence of .test file is an indication that the test passed. -# For more convenient reporting, you might want to use C++ Boost regression -# testing utilities (see http://www.boost.org/more/regression.html). -# -# For historical reason, a 'unit-test' rule is available which has the same -# syntax as 'exe' and behaves just like 'run'. - -# Things to do: -# - Teach compiler_status handle Jamfile.v2. -# Notes: -# - is not implemented, since it is Como-specific, and it is not -# clear how to implement it -# - std::locale-support is not implemented (it is used in one test). - -import b2.build.feature as feature -import b2.build.type as type -import b2.build.targets as targets -import b2.build.generators as generators -import b2.build.toolset as toolset -import b2.tools.common as common -import b2.util.option as option -import b2.build_system as build_system - - - -from b2.manager import get_manager -from b2.util import stem, bjam_signature -from b2.util.sequence import unique - -import bjam - -import re -import os.path -import sys - -def init(): - pass - -# Feature controling the command used to lanch test programs. -feature.feature("testing.launcher", [], ["free", "optional"]) - -feature.feature("test-info", [], ["free", "incidental"]) -feature.feature("testing.arg", [], ["free", "incidental"]) -feature.feature("testing.input-file", [], ["free", "dependency"]) - -feature.feature("preserve-test-targets", ["on", "off"], ["incidental", "propagated"]) - -# Register target types. -type.register("TEST", ["test"]) -type.register("COMPILE", [], "TEST") -type.register("COMPILE_FAIL", [], "TEST") - -type.register("RUN_OUTPUT", ["run"]) -type.register("RUN", [], "TEST") -type.register("RUN_FAIL", [], "TEST") - -type.register("LINK", [], "TEST") -type.register("LINK_FAIL", [], "TEST") -type.register("UNIT_TEST", ["passed"], "TEST") - -__all_tests = [] - -# Declare the rules which create main targets. While the 'type' module already -# creates rules with the same names for us, we need extra convenience: default -# name of main target, so write our own versions. - -# Helper rule. Create a test target, using basename of first source if no target -# name is explicitly passed. Remembers the created target in a global variable. -def make_test(target_type, sources, requirements, target_name=None): - - if not target_name: - target_name = stem(os.path.basename(sources[0])) - - # Having periods (".") in the target name is problematic because the typed - # generator will strip the suffix and use the bare name for the file - # targets. Even though the location-prefix averts problems most times it - # does not prevent ambiguity issues when referring to the test targets. For - # example when using the XML log output. So we rename the target to remove - # the periods, and provide an alias for users. - real_name = target_name.replace(".", "~") - - project = get_manager().projects().current() - # The forces the build system for generate paths in the - # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow - # post-processing tools to work. - t = get_manager().targets().create_typed_target( - type.type_from_rule_name(target_type), project, real_name, sources, - requirements + ["" + real_name + ".test"], [], []) - - # The alias to the real target, per period replacement above. - if real_name != target_name: - get_manager().projects().project_rules().all_names_["alias"]( - target_name, [t]) - - # Remember the test (for --dump-tests). A good way would be to collect all - # given a project. This has some technical problems: e.g. we can not call - # this dump from a Jamfile since projects referred by 'build-project' are - # not available until the whole Jamfile has been loaded. - __all_tests.append(t) - return t - - -# Note: passing more that one cpp file here is known to fail. Passing a cpp file -# and a library target works. -# -@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) -def compile(sources, requirements, target_name=None): - return make_test("compile", sources, requirements, target_name) - -@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) -def compile_fail(sources, requirements, target_name=None): - return make_test("compile-fail", sources, requirements, target_name) - -@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) -def link(sources, requirements, target_name=None): - return make_test("link", sources, requirements, target_name) - -@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) -def link_fail(sources, requirements, target_name=None): - return make_test("link-fail", sources, requirements, target_name) - -def handle_input_files(input_files): - if len(input_files) > 1: - # Check that sorting made when creating property-set instance will not - # change the ordering. - if sorted(input_files) != input_files: - get_manager().errors()("Names of input files must be sorted alphabetically\n" + - "due to internal limitations") - return ["" + f for f in input_files] - -@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"], - ["requirements", "*"], ["target_name", "?"], - ["default_build", "*"])) -def run(sources, args, input_files, requirements, target_name=None, default_build=[]): - if args: - requirements.append("" + " ".join(args)) - requirements.extend(handle_input_files(input_files)) - return make_test("run", sources, requirements, target_name) - -@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"], - ["requirements", "*"], ["target_name", "?"], - ["default_build", "*"])) -def run_fail(sources, args, input_files, requirements, target_name=None, default_build=[]): - if args: - requirements.append("" + " ".join(args)) - requirements.extend(handle_input_files(input_files)) - return make_test("run-fail", sources, requirements, target_name) - -# Register all the rules -for name in ["compile", "compile-fail", "link", "link-fail", "run", "run-fail"]: - get_manager().projects().add_rule(name, getattr(sys.modules[__name__], name.replace("-", "_"))) - -# Use 'test-suite' as a synonym for 'alias', for backward compatibility. -from b2.build.alias import alias -get_manager().projects().add_rule("test-suite", alias) - -# For all main targets in 'project-module', which are typed targets with type -# derived from 'TEST', produce some interesting information. -# -def dump_tests(): - for t in __all_tests: - dump_test(t) - -# Given a project location in normalized form (slashes are forward), compute the -# name of the Boost library. -# -__ln1 = re.compile("/(tools|libs)/(.*)/(test|example)") -__ln2 = re.compile("/(tools|libs)/(.*)$") -__ln3 = re.compile("(/status$)") -def get_library_name(path): - - path = path.replace("\\", "/") - match1 = __ln1.match(path) - match2 = __ln2.match(path) - match3 = __ln3.match(path) - - if match1: - return match1.group(2) - elif match2: - return match2.group(2) - elif match3: - return "" - elif option.get("dump-tests", False, True): - # The 'run' rule and others might be used outside boost. In that case, - # just return the path, since the 'library name' makes no sense. - return path - -# Was an XML dump requested? -__out_xml = option.get("out-xml", False, True) - -# Takes a target (instance of 'basic-target') and prints -# - its type -# - its name -# - comments specified via the property -# - relative location of all source from the project root. -# -def dump_test(target): - type = target.type() - name = target.name() - project = target.project() - - project_root = project.get('project-root') - library = get_library_name(os.path.abspath(project.get('location'))) - if library: - name = library + "/" + name - - sources = target.sources() - source_files = [] - for s in sources: - if isinstance(s, targets.FileReference): - location = os.path.abspath(os.path.join(s.location(), s.name())) - source_files.append(os.path.relpath(location, os.path.abspath(project_root))) - - target_name = project.get('location') + "//" + target.name() + ".test" - - test_info = target.requirements().get('test-info') - test_info = " ".join('"' + ti + '"' for ti in test_info) - - # If the user requested XML output on the command-line, add the test info to - # that XML file rather than dumping them to stdout. - #if $(.out-xml) - #{ -# local nl = " -#" ; -# .contents on $(.out-xml) += -# "$(nl) " -# "$(nl) " -# "$(nl) " -# "$(nl) " -# "$(nl) " -# ; -# } -# else - - source_files = " ".join('"' + s + '"' for s in source_files) - if test_info: - print 'boost-test(%s) "%s" [%s] : %s' % (type, name, test_info, source_files) - else: - print 'boost-test(%s) "%s" : %s' % (type, name, source_files) - -# Register generators. Depending on target type, either 'expect-success' or -# 'expect-failure' rule will be used. -generators.register_standard("testing.expect-success", ["OBJ"], ["COMPILE"]) -generators.register_standard("testing.expect-failure", ["OBJ"], ["COMPILE_FAIL"]) -generators.register_standard("testing.expect-success", ["RUN_OUTPUT"], ["RUN"]) -generators.register_standard("testing.expect-failure", ["RUN_OUTPUT"], ["RUN_FAIL"]) -generators.register_standard("testing.expect-success", ["EXE"], ["LINK"]) -generators.register_standard("testing.expect-failure", ["EXE"], ["LINK_FAIL"]) - -# Generator which runs an EXE and captures output. -generators.register_standard("testing.capture-output", ["EXE"], ["RUN_OUTPUT"]) - -# Generator which creates a target if sources run successfully. Differs from RUN -# in that run output is not captured. The reason why it exists is that the 'run' -# rule is much better for automated testing, but is not user-friendly (see -# http://article.gmane.org/gmane.comp.lib.boost.build/6353). -generators.register_standard("testing.unit-test", ["EXE"], ["UNIT_TEST"]) - -# FIXME: if those calls are after bjam.call, then bjam will crash -# when toolset.flags calls bjam.caller. -toolset.flags("testing.capture-output", "ARGS", [], [""]) -toolset.flags("testing.capture-output", "INPUT_FILES", [], [""]) -toolset.flags("testing.capture-output", "LAUNCHER", [], [""]) - -toolset.flags("testing.unit-test", "LAUNCHER", [], [""]) -toolset.flags("testing.unit-test", "ARGS", [], [""]) - -type.register("TIME", ["time"]) -generators.register_standard("testing.time", [], ["TIME"]) - - -# The following code sets up actions for this module. It's pretty convoluted, -# but the basic points is that we most of actions are defined by Jam code -# contained in testing-aux.jam, which we load into Jam module named 'testing' - -def run_path_setup(target, sources, ps): - - # For testing, we need to make sure that all dynamic libraries needed by the - # test are found. So, we collect all paths from dependency libraries (via - # xdll-path property) and add whatever explicit dll-path user has specified. - # The resulting paths are added to the environment on each test invocation. - dll_paths = ps.get('dll-path') - dll_paths.extend(ps.get('xdll-path')) - dll_paths.extend(bjam.call("get-target-variable", sources, "RUN_PATH")) - dll_paths = unique(dll_paths) - if dll_paths: - bjam.call("set-target-variable", target, "PATH_SETUP", - common.prepend_path_variable_command( - common.shared_library_path_variable(), dll_paths)) - -def capture_output_setup(target, sources, ps): - run_path_setup(target, sources, ps) - - if ps.get('preserve-test-targets') == ['off']: - bjam.call("set-target-variable", target, "REMOVE_TEST_TARGETS", "1") - -get_manager().engine().register_bjam_action("testing.capture-output", - capture_output_setup) - - -path = os.path.dirname(get_manager().projects().loaded_tool_module_path_[__name__]) -import b2.util.os_j -get_manager().projects().project_rules()._import_rule("testing", "os.name", - b2.util.os_j.name) -import b2.tools.common -get_manager().projects().project_rules()._import_rule("testing", "common.rm-command", - b2.tools.common.rm_command) -get_manager().projects().project_rules()._import_rule("testing", "common.file-creation-command", - b2.tools.common.file_creation_command) - -bjam.call("load", "testing", os.path.join(path, "testing-aux.jam")) - - -for name in ["expect-success", "expect-failure", "time"]: - get_manager().engine().register_bjam_action("testing." + name) - -get_manager().engine().register_bjam_action("testing.unit-test", - run_path_setup) - -if option.get("dump-tests", False, True): - build_system.add_pre_build_hook(dump_tests) diff --git a/jam-files/boost-build/tools/types/__init__.py b/jam-files/boost-build/tools/types/__init__.py deleted file mode 100644 index f972b714..00000000 --- a/jam-files/boost-build/tools/types/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -__all__ = [ - 'asm', - 'cpp', - 'exe', - 'html', - 'lib', - 'obj', - 'rsp', -] - -def register_all (): - for i in __all__: - m = __import__ (__name__ + '.' + i) - reg = i + '.register ()' - #exec (reg) - -# TODO: (PF) I thought these would be imported automatically. Anyone knows why they aren't? -register_all () diff --git a/jam-files/boost-build/tools/types/asm.jam b/jam-files/boost-build/tools/types/asm.jam deleted file mode 100644 index a340db36..00000000 --- a/jam-files/boost-build/tools/types/asm.jam +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright Craig Rodrigues 2005. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -type ASM : s S asm ; diff --git a/jam-files/boost-build/tools/types/asm.py b/jam-files/boost-build/tools/types/asm.py deleted file mode 100644 index b4e1c30e..00000000 --- a/jam-files/boost-build/tools/types/asm.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Craig Rodrigues 2005. -# Copyright (c) 2008 Steven Watanabe -# -# Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register(): - type.register_type('ASM', ['s', 'S', 'asm']) - -register() diff --git a/jam-files/boost-build/tools/types/cpp.jam b/jam-files/boost-build/tools/types/cpp.jam deleted file mode 100644 index 3159cdd7..00000000 --- a/jam-files/boost-build/tools/types/cpp.jam +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright David Abrahams 2004. -# Copyright 2002, 2003, 2004, 2005, 2006 Vladimir Prus -# Copyright 2010 Rene Rivera -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) -import type ; -import scanner ; - -class c-scanner : scanner -{ - import path ; - import regex ; - import scanner ; - import sequence ; - import virtual-target ; - - rule __init__ ( includes * ) - { - scanner.__init__ ; - - for local i in $(includes) - { - self.includes += [ sequence.transform path.native - : [ regex.split $(i:G=) "&&" ] ] ; - } - } - - rule pattern ( ) - { - return "#[ \t]*include[ ]*(<(.*)>|\"(.*)\")" ; - } - - rule process ( target : matches * : binding ) - { - local angle = [ regex.transform $(matches) : "<(.*)>" ] ; - angle = [ sequence.transform path.native : $(angle) ] ; - local quoted = [ regex.transform $(matches) : "\"(.*)\"" ] ; - quoted = [ sequence.transform path.native : $(quoted) ] ; - - # CONSIDER: the new scoping rule seem to defeat "on target" variables. - local g = [ on $(target) return $(HDRGRIST) ] ; - local b = [ NORMALIZE_PATH $(binding:D) ] ; - - # Attach binding of including file to included targets. When a target is - # directly created from virtual target this extra information is - # unnecessary. But in other cases, it allows us to distinguish between - # two headers of the same name included from different places. We do not - # need this extra information for angle includes, since they should not - # depend on including file (we can not get literal "." in include path). - local g2 = $(g)"#"$(b) ; - - angle = $(angle:G=$(g)) ; - quoted = $(quoted:G=$(g2)) ; - - local all = $(angle) $(quoted) ; - - INCLUDES $(target) : $(all) ; - NOCARE $(all) ; - SEARCH on $(angle) = $(self.includes:G=) ; - SEARCH on $(quoted) = $(b) $(self.includes:G=) ; - - # Just propagate the current scanner to includes in hope that includes - # do not change scanners. - scanner.propagate $(__name__) : $(angle) $(quoted) : $(target) ; - - ISFILE $(angle) $(quoted) ; - } -} - -scanner.register c-scanner : include ; - -type.register CPP : cpp cxx cc ; -type.register H : h ; -type.register HPP : hpp : H ; -type.register C : c ; - -# It most cases where a CPP file or a H file is a source of some action, we -# should rebuild the result if any of files included by CPP/H are changed. One -# case when this is not needed is installation, which is handled specifically. -type.set-scanner CPP : c-scanner ; -type.set-scanner C : c-scanner ; -# One case where scanning of H/HPP files is necessary is PCH generation -- if -# any header included by HPP being precompiled changes, we need to recompile the -# header. -type.set-scanner H : c-scanner ; -type.set-scanner HPP : c-scanner ; diff --git a/jam-files/boost-build/tools/types/cpp.py b/jam-files/boost-build/tools/types/cpp.py deleted file mode 100644 index 7b56111c..00000000 --- a/jam-files/boost-build/tools/types/cpp.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register (): - type.register_type ('CPP', ['cpp', 'cxx', 'cc']) - -register () diff --git a/jam-files/boost-build/tools/types/exe.jam b/jam-files/boost-build/tools/types/exe.jam deleted file mode 100644 index 47109513..00000000 --- a/jam-files/boost-build/tools/types/exe.jam +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -import type ; - -type.register EXE ; -type.set-generated-target-suffix EXE : windows : "exe" ; -type.set-generated-target-suffix EXE : cygwin : "exe" ; diff --git a/jam-files/boost-build/tools/types/exe.py b/jam-files/boost-build/tools/types/exe.py deleted file mode 100644 index a4935e24..00000000 --- a/jam-files/boost-build/tools/types/exe.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register (): - type.register_type ('EXE', ['exe'], None, ['NT', 'CYGWIN']) - type.register_type ('EXE', [], None, []) - -register () diff --git a/jam-files/boost-build/tools/types/html.jam b/jam-files/boost-build/tools/types/html.jam deleted file mode 100644 index 5cd337d0..00000000 --- a/jam-files/boost-build/tools/types/html.jam +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -type HTML : html ; diff --git a/jam-files/boost-build/tools/types/html.py b/jam-files/boost-build/tools/types/html.py deleted file mode 100644 index 63af4d90..00000000 --- a/jam-files/boost-build/tools/types/html.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register (): - type.register_type ('HTML', ['html']) - -register () diff --git a/jam-files/boost-build/tools/types/lib.jam b/jam-files/boost-build/tools/types/lib.jam deleted file mode 100644 index 854ab8fd..00000000 --- a/jam-files/boost-build/tools/types/lib.jam +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -import type ; # for set-generated-target-suffix -import os ; - -# The following naming scheme is used for libraries. -# -# On *nix: -# libxxx.a static library -# libxxx.so shared library -# -# On windows (msvc) -# libxxx.lib static library -# xxx.dll DLL -# xxx.lib import library -# -# On windows (mingw): -# libxxx.a static library -# libxxx.dll DLL -# libxxx.dll.a import library -# -# On cygwin i.e. cygwin -# libxxx.a static library -# cygxxx.dll DLL -# libxxx.dll.a import library -# - -type.register LIB ; - -# FIXME: should not register both extensions on both platforms. -type.register STATIC_LIB : a lib : LIB ; - -# The 'lib' prefix is used everywhere -type.set-generated-target-prefix STATIC_LIB : : lib ; - -# Use '.lib' suffix for windows -type.set-generated-target-suffix STATIC_LIB : windows : lib ; - -# Except with gcc. -type.set-generated-target-suffix STATIC_LIB : gcc windows : a ; - -# Use xxx.lib for import libs -type IMPORT_LIB : : STATIC_LIB ; -type.set-generated-target-prefix IMPORT_LIB : : "" ; -type.set-generated-target-suffix IMPORT_LIB : : lib ; - -# Except with gcc (mingw or cygwin), where use libxxx.dll.a -type.set-generated-target-prefix IMPORT_LIB : gcc : lib ; -type.set-generated-target-suffix IMPORT_LIB : gcc : dll.a ; - -type.register SHARED_LIB : so dll dylib : LIB ; - -# Both mingw and cygwin use libxxx.dll naming scheme. -# On Linux, use "lib" prefix -type.set-generated-target-prefix SHARED_LIB : : lib ; -# But don't use it on windows -type.set-generated-target-prefix SHARED_LIB : windows : "" ; -# But use it again on mingw -type.set-generated-target-prefix SHARED_LIB : gcc windows : lib ; -# And use 'cyg' on cygwin -type.set-generated-target-prefix SHARED_LIB : cygwin : cyg ; - - -type.set-generated-target-suffix SHARED_LIB : windows : dll ; -type.set-generated-target-suffix SHARED_LIB : cygwin : dll ; -type.set-generated-target-suffix SHARED_LIB : darwin : dylib ; - -type SEARCHED_LIB : : LIB ; -# This is needed so that when we create a target of SEARCHED_LIB -# type, there's no prefix or suffix automatically added. -type.set-generated-target-prefix SEARCHED_LIB : : "" ; -type.set-generated-target-suffix SEARCHED_LIB : : "" ; diff --git a/jam-files/boost-build/tools/types/lib.py b/jam-files/boost-build/tools/types/lib.py deleted file mode 100644 index d0ec1fb5..00000000 --- a/jam-files/boost-build/tools/types/lib.py +++ /dev/null @@ -1,77 +0,0 @@ -# Status: ported -# Base revision: 64456. -# Copyright David Abrahams 2004. -# Copyright Vladimir Prus 2010. -# Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -import b2.build.type as type - -# The following naming scheme is used for libraries. -# -# On *nix: -# libxxx.a static library -# libxxx.so shared library -# -# On windows (msvc) -# libxxx.lib static library -# xxx.dll DLL -# xxx.lib import library -# -# On windows (mingw): -# libxxx.a static library -# libxxx.dll DLL -# libxxx.dll.a import library -# -# On cygwin i.e. cygwin -# libxxx.a static library -# cygxxx.dll DLL -# libxxx.dll.a import library -# - -type.register('LIB') - -# FIXME: should not register both extensions on both platforms. -type.register('STATIC_LIB', ['a', 'lib'], 'LIB') - -# The 'lib' prefix is used everywhere -type.set_generated_target_prefix('STATIC_LIB', [], 'lib') - -# Use '.lib' suffix for windows -type.set_generated_target_suffix('STATIC_LIB', ['windows'], 'lib') - -# Except with gcc. -type.set_generated_target_suffix('STATIC_LIB', ['gcc', 'windows'], 'a') - -# Use xxx.lib for import libs -type.register('IMPORT_LIB', [], 'STATIC_LIB') -type.set_generated_target_prefix('IMPORT_LIB', [], '') -type.set_generated_target_suffix('IMPORT_LIB', [], 'lib') - -# Except with gcc (mingw or cygwin), where use libxxx.dll.a -type.set_generated_target_prefix('IMPORT_LIB', ['gcc'], 'lib') -type.set_generated_target_suffix('IMPORT_LIB', ['gcc'], 'dll.a') - -type.register('SHARED_LIB', ['so', 'dll', 'dylib'], 'LIB') - -# Both mingw and cygwin use libxxx.dll naming scheme. -# On Linux, use "lib" prefix -type.set_generated_target_prefix('SHARED_LIB', [], 'lib') -# But don't use it on windows -type.set_generated_target_prefix('SHARED_LIB', ['windows'], '') -# But use it again on mingw -type.set_generated_target_prefix('SHARED_LIB', ['gcc', 'windows'], 'lib') -# And use 'cyg' on cygwin -type.set_generated_target_prefix('SHARED_LIB', ['cygwin'], 'cyg') - - -type.set_generated_target_suffix('SHARED_LIB', ['windows'], 'dll') -type.set_generated_target_suffix('SHARED_LIB', ['cygwin'], 'dll') -type.set_generated_target_suffix('SHARED_LIB', ['darwin'], 'dylib') - -type.register('SEARCHED_LIB', [], 'LIB') -# This is needed so that when we create a target of SEARCHED_LIB -# type, there's no prefix or suffix automatically added. -type.set_generated_target_prefix('SEARCHED_LIB', [], '') -type.set_generated_target_suffix('SEARCHED_LIB', [], '') diff --git a/jam-files/boost-build/tools/types/obj.jam b/jam-files/boost-build/tools/types/obj.jam deleted file mode 100644 index 6afbcaa6..00000000 --- a/jam-files/boost-build/tools/types/obj.jam +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -import type ; - -type.register OBJ : o obj ; -type.set-generated-target-suffix OBJ : windows : obj ; -type.set-generated-target-suffix OBJ : cygwin : obj ; diff --git a/jam-files/boost-build/tools/types/obj.py b/jam-files/boost-build/tools/types/obj.py deleted file mode 100644 index e61e99a8..00000000 --- a/jam-files/boost-build/tools/types/obj.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register (): - type.register_type ('OBJ', ['obj'], None, ['NT', 'CYGWIN']) - type.register_type ('OBJ', ['o']) - -register () diff --git a/jam-files/boost-build/tools/types/objc.jam b/jam-files/boost-build/tools/types/objc.jam deleted file mode 100644 index 709cbd0c..00000000 --- a/jam-files/boost-build/tools/types/objc.jam +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright Rene Rivera 2008, 2010. -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -import type ; -import scanner ; -import types/cpp ; - -class objc-scanner : c-scanner -{ - rule __init__ ( includes * ) - { - c-scanner.__init__ $(includes) ; - } - - rule pattern ( ) - { - return "#[ \t]*include|import[ ]*(<(.*)>|\"(.*)\")" ; - } -} - -scanner.register objc-scanner : include ; - -type.register OBJECTIVE_C : m ; -type.register OBJECTIVE_CPP : mm ; -type.set-scanner OBJECTIVE_C : objc-scanner ; -type.set-scanner OBJECTIVE_CPP : objc-scanner ; diff --git a/jam-files/boost-build/tools/types/preprocessed.jam b/jam-files/boost-build/tools/types/preprocessed.jam deleted file mode 100644 index c9187ba6..00000000 --- a/jam-files/boost-build/tools/types/preprocessed.jam +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Steven Watanabe 2011 -# Distributed under the Boost Software License Version 1.0. (See -# accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -import type ; - -type.register PREPROCESSED_C : i : C ; -type.register PREPROCESSED_CPP : ii : CPP ; diff --git a/jam-files/boost-build/tools/types/qt.jam b/jam-files/boost-build/tools/types/qt.jam deleted file mode 100644 index 6d1dfbd4..00000000 --- a/jam-files/boost-build/tools/types/qt.jam +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Vladimir Prus 2005. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -type UI : ui ; -type QRC : qrc ; -type MOCCABLE_CPP ; -type MOCCABLE_H ; -# Result of running moc. -type MOC : moc : H ; diff --git a/jam-files/boost-build/tools/types/register.jam b/jam-files/boost-build/tools/types/register.jam deleted file mode 100644 index 203992ca..00000000 --- a/jam-files/boost-build/tools/types/register.jam +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -# This module's job is to automatically import all the type -# registration modules in its directory. -import type os path modules ; - -# Register the given type on the specified OSes, or on remaining OSes -# if os is not specified. This rule is injected into each of the type -# modules for the sake of convenience. -local rule type ( type : suffixes * : base-type ? : os * ) -{ - if ! [ type.registered $(type) ] - { - if ( ! $(os) ) || [ os.name ] in $(os) - { - type.register $(type) : $(suffixes) : $(base-type) ; - } - } -} - -.this-module's-file = [ modules.binding $(__name__) ] ; -.this-module's-dir = [ path.parent $(.this-module's-file) ] ; -.sibling-jamfiles = [ path.glob $(.this-module's-dir) : *.jam ] ; -.sibling-modules = [ MATCH ^(.*)\.jam$ : $(.sibling-jamfiles) ] ; - -# A loop over all modules in this directory -for m in $(.sibling-modules) -{ - m = [ path.basename $(m) ] ; - m = types/$(m) ; - - # Inject the type rule into the new module - IMPORT $(__name__) : type : $(m) : type ; - import $(m) ; -} - - diff --git a/jam-files/boost-build/tools/types/rsp.jam b/jam-files/boost-build/tools/types/rsp.jam deleted file mode 100644 index bdf8a7c9..00000000 --- a/jam-files/boost-build/tools/types/rsp.jam +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -type RSP : rsp ; diff --git a/jam-files/boost-build/tools/types/rsp.py b/jam-files/boost-build/tools/types/rsp.py deleted file mode 100644 index ccb379e9..00000000 --- a/jam-files/boost-build/tools/types/rsp.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright David Abrahams 2004. Distributed under the Boost -# Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -from b2.build import type - -def register (): - type.register_type ('RSP', ['rsp']) - -register () diff --git a/jam-files/boost-build/tools/unix.jam b/jam-files/boost-build/tools/unix.jam deleted file mode 100644 index 75949851..00000000 --- a/jam-files/boost-build/tools/unix.jam +++ /dev/null @@ -1,224 +0,0 @@ -# Copyright (c) 2004 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# This file implements linking semantic common to all unixes. On unix, static -# libraries must be specified in a fixed order on the linker command line. Generators -# declared there store information about the order and use it property. - -import feature ; -import "class" : new ; -import generators ; -import type ; -import set ; -import order ; -import builtin ; - -class unix-linking-generator : linking-generator -{ - import property-set ; - import type ; - import unix ; - - rule __init__ ( id - composing ? : # Specify if generator is composing. The generator will be - # composing if non-empty string is passed, or parameter is - # not given. To make generator non-composing, pass empty - # string ("") - source-types + : target-types + : - requirements * ) - { - composing ?= true ; - generator.__init__ $(id) $(composing) : $(source-types) : $(target-types) : - $(requirements) ; - } - - rule run ( project name ? : property-set : sources + ) - { - local result = [ linking-generator.run $(project) $(name) : $(property-set) - : $(sources) ] ; - - unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ; - - return $(result) ; - } - - rule generated-targets ( sources + : property-set : project name ? ) - { - local sources2 ; - local libraries ; - for local l in $(sources) - { - if [ type.is-derived [ $(l).type ] LIB ] - { - libraries += $(l) ; - } - else - { - sources2 += $(l) ; - } - } - - sources = $(sources2) [ unix.order-libraries $(libraries) ] ; - - return [ linking-generator.generated-targets $(sources) : $(property-set) - : $(project) $(name) ] ; - } - -} - -class unix-archive-generator : archive-generator -{ - import unix ; - - rule __init__ ( id composing ? : source-types + : target-types + : - requirements * ) - { - composing ?= true ; - archive-generator.__init__ $(id) $(composing) : $(source-types) : $(target-types) : - $(requirements) ; - } - - rule run ( project name ? : property-set : sources + ) - { - local result = [ archive-generator.run $(project) $(name) : $(property-set) - : $(sources) ] ; - - unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ; - - return $(result) ; - - } -} - -class unix-searched-lib-generator : searched-lib-generator -{ - import unix ; - rule __init__ ( * : * ) - { - generator.__init__ - $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule optional-properties ( ) - { - return $(self.requirements) ; - } - - rule run ( project name ? : property-set : sources * ) - { - local result = [ searched-lib-generator.run $(project) $(name) - : $(property-set) : $(sources) ] ; - - unix.set-library-order $(sources) : $(property-set) : $(result[2-]) ; - - return $(result) ; - } -} - -class unix-prebuilt-lib-generator : generator -{ - import unix ; - rule __init__ ( * : * ) - { - generator.__init__ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - - rule run ( project name ? : property-set : sources * ) - { - local f = [ $(property-set).get ] ; - unix.set-library-order-aux $(f) : $(sources) ; - return $(f) $(sources) ; - } -} - -generators.register - [ new unix-prebuilt-lib-generator unix.prebuilt : : LIB - : unix ] ; - -generators.override unix.prebuilt : builtin.lib-generator ; - - -# Declare generators -generators.register [ new unix-linking-generator unix.link : LIB OBJ : EXE - : unix ] ; - -generators.register [ new unix-archive-generator unix.archive : OBJ : STATIC_LIB - : unix ] ; - -generators.register [ new unix-linking-generator unix.link.dll : LIB OBJ : SHARED_LIB - : unix ] ; - -generators.register [ new unix-searched-lib-generator - unix.searched-lib-generator : : SEARCHED_LIB : unix ] ; - - -# The derived toolset must specify their own actions. -actions link { -} - -actions link.dll { -} - -actions archive { -} - -actions searched-lib-generator { -} - -actions prebuilt { -} - - - - - -.order = [ new order ] ; - -rule set-library-order-aux ( from * : to * ) -{ - for local f in $(from) - { - for local t in $(to) - { - if $(f) != $(t) - { - $(.order).add-pair $(f) $(t) ; - } - } - } -} - -rule set-library-order ( sources * : property-set : result * ) -{ - local used-libraries ; - local deps = [ $(property-set).dependency ] ; - for local l in $(sources) $(deps:G=) - { - if [ $(l).type ] && [ type.is-derived [ $(l).type ] LIB ] - { - used-libraries += $(l) ; - } - } - - local created-libraries ; - for local l in $(result) - { - if [ $(l).type ] && [ type.is-derived [ $(l).type ] LIB ] - { - created-libraries += $(l) ; - } - } - - created-libraries = [ set.difference $(created-libraries) : $(used-libraries) ] ; - set-library-order-aux $(created-libraries) : $(used-libraries) ; -} - -rule order-libraries ( libraries * ) -{ - local r = [ $(.order).order $(libraries) ] ; - return $(r) ; -} - \ No newline at end of file diff --git a/jam-files/boost-build/tools/unix.py b/jam-files/boost-build/tools/unix.py deleted file mode 100644 index d409c2e4..00000000 --- a/jam-files/boost-build/tools/unix.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2004 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -""" This file implements linking semantics common to all unixes. On unix, static - libraries must be specified in a fixed order on the linker command line. Generators - declared there store information about the order and use it properly. -""" - -import builtin -from b2.build import generators, type -from b2.util.utility import * -from b2.util import set, sequence - -class UnixLinkingGenerator (builtin.LinkingGenerator): - - def __init__ (self, id, composing, source_types, target_types, requirements): - builtin.LinkingGenerator.__init__ (self, id, composing, source_types, target_types, requirements) - - def run (self, project, name, prop_set, sources): - result = builtin.LinkingGenerator.run (self, project, name, prop_set, sources) - if result: - set_library_order (project.manager (), sources, prop_set, result [1]) - - return result - - def generated_targets (self, sources, prop_set, project, name): - sources2 = [] - libraries = [] - for l in sources: - if type.is_derived (l.type (), 'LIB'): - libraries.append (l) - - else: - sources2.append (l) - - sources = sources2 + order_libraries (libraries) - - return builtin.LinkingGenerator.generated_targets (self, sources, prop_set, project, name) - - -class UnixArchiveGenerator (builtin.ArchiveGenerator): - def __init__ (self, id, composing, source_types, target_types_and_names, requirements): - builtin.ArchiveGenerator.__init__ (self, id, composing, source_types, target_types_and_names, requirements) - - def run (self, project, name, prop_set, sources): - result = builtin.ArchiveGenerator.run(self, project, name, prop_set, sources) - set_library_order(project.manager(), sources, prop_set, result) - return result - -class UnixSearchedLibGenerator (builtin.SearchedLibGenerator): - - def __init__ (self): - builtin.SearchedLibGenerator.__init__ (self) - - def optional_properties (self): - return self.requirements () - - def run (self, project, name, prop_set, sources, multiple): - result = SearchedLibGenerator.run (project, name, prop_set, sources, multiple) - - set_library_order (sources, prop_set, result) - - return result - -class UnixPrebuiltLibGenerator (generators.Generator): - def __init__ (self, id, composing, source_types, target_types_and_names, requirements): - generators.Generator.__init__ (self, id, composing, source_types, target_types_and_names, requirements) - - def run (self, project, name, prop_set, sources, multiple): - f = prop_set.get ('') - set_library_order_aux (f, sources) - return (f, sources) - -### # The derived toolset must specify their own rules and actions. -# FIXME: restore? -# action.register ('unix.prebuilt', None, None) - - -generators.register (UnixPrebuiltLibGenerator ('unix.prebuilt', False, [], ['LIB'], ['', 'unix'])) - - - - - -### # Declare generators -### generators.register [ new UnixLinkingGenerator unix.link : LIB OBJ : EXE -### : unix ] ; -generators.register (UnixArchiveGenerator ('unix.archive', True, ['OBJ'], ['STATIC_LIB'], ['unix'])) - -### generators.register [ new UnixLinkingGenerator unix.link.dll : LIB OBJ : SHARED_LIB -### : unix ] ; -### -### generators.register [ new UnixSearchedLibGenerator -### unix.SearchedLibGenerator : : SEARCHED_LIB : unix ] ; -### -### -### # The derived toolset must specify their own actions. -### actions link { -### } -### -### actions link.dll { -### } - -def unix_archive (manager, targets, sources, properties): - pass - -# FIXME: restore? -#action.register ('unix.archive', unix_archive, ['']) - -### actions searched-lib-generator { -### } -### -### actions prebuilt { -### } - - -from b2.util.order import Order -__order = Order () - -def set_library_order_aux (from_libs, to_libs): - for f in from_libs: - for t in to_libs: - if f != t: - __order.add_pair (f, t) - -def set_library_order (manager, sources, prop_set, result): - used_libraries = [] - deps = prop_set.dependency () - - sources.extend(d.value() for d in deps) - sources = sequence.unique(sources) - - for l in sources: - if l.type () and type.is_derived (l.type (), 'LIB'): - used_libraries.append (l) - - created_libraries = [] - for l in result: - if l.type () and type.is_derived (l.type (), 'LIB'): - created_libraries.append (l) - - created_libraries = set.difference (created_libraries, used_libraries) - set_library_order_aux (created_libraries, used_libraries) - -def order_libraries (libraries): - return __order.order (libraries) - diff --git a/jam-files/boost-build/tools/vacpp.jam b/jam-files/boost-build/tools/vacpp.jam deleted file mode 100644 index f4080fc0..00000000 --- a/jam-files/boost-build/tools/vacpp.jam +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright Vladimir Prus 2004. -# Copyright Toon Knapen 2004. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt -# or copy at http://www.boost.org/LICENSE_1_0.txt) - -# -# Boost.Build V2 toolset for the IBM XL C++ compiler -# - -import toolset : flags ; -import feature ; -import common ; -import generators ; -import os ; - -feature.extend toolset : vacpp ; -toolset.inherit vacpp : unix ; -generators.override vacpp.prebuilt : builtin.prebuilt ; -generators.override vacpp.searched-lib-generator : searched-lib-generator ; - -# Configure the vacpp toolset -rule init ( version ? : command * : options * ) -{ - local condition = [ - common.check-init-parameters vacpp : version $(version) ] ; - - command = [ common.get-invocation-command vacpp : xlC - : $(command) : "/usr/vacpp/bin/xlC" ] ; - - common.handle-options vacpp : $(condition) : $(command) : $(options) ; -} - -# Declare generators -generators.register-c-compiler vacpp.compile.c : C : OBJ : vacpp ; -generators.register-c-compiler vacpp.compile.c++ : CPP : OBJ : vacpp ; - -# Allow C++ style comments in C files -flags vacpp CFLAGS : -qcpluscmt ; - -# Declare flags -flags vacpp CFLAGS off : -qNOOPTimize ; -flags vacpp CFLAGS speed : -O3 -qstrict ; -flags vacpp CFLAGS space : -O2 -qcompact ; - -# Discretionary inlining (not recommended) -flags vacpp CFLAGS off : -qnoinline ; -flags vacpp CFLAGS on : -qinline ; -#flags vacpp CFLAGS full : -qinline ; -flags vacpp CFLAGS full : ; - -# Exception handling -flags vacpp C++FLAGS off : -qnoeh ; -flags vacpp C++FLAGS on : -qeh ; - -# Run-time Type Identification -flags vacpp C++FLAGS off : -qnortti ; -flags vacpp C++FLAGS on : -qrtti ; - -# Enable 64-bit memory addressing model -flags vacpp CFLAGS 64 : -q64 ; -flags vacpp LINKFLAGS 64 : -q64 ; -flags vacpp ARFLAGS aix/64 : -X 64 ; - -# Use absolute path when generating debug information -flags vacpp CFLAGS on : -g -qfullpath ; -flags vacpp LINKFLAGS on : -g -qfullpath ; -flags vacpp LINKFLAGS off : -s ; - -if [ os.name ] = AIX -{ - flags vacpp.compile C++FLAGS : -qfuncsect ; - - # The -bnoipath strips the prepending (relative) path of libraries from - # the loader section in the target library or executable. Hence, during - # load-time LIBPATH (identical to LD_LIBRARY_PATH) or a hard-coded - # -blibpath (*similar* to -lrpath/-lrpath-link) is searched. Without - # this option, the prepending (relative) path + library name is - # hard-coded in the loader section, causing *only* this path to be - # searched during load-time. Note that the AIX linker does not have an - # -soname equivalent, this is as close as it gets. - # - # The above options are definately for AIX 5.x, and most likely also for - # AIX 4.x and AIX 6.x. For details about the AIX linker see: - # http://download.boulder.ibm.com/ibmdl/pub/software/dw/aix/es-aix_ll.pdf - # - flags vacpp.link LINKFLAGS shared : -bnoipath ; - - # Run-time linking - flags vacpp.link EXE-LINKFLAGS shared : -brtl ; -} -else -{ - # Linux PPC - flags vacpp.compile CFLAGS shared : -qpic=large ; - flags vacpp FINDLIBS : rt ; -} - -# Profiling -flags vacpp CFLAGS on : -pg ; -flags vacpp LINKFLAGS on : -pg ; - -flags vacpp.compile OPTIONS ; -flags vacpp.compile.c++ OPTIONS ; -flags vacpp DEFINES ; -flags vacpp UNDEFS ; -flags vacpp HDRS ; -flags vacpp STDHDRS ; -flags vacpp.link OPTIONS ; -flags vacpp ARFLAGS ; - -flags vacpp LIBPATH ; -flags vacpp NEEDLIBS ; -flags vacpp FINDLIBS ; -flags vacpp FINDLIBS ; - -# Select the compiler name according to the threading model. -flags vacpp VA_C_COMPILER single : xlc ; -flags vacpp VA_C_COMPILER multi : xlc_r ; -flags vacpp VA_CXX_COMPILER single : xlC ; -flags vacpp VA_CXX_COMPILER multi : xlC_r ; - -SPACE = " " ; - -flags vacpp.link.dll HAVE_SONAME linux : "" ; - -actions vacpp.link bind NEEDLIBS -{ - $(VA_CXX_COMPILER) $(EXE-LINKFLAGS) $(LINKFLAGS) -o "$(<[1])" -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS) $(USER_OPTIONS) -} - -actions vacpp.link.dll bind NEEDLIBS -{ - xlC_r -G $(LINKFLAGS) -o "$(<[1])" $(HAVE_SONAME)-Wl,-soname$(SPACE)-Wl,$(<[-1]:D=) -L$(LIBPATH) -L$(STDLIBPATH) "$(>)" "$(NEEDLIBS)" "$(NEEDLIBS)" -l$(FINDLIBS) $(OPTIONS) $(USER_OPTIONS) -} - -actions vacpp.compile.c -{ - $(VA_C_COMPILER) -c $(OPTIONS) $(USER_OPTIONS) -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" -} - -actions vacpp.compile.c++ -{ - $(VA_CXX_COMPILER) -c $(OPTIONS) $(USER_OPTIONS) -I$(BOOST_ROOT) -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" -} - -actions updated together piecemeal vacpp.archive -{ - ar $(ARFLAGS) ru "$(<)" "$(>)" -} diff --git a/jam-files/boost-build/tools/whale.jam b/jam-files/boost-build/tools/whale.jam deleted file mode 100644 index 9335ff0c..00000000 --- a/jam-files/boost-build/tools/whale.jam +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (C) Vladimir Prus 2002-2005. - -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# This module implements support for Whale/Dolphin/WD parser/lexer tools. -# See http://www.cs.queensu.ca/home/okhotin/whale/ for details. -# -# There are three interesting target types: -# - WHL (the parser sources), that are converted to CPP and H -# - DLP (the lexer sources), that are converted to CPP and H -# - WD (combined parser/lexer sources), that are converted to WHL + DLP - -import type ; -import generators ; -import path ; -import "class" : new ; -import errors ; - -rule init ( path # path the Whale/Dolphin/WD binaries - ) -{ - if $(.configured) && $(.path) != $(path) - { - errors.user-error "Attempt to reconfigure Whale support" : - "Previously configured with path \"$(.path:E=)\"" : - "Now configuring with path \"$(path:E=)\"" ; - - } - .configured = true ; - .path = $(path) ; - - .whale = [ path.join $(path) whale ] ; - .dolphin = [ path.join $(path) dolphin ] ; - .wd = [ path.join $(path) wd ] ; -} - - -# Declare the types. -type.register WHL : whl ; -type.register DLP : dlp ; -type.register WHL_LR0 : lr0 ; -type.register WD : wd ; - -# Declare standard generators. -generators.register-standard whale.whale : WHL : CPP H H(%_symbols) ; -generators.register-standard whale.dolphin : DLP : CPP H ; -generators.register-standard whale.wd : WD : WHL(%_parser) DLP(%_lexer) ; - -# The conversions defines above a ambiguious when we generated CPP from WD. -# We can either go via WHL type, or via DLP type. -# The following custom generator handles this by running both conversions. - -class wd-to-cpp : generator -{ - rule __init__ ( * : * : * ) - { - generator.__init__ $(1) : $(2) : $(3) ; - } - - rule run ( project name ? : property-set : source * ) - { - if ! $(source[2]) - { - local new-sources ; - if ! [ $(source).type ] in WHL DLP - { - local r1 = [ generators.construct $(project) $(name) - : WHL : $(property-set) : $(source) ] ; - local r2 = [ generators.construct $(project) $(name) - : DLP : $(property-set) : $(source) ] ; - - new-sources = [ sequence.unique $(r1[2-]) $(r2[2-]) ] ; - } - else - { - new-sources = $(source) ; - } - - local result ; - for local i in $(new-sources) - { - local t = [ generators.construct $(project) $(name) : CPP - : $(property-set) : $(i) ] ; - result += $(t[2-]) ; - } - return $(result) ; - } - } - -} - - -generators.override whale.wd-to-cpp : whale.whale ; -generators.override whale.wd-to-cpp : whale.dolphin ; - - -generators.register [ new wd-to-cpp whale.wd-to-cpp : : CPP ] ; - - -actions whale -{ - $(.whale) -d $(<[1]:D) $(>) -} - -actions dolphin -{ - $(.dolphin) -d $(<[1]:D) $(>) -} - -actions wd -{ - $(.wd) -d $(<[1]:D) -g $(>) -} - diff --git a/jam-files/boost-build/tools/xlf.jam b/jam-files/boost-build/tools/xlf.jam deleted file mode 100644 index e7fcc608..00000000 --- a/jam-files/boost-build/tools/xlf.jam +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2004 Toon Knapen -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# -# toolset configuration for the IBM Fortran compiler (xlf) -# - -import toolset : flags ; -import feature ; -import fortran ; - -rule init ( version ? : command * : options * ) -{ -} - -# Declare flags and action for compilation -flags xlf OPTIONS off : -O0 ; -flags xlf OPTIONS speed : -O3 ; -flags xlf OPTIONS space : -Os ; - -flags xlf OPTIONS on : -g ; -flags xlf OPTIONS on : -pg ; - -flags xlf DEFINES ; -flags xlf INCLUDES ; - -rule compile-fortran -{ -} - -actions compile-fortran -{ - xlf $(OPTIONS) -I$(INCLUDES) -c -o "$(<)" "$(>)" -} - -generators.register-fortran-compiler xlf.compile-fortran : FORTRAN : OBJ ; diff --git a/jam-files/boost-build/tools/xsltproc-config.jam b/jam-files/boost-build/tools/xsltproc-config.jam deleted file mode 100644 index de54a2eb..00000000 --- a/jam-files/boost-build/tools/xsltproc-config.jam +++ /dev/null @@ -1,37 +0,0 @@ -#~ Copyright 2005 Rene Rivera. -#~ Distributed under the Boost Software License, Version 1.0. -#~ (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Automatic configuration for Python tools and librries. To use, just import this module. - -import os ; -import toolset : using ; - -if [ os.name ] = NT -{ - local xsltproc-path = [ GLOB [ modules.peek : PATH ] "C:\\Boost\\bin" : xsltproc\.exe ] ; - xsltproc-path = $(xsltproc-path[1]) ; - - if $(xsltproc-path) - { - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using xsltproc ":" $(xsltproc-path) ; - } - using xsltproc : $(xsltproc-path) ; - } -} -else -{ - local xsltproc-path = [ GLOB [ modules.peek : PATH ] : xsltproc ] ; - xsltproc-path = $(xsltproc-path[1]) ; - - if $(xsltproc-path) - { - if --debug-configuration in [ modules.peek : ARGV ] - { - ECHO "notice:" using xsltproc ":" $(xsltproc-path) ; - } - using xsltproc : $(xsltproc-path) ; - } -} diff --git a/jam-files/boost-build/tools/xsltproc.jam b/jam-files/boost-build/tools/xsltproc.jam deleted file mode 100644 index 96f5170b..00000000 --- a/jam-files/boost-build/tools/xsltproc.jam +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (C) 2003 Doug Gregor. Permission to copy, use, modify, sell and -# distribute this software is granted provided this copyright notice appears in -# all copies. This software is provided "as is" without express or implied -# warranty, and with no claim as to its suitability for any purpose. - -# This module defines rules to apply an XSLT stylesheet to an XML file using the -# xsltproc driver, part of libxslt. -# -# Note: except for 'init', this modules does not provide any rules for end -# users. - -import feature ; -import regex ; -import sequence ; -import common ; -import os ; -import modules ; -import path ; -import errors ; - -feature.feature xsl:param : : free ; -feature.feature xsl:path : : free ; -feature.feature catalog : : free ; - - -# Initialize xsltproc support. The parameters are: -# xsltproc: The xsltproc executable -# -rule init ( xsltproc ? ) -{ - if $(xsltproc) - { - modify-config ; - .xsltproc = $(xsltproc) ; - check-xsltproc ; - } -} - -rule freeze-config ( ) -{ - if ! $(.config-frozen) - { - .config-frozen = true ; - .xsltproc ?= [ modules.peek : XSLTPROC ] ; - .xsltproc ?= xsltproc ; - check-xsltproc ; - .is-cygwin = [ .is-cygwin $(.xsltproc) ] ; - } -} - -rule modify-config -{ - if $(.config-frozen) - { - errors.user-error "xsltproc: Cannot change xsltproc command after it has been used." ; - } -} - -rule check-xsltproc ( ) -{ - if $(.xsltproc) - { - local status = [ SHELL "\"$(.xsltproc)\" -V" : no-output : exit-status ] ; - if $(status[2]) != "0" - { - errors.user-error "xsltproc: Could not run \"$(.xsltproc)\" -V." ; - } - } -} - -# Returns a non-empty string if a cygwin xsltproc binary was specified. -rule is-cygwin ( ) -{ - freeze-config ; - return $(.is-cygwin) ; -} - -rule .is-cygwin ( xsltproc ) -{ - if [ os.on-windows ] - { - local file = [ path.make [ modules.binding $(__name__) ] ] ; - local dir = [ path.native - [ path.join [ path.parent $(file) ] xsltproc ] ] ; - if [ os.name ] = CYGWIN - { - dir = $(dir:W) ; - } - local command = - "\"$(xsltproc)\" \"$(dir)\\test.xsl\" \"$(dir)\\test.xml\" 2>&1" ; - local status = [ SHELL $(command) : no-output : exit-status ] ; - if $(status[2]) != "0" - { - return true ; - } - } -} - -rule compute-xslt-flags ( target : properties * ) -{ - local flags ; - - # Raw flags. - flags += [ feature.get-values : $(properties) ] ; - - # Translate into command line flags. - for local param in [ feature.get-values : $(properties) ] - { - local namevalue = [ regex.split $(param) "=" ] ; - flags += --stringparam $(namevalue[1]) \"$(namevalue[2])\" ; - } - - # Translate . - for local path in [ feature.get-values : $(properties) ] - { - flags += --path \"$(path:G=)\" ; - } - - # Take care of implicit dependencies. - local other-deps ; - for local dep in [ feature.get-values : $(properties) ] - { - other-deps += [ $(dep:G=).creating-subvariant ] ; - } - - local implicit-target-directories ; - for local dep in [ sequence.unique $(other-deps) ] - { - implicit-target-directories += [ $(dep).all-target-directories ] ; - } - - for local dir in $(implicit-target-directories) - { - flags += --path \"$(dir:T)\" ; - } - - return $(flags) ; -} - - -local rule .xsltproc ( target : source stylesheet : properties * : dirname ? : action ) -{ - freeze-config ; - STYLESHEET on $(target) = $(stylesheet) ; - FLAGS on $(target) += [ compute-xslt-flags $(target) : $(properties) ] ; - NAME on $(target) = $(.xsltproc) ; - - for local catalog in [ feature.get-values : $(properties) ] - { - CATALOG = [ common.variable-setting-command XML_CATALOG_FILES : $(catalog:T) ] ; - } - - if [ os.on-windows ] && ! [ is-cygwin ] - { - action = $(action).windows ; - } - - $(action) $(target) : $(source) ; -} - - -rule xslt ( target : source stylesheet : properties * ) -{ - return [ .xsltproc $(target) : $(source) $(stylesheet) : $(properties) : : xslt-xsltproc ] ; -} - - -rule xslt-dir ( target : source stylesheet : properties * : dirname ) -{ - return [ .xsltproc $(target) : $(source) $(stylesheet) : $(properties) : $(dirname) : xslt-xsltproc-dir ] ; -} - -actions xslt-xsltproc.windows -{ - $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<)" "$(STYLESHEET:W)" "$(>:W)" -} - - -actions xslt-xsltproc bind STYLESHEET -{ - $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<)" "$(STYLESHEET:T)" "$(>:T)" -} - - -actions xslt-xsltproc-dir.windows bind STYLESHEET -{ - $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<:D)/" "$(STYLESHEET:W)" "$(>:W)" -} - - -actions xslt-xsltproc-dir bind STYLESHEET -{ - $(CATALOG) "$(NAME:E=xsltproc)" $(FLAGS) --xinclude -o "$(<:D)/" "$(STYLESHEET:T)" "$(>:T)" -} diff --git a/jam-files/boost-build/tools/xsltproc/included.xsl b/jam-files/boost-build/tools/xsltproc/included.xsl deleted file mode 100644 index ef86394a..00000000 --- a/jam-files/boost-build/tools/xsltproc/included.xsl +++ /dev/null @@ -1,11 +0,0 @@ - - - - diff --git a/jam-files/boost-build/tools/xsltproc/test.xml b/jam-files/boost-build/tools/xsltproc/test.xml deleted file mode 100644 index 57c8ba18..00000000 --- a/jam-files/boost-build/tools/xsltproc/test.xml +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/jam-files/boost-build/tools/xsltproc/test.xsl b/jam-files/boost-build/tools/xsltproc/test.xsl deleted file mode 100644 index a142c91d..00000000 --- a/jam-files/boost-build/tools/xsltproc/test.xsl +++ /dev/null @@ -1,12 +0,0 @@ - - - - - diff --git a/jam-files/boost-build/tools/zlib.jam b/jam-files/boost-build/tools/zlib.jam deleted file mode 100644 index f9138fd5..00000000 --- a/jam-files/boost-build/tools/zlib.jam +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2010 Vladimir Prus. -# -# Use, modification and distribution is subject to the Boost Software -# License Version 1.0. (See accompanying file LICENSE_1_0.txt or -# http://www.boost.org/LICENSE_1_0.txt) - -# Supports the zlib library -# -# After 'using zlib', the following targets are available: -# -# /zlib//zlib -- The zlib library - - -# In addition to direct purpose of supporting zlib, this module also -# serves as canonical example of how third-party condiguration works -# in Boost.Build. The operation is as follows -# -# - For each 'using zlib : condition ... : ...' we create a target alternative -# for zlib, with the specified condition. -# - There's one target alternative for 'zlib' with no specific condition -# properties. -# -# Two invocations of 'using zlib' with the same condition but different -# properties are not permitted, e.g.: -# -# using zlib : condition windows : include foo ; -# using zlib : condition windows : include bar ; -# -# is in error. One exception is for empty condition, 'using' without any -# parameters is overridable. That is: -# -# using zlib ; -# using zlib : include foo ; -# -# Is OK then the first 'using' is ignored. Likewise if the order of the statements -# is reversed. -# -# When 'zlib' target is built, a target alternative is selected as usual for -# Boost.Build. The selected alternative is a custom target class, which: -# -# - calls ac.find-include-path to find header path. If explicit path is provided -# in 'using', only that path is checked, and if no header is found there, error -# is emitted. Otherwise, we check a directory specified using ZLIB_INCLUDE -# environment variable, and failing that, in standard directories. -# [TODO: document sysroot handling] -# - calls ac.find-library to find the library, in an identical fashion. -# - -import project ; -import ac ; -import errors ; -import "class" : new ; -import targets ; - -project.initialize $(__name__) ; -project = [ project.current ] ; -project zlib ; - -header = zlib.h ; -names = z zlib zll zdll ; - -.default-alternative = [ new ac-library zlib : $(project) ] ; -$(.default-alternative).set-header $(header) ; -$(.default-alternative).set-default-names $(names) ; -targets.main-target-alternative $(.default-alternative) ; - -rule init ( * : * ) -{ - if ! $(condition) - { - # Special case the no-condition case so that 'using' without parameters - # can mix with more specific 'using'. - $(.default-alternative).reconfigure $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ; - } - else - { - # FIXME: consider if we should allow overriding definitions for a given - # condition -- e.g. project-config.jam might want to override whatever is - # in user-config.jam. - local mt = [ new ac-library zlib : $(project) - : $(1) : $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ] ; - $(mt).set-header $(header) ; - $(mt).set-default-names $(names) ; - targets.main-target-alternative $(mt) ; - } -} - - - - - - diff --git a/jam-files/boost-build/user-config.jam b/jam-files/boost-build/user-config.jam deleted file mode 100644 index fbbf13fd..00000000 --- a/jam-files/boost-build/user-config.jam +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2003, 2005 Douglas Gregor -# Copyright 2004 John Maddock -# Copyright 2002, 2003, 2004, 2007 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# This file is used to configure your Boost.Build installation. You can modify -# this file in place, or you can place it in a permanent location so that it -# does not get overwritten should you get a new version of Boost.Build. See: -# -# http://www.boost.org/boost-build2/doc/html/bbv2/overview/configuration.html -# -# for documentation about possible permanent locations. - -# This file specifies which toolsets (C++ compilers), libraries, and other -# tools are available. Often, you should be able to just uncomment existing -# example lines and adjust them to taste. The complete list of supported tools, -# and configuration instructions can be found at: -# -# http://boost.org/boost-build2/doc/html/bbv2/reference/tools.html -# - -# This file uses Jam language syntax to describe available tools. Mostly, -# there are 'using' lines, that contain the name of the used tools, and -# parameters to pass to those tools -- where paremeters are separated by -# semicolons. Important syntax notes: -# -# - Both ':' and ';' must be separated from other tokens by whitespace -# - The '\' symbol is a quote character, so when specifying Windows paths you -# should use '/' or '\\' instead. -# -# More details about the syntax can be found at: -# -# http://boost.org/boost-build2/doc/html/bbv2/advanced.html#bbv2.advanced.jam_language -# - -# ------------------ -# GCC configuration. -# ------------------ - -# Configure gcc (default version). -# using gcc ; - -# Configure specific gcc version, giving alternative name to use. -# using gcc : 3.2 : g++-3.2 ; - - -# ------------------- -# MSVC configuration. -# ------------------- - -# Configure msvc (default version, searched for in standard locations and PATH). -# using msvc ; - -# Configure specific msvc version (searched for in standard locations and PATH). -# using msvc : 8.0 ; - - -# ---------------------- -# Borland configuration. -# ---------------------- -# using borland ; - - -# ---------------------- -# STLPort configuration. -# ---------------------- - -# Configure specifying location of STLPort headers. Libraries must be either -# not needed or available to the compiler by default. -# using stlport : : /usr/include/stlport ; - -# Configure specifying location of both headers and libraries explicitly. -# using stlport : : /usr/include/stlport /usr/lib ; - - -# ----------------- -# QT configuration. -# ----------------- - -# Configure assuming QTDIR gives the installation prefix. -# using qt ; - -# Configure with an explicit installation prefix. -# using qt : /usr/opt/qt ; - -# --------------------- -# Python configuration. -# --------------------- - -# Configure specific Python version. -# using python : 3.1 : /usr/bin/python3 : /usr/include/python3.1 : /usr/lib ; diff --git a/jam-files/boost-build/util/__init__.py b/jam-files/boost-build/util/__init__.py deleted file mode 100644 index f80fe70e..00000000 --- a/jam-files/boost-build/util/__init__.py +++ /dev/null @@ -1,136 +0,0 @@ - -import bjam -import re -import types - -# Decorator the specifies bjam-side prototype for a Python function -def bjam_signature(s): - - def wrap(f): - f.bjam_signature = s - return f - - return wrap - -def metatarget(f): - - f.bjam_signature = (["name"], ["sources", "*"], ["requirements", "*"], - ["default_build", "*"], ["usage_requirements", "*"]) - return f - -class cached(object): - - def __init__(self, function): - self.function = function - self.cache = {} - - def __call__(self, *args): - try: - return self.cache[args] - except KeyError: - v = self.function(*args) - self.cache[args] = v - return v - - def __get__(self, instance, type): - return types.MethodType(self, instance, type) - -def unquote(s): - if s and s[0] == '"' and s[-1] == '"': - return s[1:-1] - else: - return s - -_extract_jamfile_and_rule = re.compile("(Jamfile<.*>)%(.*)") - -def qualify_jam_action(action_name, context_module): - - if action_name.startswith("###"): - # Callable exported from Python. Don't touch - return action_name - elif _extract_jamfile_and_rule.match(action_name): - # Rule is already in indirect format - return action_name - else: - ix = action_name.find('.') - if ix != -1 and action_name[:ix] == context_module: - return context_module + '%' + action_name[ix+1:] - - return context_module + '%' + action_name - - -def set_jam_action(name, *args): - - m = _extract_jamfile_and_rule.match(name) - if m: - args = ("set-update-action-in-module", m.group(1), m.group(2)) + args - else: - args = ("set-update-action", name) + args - - return bjam.call(*args) - - -def call_jam_function(name, *args): - - m = _extract_jamfile_and_rule.match(name) - if m: - args = ("call-in-module", m.group(1), m.group(2)) + args - return bjam.call(*args) - else: - return bjam.call(*((name,) + args)) - -__value_id = 0 -__python_to_jam = {} -__jam_to_python = {} - -def value_to_jam(value, methods=False): - """Makes a token to refer to a Python value inside Jam language code. - - The token is merely a string that can be passed around in Jam code and - eventually passed back. For example, we might want to pass PropertySet - instance to a tag function and it might eventually call back - to virtual_target.add_suffix_and_prefix, passing the same instance. - - For values that are classes, we'll also make class methods callable - from Jam. - - Note that this is necessary to make a bit more of existing Jamfiles work. - This trick should not be used to much, or else the performance benefits of - Python port will be eaten. - """ - - global __value_id - - r = __python_to_jam.get(value, None) - if r: - return r - - exported_name = '###_' + str(__value_id) - __value_id = __value_id + 1 - __python_to_jam[value] = exported_name - __jam_to_python[exported_name] = value - - if methods and type(value) == types.InstanceType: - for field_name in dir(value): - field = getattr(value, field_name) - if callable(field) and not field_name.startswith("__"): - bjam.import_rule("", exported_name + "." + field_name, field) - - return exported_name - -def record_jam_to_value_mapping(jam_value, python_value): - __jam_to_python[jam_value] = python_value - -def jam_to_value_maybe(jam_value): - - if type(jam_value) == type(""): - return __jam_to_python.get(jam_value, jam_value) - else: - return jam_value - -def stem(filename): - i = filename.find('.') - if i != -1: - return filename[0:i] - else: - return filename diff --git a/jam-files/boost-build/util/assert.jam b/jam-files/boost-build/util/assert.jam deleted file mode 100644 index abedad52..00000000 --- a/jam-files/boost-build/util/assert.jam +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright 2001, 2002, 2003 Dave Abrahams -# Copyright 2006 Rene Rivera -# Copyright 2002, 2003 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -import errors ; -import modules ; - - -################################################################################ -# -# Private implementation details. -# -################################################################################ - -# Rule added as a replacement for the regular Jam = operator but which does not -# ignore trailing empty string elements. -# -local rule exact-equal-test ( lhs * : rhs * ) -{ - local lhs_extended = $(lhs) xxx ; - local rhs_extended = $(rhs) xxx ; - if $(lhs_extended) = $(rhs_extended) - { - return true ; - } -} - - -# Two lists are considered set-equal if they contain the same elements, ignoring -# duplicates and ordering. -# -local rule set-equal-test ( set1 * : set2 * ) -{ - if ( $(set1) in $(set2) ) && ( $(set2) in $(set1) ) - { - return true ; - } -} - - -################################################################################ -# -# Public interface. -# -################################################################################ - -# Assert the equality of A and B, ignoring trailing empty string elements. -# -rule equal ( a * : b * ) -{ - if $(a) != $(b) - { - errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\" - (ignoring trailing empty strings) ; - } -} - - -# Assert that the result of calling RULE-NAME on the given arguments has a false -# logical value (is either an empty list or all empty strings). -# -rule false ( rule-name args * : * ) -{ - local result ; - module [ CALLER_MODULE ] - { - modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) - : $(7) : $(8) : $(9) ] ; - } - - if $(result) - { - errors.error-skip-frames 3 assertion failure: Expected false result from - "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) : - $(5) : $(6) : $(7) : $(8) : $(9) ] "]" : Got: "[" \"$(result)\" "]" ; - } -} - - -# Assert that ELEMENT is present in LIST. -# -rule "in" ( element : list * ) -{ - if ! $(element) in $(list) - { - errors.error-skip-frames 3 assertion failure: Expected \"$(element)\" in - "[" \"$(list)\" "]" ; - } -} - - -# Assert the inequality of A and B, ignoring trailing empty string elements. -# -rule not-equal ( a * : b * ) -{ - if $(a) = $(b) - { - errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\" - (ignoring trailing empty strings) ; - } -} - - -# Assert that ELEMENT is not present in LIST. -# -rule not-in ( element : list * ) -{ - if $(element) in $(list) - { - errors.error-skip-frames 3 assertion failure: Did not expect - \"$(element)\" in "[" \"$(list)\" "]" ; - } -} - - -# Assert the inequality of A and B as sets. -# -rule not-set-equal ( a * : b * ) -{ - if [ set-equal-test $(a) : $(b) ] - { - errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]" - and "[" \"$(b)\" "]" to not be equal as sets ; - } -} - - -# Assert that A and B are not exactly equal, not ignoring trailing empty string -# elements. -# -rule not-exact-equal ( a * : b * ) -{ - if [ exact-equal-test $(a) : $(b) ] - { - errors.error-skip-frames 3 assertion failure: \"$(a)\" "!=" \"$(b)\" ; - } -} - - -# Assert that EXPECTED is the result of calling RULE-NAME with the given -# arguments. -# -rule result ( expected * : rule-name args * : * ) -{ - local result ; - module [ CALLER_MODULE ] - { - modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7) - : $(8) : $(9) ] ; - } - - if ! [ exact-equal-test $(result) : $(expected) ] - { - errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [ - errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : - $(9) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "[" - \"$(result)\" "]" ; - } -} - - -# Assert that EXPECTED is set-equal (i.e. duplicates and ordering are ignored) -# to the result of calling RULE-NAME with the given arguments. Note that rules -# called this way may accept at most 8 parameters. -# -rule result-set-equal ( expected * : rule-name args * : * ) -{ - local result ; - module [ CALLER_MODULE ] - { - modules.poke assert : result : [ $(2) : $(3) : $(4) : $(5) : $(6) : $(7) - : $(8) : $(9) ] ; - } - - if ! [ set-equal-test $(result) : $(expected) ] - { - errors.error-skip-frames 3 assertion failure: "[" $(rule-name) [ - errors.lol->list $(args) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : - $(9) ] "]" : Expected: "[" \"$(expected)\" "]" : Got: "[" - \"$(result)\" "]" ; - } -} - - -# Assert the equality of A and B as sets. -# -rule set-equal ( a * : b * ) -{ - if ! [ set-equal-test $(a) : $(b) ] - { - errors.error-skip-frames 3 assertion failure: Expected "[" \"$(a)\" "]" - and "[" \"$(b)\" "]" to be equal as sets ; - } -} - - -# Assert that the result of calling RULE-NAME on the given arguments has a true -# logical value (is neither an empty list nor all empty strings). -# -rule true ( rule-name args * : * ) -{ - local result ; - module [ CALLER_MODULE ] - { - modules.poke assert : result : [ $(1) : $(2) : $(3) : $(4) : $(5) : $(6) - : $(7) : $(8) : $(9) ] ; - } - - if ! $(result) - { - errors.error-skip-frames 3 assertion failure: Expected true result from - "[" $(rule-name) [ errors.lol->list $(args) : $(2) : $(3) : $(4) : - $(5) : $(6) : $(7) : $(8) : $(9) ] "]" ; - } -} - - -# Assert the exact equality of A and B, not ignoring trailing empty string -# elements. -# -rule exact-equal ( a * : b * ) -{ - if ! [ exact-equal-test $(a) : $(b) ] - { - errors.error-skip-frames 3 assertion failure: \"$(a)\" "==" \"$(b)\" ; - } -} - - -# Assert that the given variable is not an empty list. -# -rule variable-not-empty ( name ) -{ - local value = [ modules.peek [ CALLER_MODULE ] : $(name) ] ; - if ! $(value)-is-not-empty - { - errors.error-skip-frames 3 assertion failure: Expected variable - \"$(name)\" not to be an empty list ; - } -} - - -rule __test__ ( ) -{ - # Helper rule used to avoid test duplication related to different list - # equality test rules. - # - local rule run-equality-test ( equality-assert : ignore-trailing-empty-strings ? ) - { - local not-equality-assert = not-$(equality-assert) ; - - # When the given equality test is expected to ignore trailing empty - # strings some of the test results should be inverted. - local not-equality-assert-i = not-$(equality-assert) ; - if $(ignore-trailing-empty-strings) - { - not-equality-assert-i = $(equality-assert) ; - } - - $(equality-assert) : ; - $(equality-assert) "" "" : "" "" ; - $(not-equality-assert-i) : "" "" ; - $(equality-assert) x : x ; - $(not-equality-assert) : x ; - $(not-equality-assert) "" : x ; - $(not-equality-assert) "" "" : x ; - $(not-equality-assert-i) x : x "" ; - $(equality-assert) x "" : x "" ; - $(not-equality-assert) x : "" x ; - $(equality-assert) "" x : "" x ; - - $(equality-assert) 1 2 3 : 1 2 3 ; - $(not-equality-assert) 1 2 3 : 3 2 1 ; - $(not-equality-assert) 1 2 3 : 1 5 3 ; - $(not-equality-assert) 1 2 3 : 1 "" 3 ; - $(not-equality-assert) 1 2 3 : 1 1 2 3 ; - $(not-equality-assert) 1 2 3 : 1 2 2 3 ; - $(not-equality-assert) 1 2 3 : 5 6 7 ; - - # Extra variables used here just to make sure Boost Jam or Boost Build - # do not handle lists with empty strings differently depending on - # whether they are literals or stored in variables. - - local empty = ; - local empty-strings = "" "" ; - local x-empty-strings = x "" "" ; - local empty-strings-x = "" "" x ; - - $(equality-assert) : $(empty) ; - $(not-equality-assert-i) "" : $(empty) ; - $(not-equality-assert-i) "" "" : $(empty) ; - $(not-equality-assert-i) : $(empty-strings) ; - $(not-equality-assert-i) "" : $(empty-strings) ; - $(equality-assert) "" "" : $(empty-strings) ; - $(equality-assert) $(empty) : $(empty) ; - $(equality-assert) $(empty-strings) : $(empty-strings) ; - $(not-equality-assert-i) $(empty) : $(empty-strings) ; - $(equality-assert) $(x-empty-strings) : $(x-empty-strings) ; - $(equality-assert) $(empty-strings-x) : $(empty-strings-x) ; - $(not-equality-assert) $(empty-strings-x) : $(x-empty-strings) ; - $(not-equality-assert-i) x : $(x-empty-strings) ; - $(not-equality-assert) x : $(empty-strings-x) ; - $(not-equality-assert-i) x : $(x-empty-strings) ; - $(not-equality-assert-i) x "" : $(x-empty-strings) ; - $(equality-assert) x "" "" : $(x-empty-strings) ; - $(not-equality-assert) x : $(empty-strings-x) ; - $(not-equality-assert) "" x : $(empty-strings-x) ; - $(equality-assert) "" "" x : $(empty-strings-x) ; - } - - - # --------------- - # Equality tests. - # --------------- - - run-equality-test equal : ignore-trailing-empty-strings ; - run-equality-test exact-equal ; - - - # ------------------------- - # assert.set-equal() tests. - # ------------------------- - - set-equal : ; - not-set-equal "" "" : ; - set-equal "" "" : "" ; - set-equal "" "" : "" "" ; - set-equal a b c : a b c ; - set-equal a b c : b c a ; - set-equal a b c a : a b c ; - set-equal a b c : a b c a ; - not-set-equal a b c : a b c d ; - not-set-equal a b c d : a b c ; -} diff --git a/jam-files/boost-build/util/container.jam b/jam-files/boost-build/util/container.jam deleted file mode 100644 index dd496393..00000000 --- a/jam-files/boost-build/util/container.jam +++ /dev/null @@ -1,339 +0,0 @@ -# Copyright 2003 Dave Abrahams -# Copyright 2002, 2003 Rene Rivera -# Copyright 2002, 2003, 2004 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Various container classes. - -# Base for container objects. This lets us construct recursive structures. That -# is containers with containers in them, specifically so we can tell literal -# values from node values. -# -class node -{ - rule __init__ ( - value ? # Optional value to set node to initially. - ) - { - self.value = $(value) ; - } - - # Set the value of this node, passing nothing will clear it. - # - rule set ( value * ) - { - self.value = $(value) ; - } - - # Get the value of this node. - # - rule get ( ) - { - return $(self.value) ; - } -} - - -# A simple vector. Interface mimics the C++ std::vector and std::list, with the -# exception that indices are one (1) based to follow Jam standard. -# -# TODO: Possibly add assertion checks. -# -class vector : node -{ - import numbers ; - import utility ; - import sequence ; - - rule __init__ ( - values * # Initial contents of vector. - ) - { - node.__init__ ; - self.value = $(values) ; - } - - # Get the value of the first element. - # - rule front ( ) - { - return $(self.value[1]) ; - } - - # Get the value of the last element. - # - rule back ( ) - { - return $(self.value[-1]) ; - } - - # Get the value of the element at the given index, one based. Access to - # elements of recursive structures is supported directly. Specifying - # additional index values recursively accesses the elements as containers. - # For example: [ $(v).at 1 : 2 ] would retrieve the second element of our - # first element, assuming the first element is a container. - # - rule at ( - index # The element index, one based. - : * # Additional indices to access recursively. - ) - { - local r = $(self.value[$(index)]) ; - if $(2) - { - r = [ $(r).at $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ] ; - } - return $(r) ; - } - - # Get the value contained in the given element. This has the same - # functionality and interface as "at" but in addition gets the value of the - # referenced element, assuming it is a "node". - # - rule get-at ( - index # The element index, one based. - : * # Additional indices to access recursively. - ) - { - local r = $(self.value[$(index)]) ; - if $(2) - { - r = [ $(r).at $(2) : $(3) : $(4) : $(5) : $(6) : $(7) : $(8) : $(9) ] ; - } - return [ $(r).get ] ; - } - - # Insert the given value into the front of the vector pushing the rest of - # the elements back. - # - rule push-front ( - value # Value to become first element. - ) - { - self.value = $(value) $(self.value) ; - } - - # Remove the front element from the vector. Does not return the value. No - # effect if vector is empty. - # - rule pop-front ( ) - { - self.value = $(self.value[2-]) ; - } - - # Add the given value at the end of the vector. - # - rule push-back ( - value # Value to become back element. - ) - { - self.value += $(value) ; - } - - # Remove the back element from the vector. Does not return the value. No - # effect if vector is empty. - # - rule pop-back ( ) - { - self.value = $(self.value[1--2]) ; - } - - # Insert the given value at the given index, one based. The values at and to - # the right of the index are pushed back to make room for the new value. - # If the index is passed the end of the vector the element is added to the - # end. - # - rule insert ( - index # The index to insert at, one based. - : value # The value to insert. - ) - { - local left = $(self.value[1-$(index)]) ; - local right = $(self.value[$(index)-]) ; - if $(right)-is-not-empty - { - left = $(left[1--2]) ; - } - self.value = $(left) $(value) $(right) ; - } - - # Remove one or more elements from the vector. The range is inclusive, and - # not specifying an end is equivalent to the [start, start] range. - # - rule erase ( - start # Index of first element to remove. - end ? # Optional, index of last element to remove. - ) - { - end ?= $(start) ; - local left = $(self.value[1-$(start)]) ; - left = $(left[1--2]) ; - local right = $(self.value[$(end)-]) ; - right = $(right[2-]) ; - self.value = $(left) $(right) ; - } - - # Remove all elements from the vector. - # - rule clear ( ) - { - self.value = ; - } - - # The number of elements in the vector. - # - rule size ( ) - { - return [ sequence.length $(self.value) ] ; - } - - # Returns "true" if there are NO elements in the vector, empty otherwise. - # - rule empty ( ) - { - if ! $(self.value)-is-not-empty - { - return true ; - } - } - - # Returns the textual representation of content. - # - rule str ( ) - { - return "[" [ sequence.transform utility.str : $(self.value) ] "]" ; - } - - # Sorts the vector inplace, calling 'utility.less' for comparisons. - # - rule sort ( ) - { - self.value = [ sequence.insertion-sort $(self.value) : utility.less ] ; - } - - # Returns true if content is equal to the content of other vector. Uses - # 'utility.equal' for comparison. - # - rule equal ( another ) - { - local mismatch ; - local size = [ size ] ; - if $(size) = [ $(another).size ] - { - for local i in [ numbers.range 1 $(size) ] - { - if ! [ utility.equal [ at $(i) ] [ $(another).at $(i) ] ] - { - mismatch = true ; - } - } - } - else - { - mismatch = true ; - } - - if ! $(mismatch) - { - return true ; - } - } -} - - -rule __test__ ( ) -{ - import assert ; - import "class" : new ; - - local v1 = [ new vector ] ; - assert.true $(v1).equal $(v1) ; - assert.true $(v1).empty ; - assert.result 0 : $(v1).size ; - assert.result "[" "]" : $(v1).str ; - $(v1).push-back b ; - $(v1).push-front a ; - assert.result "[" a b "]" : $(v1).str ; - assert.result a : $(v1).front ; - assert.result b : $(v1).back ; - $(v1).insert 2 : d ; - $(v1).insert 2 : c ; - $(v1).insert 4 : f ; - $(v1).insert 4 : e ; - $(v1).pop-back ; - assert.result 5 : $(v1).size ; - assert.result d : $(v1).at 3 ; - $(v1).pop-front ; - assert.result c : $(v1).front ; - assert.false $(v1).empty ; - $(v1).erase 3 4 ; - assert.result 2 : $(v1).size ; - - local v2 = [ new vector q w e r t y ] ; - assert.result 6 : $(v2).size ; - $(v1).push-back $(v2) ; - assert.result 3 : $(v1).size ; - local v2-alias = [ $(v1).back ] ; - assert.result e : $(v2-alias).at 3 ; - $(v1).clear ; - assert.true $(v1).empty ; - assert.false $(v2-alias).empty ; - $(v2).pop-back ; - assert.result t : $(v2-alias).back ; - - local v3 = [ new vector ] ; - $(v3).push-back [ new vector 1 2 3 4 5 ] ; - $(v3).push-back [ new vector a b c ] ; - assert.result "[" "[" 1 2 3 4 5 "]" "[" a b c "]" "]" : $(v3).str ; - $(v3).push-back [ new vector [ new vector x y z ] [ new vector 7 8 9 ] ] ; - assert.result 1 : $(v3).at 1 : 1 ; - assert.result b : $(v3).at 2 : 2 ; - assert.result a b c : $(v3).get-at 2 ; - assert.result 7 8 9 : $(v3).get-at 3 : 2 ; - - local v4 = [ new vector 4 3 6 ] ; - $(v4).sort ; - assert.result 3 4 6 : $(v4).get ; - assert.false $(v4).equal $(v3) ; - - local v5 = [ new vector 3 4 6 ] ; - assert.true $(v4).equal $(v5) ; - # Check that vectors of different sizes are considered non-equal. - $(v5).pop-back ; - assert.false $(v4).equal $(v5) ; - - local v6 = [ new vector [ new vector 1 2 3 ] ] ; - assert.true $(v6).equal [ new vector [ new vector 1 2 3 ] ] ; - - local v7 = [ new vector 111 222 333 ] ; - assert.true $(v7).equal $(v7) ; - $(v7).insert 4 : 444 ; - assert.result 111 222 333 444 : $(v7).get ; - $(v7).insert 999 : xxx ; - assert.result 111 222 333 444 xxx : $(v7).get ; - - local v8 = [ new vector "" "" "" ] ; - assert.true $(v8).equal $(v8) ; - assert.false $(v8).empty ; - assert.result 3 : $(v8).size ; - assert.result "" : $(v8).at 1 ; - assert.result "" : $(v8).at 2 ; - assert.result "" : $(v8).at 3 ; - assert.result : $(v8).at 4 ; - $(v8).insert 2 : 222 ; - assert.result 4 : $(v8).size ; - assert.result "" 222 "" "" : $(v8).get ; - $(v8).insert 999 : "" ; - assert.result 5 : $(v8).size ; - assert.result "" 222 "" "" "" : $(v8).get ; - $(v8).insert 999 : xxx ; - assert.result 6 : $(v8).size ; - assert.result "" 222 "" "" "" xxx : $(v8).get ; - - # Regression test for a bug causing vector.equal to compare only the first - # and the last element in the given vectors. - local v9 = [ new vector 111 xxx 222 ] ; - local v10 = [ new vector 111 yyy 222 ] ; - assert.false $(v9).equal $(v10) ; -} diff --git a/jam-files/boost-build/util/doc.jam b/jam-files/boost-build/util/doc.jam deleted file mode 100644 index a7515588..00000000 --- a/jam-files/boost-build/util/doc.jam +++ /dev/null @@ -1,997 +0,0 @@ -# Copyright 2002, 2005 Dave Abrahams -# Copyright 2002, 2003, 2006 Rene Rivera -# Copyright 2003 Vladimir Prus -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) - -# Documentation system, handles --help requests. -# It defines rules that attach documentation to modules, rules, and variables. -# Collects and generates documentation for the various parts of the build -# system. The documentation is collected from comments integrated into the code. - -import modules ; -import print ; -import set ; -import container ; -import "class" ; -import sequence ; -import path ; - - -# The type of output to generate. -# "console" is formated text echoed to the console (the default); -# "text" is formated text appended to the output file; -# "html" is HTML output to the file. -# -help-output = console ; - - -# The file to output documentation to when generating "text" or "html" help. -# This is without extension as the extension is determined by the type of -# output. -# -help-output-file = help ; - -# Whether to include local rules in help output. -# -.option.show-locals ?= ; - -# When showing documentation for a module, whether to also generate -# automatically the detailed docs for each item in the module. -# -.option.detailed ?= ; - -# Generate debug output as the help is generated and modules are parsed. -# -.option.debug ?= ; - -# Enable or disable a documentation option. -# -local rule set-option ( - option # The option name. - : value ? # Enabled (non-empty), or disabled (empty) -) -{ - .option.$(option) = $(value) ; -} - - -# Set the type of output. -# -local rule set-output ( type ) -{ - help-output = $(type) ; -} - - -# Set the output to a file. -# -local rule set-output-file ( file ) -{ - help-output-file = $(file) ; -} - - -# Extracts the brief comment from a complete comment. The brief comment is the -# first sentence. -# -local rule brief-comment ( - docs * # The comment documentation. -) -{ - local d = $(docs:J=" ") ; - local p = [ MATCH ".*([.])$" : $(d) ] ; - if ! $(p) { d = $(d)"." ; } - d = $(d)" " ; - local m = [ MATCH "^([^.]+[.])(.*)" : $(d) ] ; - local brief = $(m[1]) ; - while $(m[2]) && [ MATCH "^([^ ])" : $(m[2]) ] - { - m = [ MATCH "^([^.]+[.])(.*)" : $(m[2]) ] ; - brief += $(m[1]) ; - } - return $(brief:J="") ; -} - - -# Specifies the documentation for the current module. -# -local rule set-module-doc ( - module-name ? # The name of the module to document. - : docs * # The documentation for the module. -) -{ - module-name ?= * ; - - $(module-name).brief = [ brief-comment $(docs) ] ; - $(module-name).docs = $(docs) ; - - if ! $(module-name) in $(documented-modules) - { - documented-modules += $(module-name) ; - } -} - - -# Specifies the documentation for the current module. -# -local rule set-module-copyright ( - module-name ? # The name of the module to document. - : copyright * # The copyright for the module. -) -{ - module-name ?= * ; - - $(module-name).copy-brief = [ brief-comment $(copyright) ] ; - $(module-name).copy-docs = $(docs) ; - - if ! $(module-name) in $(documented-modules) - { - documented-modules += $(module-name) ; - } -} - - -# Specifies the documentation for a rule in the current module. If called in the -# global module, this documents a global rule. -# -local rule set-rule-doc ( - name # The name of the rule. - module-name ? # The name of the module to document. - is-local ? # Whether the rule is local to the module. - : docs * # The documentation for the rule. -) -{ - module-name ?= * ; - - $(module-name).$(name).brief = [ brief-comment $(docs) ] ; - $(module-name).$(name).docs = $(docs) ; - $(module-name).$(name).is-local = $(is-local) ; - - if ! $(name) in $($(module-name).rules) - { - $(module-name).rules += $(name) ; - } -} - - -# Specify a class, will turn a rule into a class. -# -local rule set-class-doc ( - name # The name of the class. - module-name ? # The name of the module to document. - : super-name ? # The super class name. -) -{ - module-name ?= * ; - - $(module-name).$(name).is-class = true ; - $(module-name).$(name).super-name = $(super-name) ; - $(module-name).$(name).class-rules = - [ MATCH "^($(name)[.].*)" : $($(module-name).rules) ] ; - $(module-name).$($(module-name).$(name).class-rules).is-class-rule = true ; - - $(module-name).classes += $(name) ; - $(module-name).class-rules += $($(module-name).$(name).class-rules) ; - $(module-name).rules = - [ set.difference $($(module-name).rules) : - $(name) $($(module-name).$(name).class-rules) ] ; -} - - -# Set the argument call signature of a rule. -# -local rule set-rule-arguments-signature ( - name # The name of the rule. - module-name ? # The name of the module to document. - : signature * # The arguments signature. -) -{ - module-name ?= * ; - - $(module-name).$(name).signature = $(signature) ; -} - - -# Specifies the documentation for an argument of a rule. -# -local rule set-argument-doc ( - name # The name of the argument. - qualifier # Argument syntax qualifier, "*", "+", etc. - rule-name # The name of the rule. - module-name ? # THe optional name of the module. - : docs * # The documentation. -) -{ - module-name ?= * ; - - $(module-name).$(rule-name).args.$(name).qualifier = $(qualifier) ; - $(module-name).$(rule-name).args.$(name).docs = $(docs) ; - - if ! $(name) in $($(module-name).$(rule-name).args) - { - $(module-name).$(rule-name).args += $(name) ; - } -} - - -# Specifies the documentation for a variable in the current module. If called in -# the global module, the global variable is documented. -# -local rule set-variable-doc ( - name # The name of the variable. - default # The default value. - initial # The initial value. - module-name ? # The name of the module to document. - : docs * # The documentation for the variable. -) -{ - module-name ?= * ; - - $(module-name).$(name).brief = [ brief-comment $(docs) ] ; - $(module-name).$(name).default = $(default) ; - $(module-name).$(name).initial = $(initial) ; - $(module-name).$(name).docs = $(docs) ; - - if ! $(name) in $($(module-name).variables) - { - $(module-name).variables += $(name) ; - } -} - - -# Generates a general description of the documentation and help system. -# -local rule print-help-top ( ) -{ - print.section "General command line usage" ; - - print.text " bjam [options] [properties] [targets] - - Options, properties and targets can be specified in any order. - " ; - - print.section "Important Options" ; - - print.list-start ; - print.list-item "--clean Remove targets instead of building" ; - print.list-item "-a Rebuild everything" ; - print.list-item "-n Don't execute the commands, only print them" ; - print.list-item "-d+2 Show commands as they are executed" ; - print.list-item "-d0 Supress all informational messages" ; - print.list-item "-q Stop at first error" ; - print.list-item "--debug-configuration Diagnose configuration" ; - print.list-item "--debug-building Report which targets are built with what properties" ; - print.list-item "--debug-generator Diagnose generator search/execution" ; - print.list-end ; - - print.section "Further Help" - The following options can be used to obtain additional documentation. - ; - - print.list-start ; - print.list-item "--help-options Print more obscure command line options." ; - print.list-item "--help-internal Boost.Build implementation details." ; - print.list-item "--help-doc-options Implementation details doc formatting." ; - print.list-end ; -} - - -# Generate Jam/Boost.Jam command usage information. -# -local rule print-help-usage ( ) -{ - print.section "Boost.Jam Usage" - "bjam [ options... ] targets..." - ; - print.list-start ; - print.list-item -a; - Build all targets, even if they are current. ; - print.list-item -fx; - Read '"x"' as the Jamfile for building instead of searching for the - Boost.Build system. ; - print.list-item -jx; - Run up to '"x"' commands concurrently. ; - print.list-item -n; - Do not execute build commands. Instead print out the commands as they - would be executed if building. ; - print.list-item -ox; - Output the used build commands to file '"x"'. ; - print.list-item -q; - Quit as soon as a build failure is encountered. Without this option - Boost.Jam will continue building as many targets as it can. - print.list-item -sx=y; - Sets a Jam variable '"x"' to the value '"y"', overriding any value that - variable would have from the environment. ; - print.list-item -tx; - Rebuild the target '"x"', even if it is up-to-date. ; - print.list-item -v; - Display the version of bjam. ; - print.list-item --x; - Any option not explicitly handled by Boost.Jam remains available to - build scripts using the '"ARGV"' variable. ; - print.list-item -dn; - Enables output of diagnostic messages. The debug level '"n"' and all - below it are enabled by this option. ; - print.list-item -d+n; - Enables output of diagnostic messages. Only the output for debug level - '"n"' is enabled. ; - print.list-end ; - print.section "Debug Levels" - Each debug level shows a different set of information. Usually with - higher levels producing more verbose information. The following levels - are supported: ; - print.list-start ; - print.list-item 0; - Turn off all diagnostic output. Only errors are reported. ; - print.list-item 1; - Show the actions taken for building targets, as they are executed. ; - print.list-item 2; - Show "quiet" actions and display all action text, as they are executed. ; - print.list-item 3; - Show dependency analysis, and target/source timestamps/paths. ; - print.list-item 4; - Show arguments of shell invocations. ; - print.list-item 5; - Show rule invocations and variable expansions. ; - print.list-item 6; - Show directory/header file/archive scans, and attempts at binding to targets. ; - print.list-item 7; - Show variable settings. ; - print.list-item 8; - Show variable fetches, variable expansions, and evaluation of '"if"' expressions. ; - print.list-item 9; - Show variable manipulation, scanner tokens, and memory usage. ; - print.list-item 10; - Show execution times for rules. ; - print.list-item 11; - Show parsing progress of Jamfiles. ; - print.list-item 12; - Show graph for target dependencies. ; - print.list-item 13; - Show changes in target status (fate). ; - print.list-end ; -} - - -# Generates description of options controlling the help system. This -# automatically reads the options as all variables in the doc module of the form -# ".option.*". -# -local rule print-help-options ( - module-name # The doc module. -) -{ - print.section "Help Options" - These are all the options available for enabling or disabling to control - the help system in various ways. Options can be enabled or disabled with - '"--help-enable-