dynamic fsa ff, factory for fsa and ff shares code, factory moved to ff_factory.cc

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@483 ec762483-ff6d-05da-a07a-a48fb63a330f

17 files changed, 318 insertions, 144 deletions

diff --git a/decoder/apply_fsa_models.h b/decoder/apply_fsa_models.h
index d22397e3..0a8615b5 100755
--- a/decoder/apply_fsa_models.h
+++ b/decoder/apply_fsa_models.h
@@ -1,7 +1,7 @@
 #ifndef _APPLY_FSA_MODELS_H_
 #define _APPLY_FSA_MODELS_H_
 
-//#include "ff_fsa_dynamic.h"
+#include "ff_fsa_dynamic.h"
 
 struct FsaFeatureFunction;
 struct Hypergraph;
diff --git a/decoder/cdec.cc b/decoder/cdec.cc
index 876dee18..e896a484 100644
--- a/decoder/cdec.cc
+++ b/decoder/cdec.cc
@@ -34,6 +34,7 @@
 #include "exp_semiring.h"
 #include "sentence_metadata.h"
 #include "../vest/scorer.h"
+#include "apply_fsa_models.h"
 
 using namespace std;
 using namespace std::tr1;
@@ -69,15 +70,27 @@ shared_ptr<FeatureFunction> make_ff(string const& ffp,bool verbose_feature_funct
   cerr << "Feature: " << ff;
   if (param.size() > 0) cerr << " (with config parameters '" << param << "')\n";
   else cerr << " (no config parameters)\n";
-  shared_ptr<FeatureFunction> pf = global_ff_registry->Create(ff, param);
-  if (!pf)
-    exit(1);
+  shared_ptr<FeatureFunction> pf = ff_registry.Create(ff, param);
+  if (!pf) exit(1);
   int nbyte=pf->NumBytesContext();
   if (verbose_feature_functions)
     cerr<<"State is "<<nbyte<<" bytes for "<<pre<<"feature "<<ffp<<endl;
   return pf;
 }
 
+shared_ptr<FsaFeatureFunction> make_fsa_ff(string const& ffp,bool verbose_feature_functions,char const* pre="") {
+  string ff, param;
+  SplitCommandAndParam(ffp, &ff, &param);
+  cerr << "FSA Feature: " << ff;
+  if (param.size() > 0) cerr << " (with config parameters '" << param << "')\n";
+  else cerr << " (no config parameters)\n";
+  shared_ptr<FsaFeatureFunction> pf = fsa_ff_registry.Create(ff, param);
+  if (!pf) exit(1);
+  if (verbose_feature_functions)
+    cerr<<"State is "<<pf->state_bytes()<<" bytes for "<<pre<<"feature "<<ffp<<endl;
+  return pf;
+}
+
 // print just the --long_opt names suitable for bash compgen
 void print_options(std::ostream &out,po::options_description const& opts) {
   typedef std::vector< shared_ptr<po::option_description> > Ds;
@@ -106,6 +119,7 @@ void InitCommandLine(int argc, char** argv, OracleBleu &ob, po::variables_map* c
     ("warn_0_weight","Warn about any feature id that has a 0 weight (this is perfectly safe if you intend 0 weight, though)")
         ("no_freeze_feature_set,Z", "Do not freeze feature set after reading feature weights file")
         ("feature_function,F",po::value<vector<string> >()->composing(), "Additional feature function(s) (-L for list)")
+        ("fsa_feature_function",po::value<vector<string> >()->composing(), "Additional FSA feature function(s) (-L for list)")
         ("list_feature_functions,L","List available feature functions")
         ("add_pass_through_rules,P","Add rules to translate OOV words as themselves")
 	("k_best,k",po::value<int>(),"Extract the k best derivations")
@@ -185,13 +199,15 @@ void InitCommandLine(int argc, char** argv, OracleBleu &ob, po::variables_map* c
 
   if (conf.count("list_feature_functions")) {
     cerr << "Available feature functions (specify with -F; describe with -u FeatureName):\n";
-    global_ff_registry->DisplayList();
+    ff_registry.DisplayList();
+    cerr << "Available feature functions (specify with --fsa_feature_function):\n";
+    fsa_ff_registry.DisplayList();
     cerr << endl;
     exit(1);
   }
 
   if (conf.count("usage")) {
-    cout<<global_ff_registry->usage(str("usage",conf),true,true)<<endl;
+    cout<<ff_registry.usage(str("usage",conf),true,true)<<endl;
     exit(0);
   }
   if (conf.count("help")) {
@@ -358,8 +374,17 @@ void show_models(po::variables_map const& conf,ModelSet &ms,char const* header)
 }
 
 
+template <class V>
+bool store_conf(po::variables_map const& conf,std::string const& name,V *v) {
+  if (conf.count(name)) {
+    *v=conf[name].as<V>();
+    return true;
+  }
+  return false;
+}
+
+
 int main(int argc, char** argv) {
-  global_ff_registry.reset(new FFRegistry);
   register_feature_functions();
   po::variables_map conf;
   OracleBleu oracle;
@@ -441,7 +466,6 @@ int main(int argc, char** argv) {
 
   // set up additional scoring features
   vector<shared_ptr<FeatureFunction> > pffs,prelm_only_ffs;
-
   vector<const FeatureFunction*> late_ffs,prelm_ffs;
   if (conf.count("feature_function") > 0) {
     const vector<string>& add_ffs = conf["feature_function"].as<vector<string> >();
@@ -454,7 +478,7 @@ int main(int argc, char** argv) {
           prelm_ffs.push_back(p);
         else
           cerr << "Excluding stateful feature from prelm pruning: "<<add_ffs[i]<<endl;
-}
+      }
     }
   }
   if (conf.count("prelm_feature_function") > 0) {
@@ -465,6 +489,17 @@ int main(int argc, char** argv) {
     }
   }
 
+  vector<shared_ptr<FsaFeatureFunction> > fsa_ffs;
+  vector<string> fsa_names;
+  store_conf(conf,"fsa_feature_function",&fsa_names);
+  if (fsa_ffs.size()>1) {
+    //FIXME: support N fsa ffs.
+    cerr<<"Only the first fsa FF will be used (FIXME).\n";
+    fsa_names.resize(1);
+    for (int i=0;i<fsa_names.size();++i)
+      fsa_ffs.push_back(make_fsa_ff(fsa_names[i],verbose_feature_functions,"FSA "));
+  }
+
   if (late_freeze) {
     cerr << "Late freezing feature set (use --no_freeze_feature_set to prevent)." << endl;
     FD::Freeze(); // this means we can't see the feature names of not-weighted features
diff --git a/decoder/cdec_ff.cc b/decoder/cdec_ff.cc
index fc64e5c4..39f018e1 100644
--- a/decoder/cdec_ff.cc
+++ b/decoder/cdec_ff.cc
@@ -10,35 +10,39 @@
 #include "ff_bleu.h"
 #include "ff_lm_fsa.h"
 #include "ff_sample_fsa.h"
-
-boost::shared_ptr<FFRegistry> global_ff_registry;
+#include "ff_register.h"
 
 void register_feature_functions() {
-  global_ff_registry->Register(new FFFactory<LanguageModel>);
-  global_ff_registry->Register(new FFFactory<FeatureFunctionFromFsa<LanguageModelFsa> >); // same as LM but using fsa wrapper
+  RegisterFF<LanguageModel>();
+  RegisterFsaImpl<LanguageModelFsa>(true,false); // same as LM but using fsa wrapper
+
+  RegisterFF<WordPenalty>();
+  RegisterFF<SourceWordPenalty>();
+  RegisterFF<ArityPenalty>();
+  RegisterFF<BLEUModel>();
 
-  global_ff_registry->Register(new FFFactory<WordPenaltyFromFsa>); // same as WordPenalty, but implemented using ff_fsa
-  global_ff_registry->Register(new FFFactory<FeatureFunctionFromFsa<LongerThanPrev> >);
-  global_ff_registry->Register(new FFFactory<FeatureFunctionFromFsa<ShorterThanPrev> >);
+  //TODO: worthless example target FSA ffs.  remove later
+  ff_registry.Register(new FFFactory<WordPenaltyFromFsa>); // same as WordPenalty, but implemented using ff_fsa
+  ff_registry.Register(new FFFactory<FeatureFunctionFromFsa<SameFirstLetter> >);
+  ff_registry.Register(new FFFactory<FeatureFunctionFromFsa<LongerThanPrev> >);
+  ff_registry.Register(new FFFactory<FeatureFunctionFromFsa<ShorterThanPrev> >);
 
-  //TODO: use for all features the new Register which requires usage(...)
+  //TODO: use for all features the new Register which requires static FF::usage(false,false) give name
 #ifdef HAVE_RANDLM
-  global_ff_registry->Register("RandLM", new FFFactory<LanguageModelRandLM>);
+  ff_registry.Register("RandLM", new FFFactory<LanguageModelRandLM>);
 #endif
-  global_ff_registry->Register(new FFFactory<WordPenalty>);
-  global_ff_registry->Register(new FFFactory<SourceWordPenalty>);
-  global_ff_registry->Register(new FFFactory<ArityPenalty>);
-  global_ff_registry->Register(new FFFactory<BLEUModel>);
-  global_ff_registry->Register("RuleShape", new FFFactory<RuleShapeFeatures>);
-  global_ff_registry->Register("RelativeSentencePosition", new FFFactory<RelativeSentencePosition>);
-  global_ff_registry->Register("Model2BinaryFeatures", new FFFactory<Model2BinaryFeatures>);
-  global_ff_registry->Register("MarkovJump", new FFFactory<MarkovJump>);
-  global_ff_registry->Register("MarkovJumpFClass", new FFFactory<MarkovJumpFClass>);
-  global_ff_registry->Register("SourcePOSBigram", new FFFactory<SourcePOSBigram>);
-  global_ff_registry->Register("BlunsomSynchronousParseHack", new FFFactory<BlunsomSynchronousParseHack>);
-  global_ff_registry->Register("AlignerResults", new FFFactory<AlignerResults>);
-  global_ff_registry->Register("CSplit_BasicFeatures", new FFFactory<BasicCSplitFeatures>);
-  global_ff_registry->Register("CSplit_ReverseCharLM", new FFFactory<ReverseCharLMCSplitFeature>);
-  global_ff_registry->Register("Tagger_BigramIdentity", new FFFactory<Tagger_BigramIdentity>);
-  global_ff_registry->Register("LexicalPairIdentity", new FFFactory<LexicalPairIdentity>);
+  ff_registry.Register("RuleShape", new FFFactory<RuleShapeFeatures>);
+  ff_registry.Register("RelativeSentencePosition", new FFFactory<RelativeSentencePosition>);
+  ff_registry.Register("Model2BinaryFeatures", new FFFactory<Model2BinaryFeatures>);
+  ff_registry.Register("MarkovJump", new FFFactory<MarkovJump>);
+  ff_registry.Register("MarkovJumpFClass", new FFFactory<MarkovJumpFClass>);
+  ff_registry.Register("SourcePOSBigram", new FFFactory<SourcePOSBigram>);
+  ff_registry.Register("BlunsomSynchronousParseHack", new FFFactory<BlunsomSynchronousParseHack>);
+  ff_registry.Register("AlignerResults", new FFFactory<AlignerResults>);
+  ff_registry.Register("CSplit_BasicFeatures", new FFFactory<BasicCSplitFeatures>);
+  ff_registry.Register("CSplit_ReverseCharLM", new FFFactory<ReverseCharLMCSplitFeature>);
+  ff_registry.Register("Tagger_BigramIdentity", new FFFactory<Tagger_BigramIdentity>);
+  ff_registry.Register("LexicalPairIdentity", new FFFactory<LexicalPairIdentity>);
+
 }
+
diff --git a/decoder/do.tests.sh b/decoder/do.tests.sh
new file mode 100755
index 00000000..b3ddeb18
--- /dev/null
+++ b/decoder/do.tests.sh
@@ -0,0 +1 @@
+for f in *_test; do ./$f; done
diff --git a/decoder/ff.cc b/decoder/ff.cc
index 28620bab..a23f1655 100644
--- a/decoder/ff.cc
+++ b/decoder/ff.cc
@@ -43,7 +43,7 @@ Features ModelSet::all_features(std::ostream *warn,bool warn0) {
   FFM ff_from;
   for (unsigned i=0;i<models_.size();++i) {
     FeatureFunction const& ff=*models_[i];
-    string const& ffname=ff.name;
+    string const& ffname=ff.name_;
     FFS si=ff.features();
     if (si.empty()) {
       WARNFF(ffname<<" doesn't yet report any feature IDs - implement features() method?");
diff --git a/decoder/ff.h b/decoder/ff.h
index f563fb93..2de3c1fb 100644
--- a/decoder/ff.h
+++ b/decoder/ff.h
@@ -17,7 +17,7 @@ typedef std::vector<WordID> Features; // set of features ids
 // FinalTraversalFeatures(...)
 class FeatureFunction {
  public:
-  std::string name; // set by FF factory using usage()
+  std::string name_; // set by FF factory using usage()
   bool debug_; // also set by FF factory checking param for immediate initial "debug"
   bool debug() const { return debug_; }
   FeatureFunction() : state_size_() {}
diff --git a/decoder/ff_factory.cc b/decoder/ff_factory.cc
index 88991fbf..b3aeeac1 100644
--- a/decoder/ff_factory.cc
+++ b/decoder/ff_factory.cc
@@ -6,49 +6,42 @@
 using boost::shared_ptr;
 using namespace std;
 
-FFFactoryBase::~FFFactoryBase() {}
+UntypedFactory::~UntypedFactory() {  }
 
-void FFRegistry::DisplayList() const {
-  for (map<string, shared_ptr<FFFactoryBase> >::const_iterator it = reg_.begin();
+namespace {
+std::string const& debug_pre="debug";
+}
+
+void UntypedFactoryRegistry::clear() {
+  reg_.clear();
+}
+
+bool UntypedFactoryRegistry::parse_debug(std::string & param) {
+  int pl=debug_pre.size();
+  bool space=false;
+  std::string p=param;
+  bool debug=match_begin(p,debug_pre)&&
+    (p.size()==pl || (space=(p[pl]==' ')));
+  if (debug)
+    p.erase(0,debug_pre.size()+space);
+  return debug;
+}
+
+void UntypedFactoryRegistry::DisplayList() const {
+  for (Factmap::const_iterator it = reg_.begin();
        it != reg_.end(); ++it) {
     cerr << "  " << it->first << endl;
   }
 }
 
-string FFRegistry::usage(string const& ffname,bool params,bool verbose) const {
-  map<string, shared_ptr<FFFactoryBase> >::const_iterator it = reg_.find(ffname);
+string UntypedFactoryRegistry::usage(string const& ffname,bool params,bool verbose) const {
+  Factmap::const_iterator it = reg_.find(ffname);
   return it == reg_.end()
     ? "Unknown feature " + ffname
     : it->second->usage(params,verbose);
 }
 
-namespace {
-std::string const& debug_pre="debug";
-}
-
-shared_ptr<FeatureFunction> FFRegistry::Create(const string& ffname, const string& param) const {
-  map<string, shared_ptr<FFFactoryBase> >::const_iterator it = reg_.find(ffname);
-  shared_ptr<FeatureFunction> res;
-  if (it == reg_.end()) {
-    cerr << "I don't know how to create feature " << ffname << endl;
-  } else {
-    int pl=debug_pre.size();
-    bool space=false;
-    std::string p=param;
-    bool debug=match_begin(p,debug_pre)&&
-      (p.size()==pl || (space=(p[pl]==' ')));
-    if (debug) {
-      p.erase(0,debug_pre.size()+space);
-      cerr<<"debug enabled for "<<ffname<< " - remaining options: '"<<p<<"'\n";
-    }
-    res = it->second->Create(p);
-    res->name=ffname;
-    res->debug_=debug;
-  }
-  return res;
-}
-
-void FFRegistry::Register(const string& ffname, FFFactoryBase* factory) {
+void UntypedFactoryRegistry::Register(const string& ffname, UntypedFactory* factory) {
   if (reg_.find(ffname) != reg_.end()) {
     cerr << "Duplicate registration of FeatureFunction with name " << ffname << "!\n";
     abort();
@@ -57,7 +50,28 @@ void FFRegistry::Register(const string& ffname, FFFactoryBase* factory) {
 }
 
 
-void FFRegistry::Register(FFFactoryBase* factory)
+void UntypedFactoryRegistry::Register(UntypedFactory* factory)
 {
   Register(factory->usage(false,false),factory);
 }
+
+
+/*FIXME: I want these to go in ff_factory.cc, but extern etc. isn't workign right:
+  ../decoder/libcdec.a(ff_factory.o): In function `~UntypedFactory':
+/nfs/topaz/graehl/ws10smt/decoder/ff_factory.cc:9: multiple definition of `global_ff_registry'
+mr_vest_generate_mapper_input.o:/nfs/topaz/graehl/ws10smt/vest/mr_vest_generate_mapper_input.cc:307: first defined here
+*/
+FsaFFRegistry fsa_ff_registry;
+FFRegistry ff_registry;
+
+/*
+namespace {
+struct null_deleter
+{
+  template <class F>
+  void operator()(F const& f) const {  }
+};
+
+boost::shared_ptr<FsaFFRegistry> global_fsa_ff_registry(&fsa_ff_registry,null_deleter());
+boost::shared_ptr<FFRegistry> global_ff_registry(&ff_registry,null_deleter());
+*/
diff --git a/decoder/ff_factory.h b/decoder/ff_factory.h
index 93681c5e..e5821d44 100644
--- a/decoder/ff_factory.h
+++ b/decoder/ff_factory.h
@@ -1,6 +1,10 @@
 #ifndef _FF_FACTORY_H_
 #define _FF_FACTORY_H_
 
+// FsaF* vs F* (regular ff/factory).
+
+//TODO: use http://www.boost.org/doc/libs/1_43_0/libs/functional/factory/doc/html/index.html ?
+
 /*TODO: register state identity separately from feature function identity?  as
  * in: string registry for name of state somewhere, assert that same result is
  * computed by all users?  or, we can just require that ff sharing same state
@@ -8,49 +12,102 @@
  * once.  that's fine.
  */
 
-//TODO: use http://www.boost.org/doc/libs/1_43_0/libs/functional/factory/doc/html/index.html ?
 
 #include <iostream>
 #include <string>
 #include <map>
+#include <stdexcept>
 
 #include <boost/shared_ptr.hpp>
 
 class FeatureFunction;
-class FFRegistry;
-class FFFactoryBase;
-extern boost::shared_ptr<FFRegistry> global_ff_registry;
 
-class FFRegistry {
-  friend int main(int argc, char** argv);
-  friend class FFFactoryBase;
- public:
-  boost::shared_ptr<FeatureFunction> Create(const std::string& ffname, const std::string& param) const;
-  std::string usage(std::string const& ffname,bool params=true,bool verbose=true) const;
-  void DisplayList() const;
-  void Register(const std::string& ffname, FFFactoryBase* factory);
-  void Register(FFFactoryBase* factory);
-  FFRegistry() {}
- private:
-  std::map<std::string, boost::shared_ptr<FFFactoryBase> > reg_;
-};
+class FsaFeatureFunction;
 
-struct FFFactoryBase {
-  virtual ~FFFactoryBase();
-  virtual boost::shared_ptr<FeatureFunction> Create(const std::string& param) const = 0;
+
+struct UntypedFactory {
+  virtual ~UntypedFactory();
   virtual std::string usage(bool params,bool verbose) const = 0;
 };
 
+template <class FF>
+struct FactoryBase : public UntypedFactory {
+  typedef FF F;
+  typedef boost::shared_ptr<F> FP;
+
+  virtual FP Create(std::string param) const = 0;
+};
+
+/* see cdec_ff.cc for example usage: this create concrete factories to be registered */
 template<class FF>
-class FFFactory : public FFFactoryBase {
-  boost::shared_ptr<FeatureFunction> Create(const std::string& param) const {
-    return boost::shared_ptr<FeatureFunction>(new FF(param));
+struct FFFactory : public FactoryBase<FeatureFunction> {
+  FP Create(std::string param) const {
+    return FP(new FF(param));
   }
-  // called with false,false just gives feature name
   virtual std::string usage(bool params,bool verbose) const {
     return FF::usage(params,verbose);
   }
+};
+
 
+// same as above, but we didn't want to require a typedef e.g. Parent in FF class, and template typedef isn't available
+template<class FF>
+struct FsaFactory : public FactoryBase<FsaFeatureFunction> {
+  FP Create(std::string param) const {
+    return FP(new FF(param));
+  }
+  virtual std::string usage(bool params,bool verbose) const {
+    return FF::usage(params,verbose);
+  }
+};
+
+struct UntypedFactoryRegistry {
+  std::string usage(std::string const& ffname,bool params=true,bool verbose=true) const;
+  void DisplayList() const;
+  void Register(const std::string& ffname, UntypedFactory* factory);
+  void Register(UntypedFactory* factory);
+  void clear();
+  static bool parse_debug(std::string & param_in_out); // returns true iff param starts w/ debug (and remove that prefix from param)
+ protected:
+  typedef boost::shared_ptr<UntypedFactory> FactoryP;
+  typedef std::map<std::string, FactoryP > Factmap;
+  Factmap reg_;
+  friend int main(int argc, char** argv);
+  friend class UntypedFactory;
 };
 
+
+
+template <class Feat>
+struct FactoryRegistry : public UntypedFactoryRegistry {
+  typedef Feat F;
+  typedef boost::shared_ptr<F> FP;
+  typedef FactoryBase<F> FB;
+
+  FP Create(const std::string& ffname, std::string param) const {
+    using namespace std;
+    Factmap::const_iterator it = reg_.find(ffname);
+    if (it == reg_.end())
+      throw std::runtime_error("I don't know how to create feature "+ffname);
+    bool debug=parse_debug(param);
+    if (debug)
+      cerr<<"debug enabled for "<<ffname<< " - remaining options: '"<<param<<"'\n";
+    FP res = dynamic_cast<FB const&>(*it->second).Create(param);
+    res->name_ = ffname;
+    res->debug_ = debug;
+    return res;
+  }
+};
+
+typedef FactoryRegistry<FeatureFunction> FFRegistry;
+typedef FactoryRegistry<FsaFeatureFunction> FsaFFRegistry;
+
+extern FsaFFRegistry fsa_ff_registry;
+inline FsaFFRegistry & global_fsa_ff_registry() { return fsa_ff_registry; }
+extern FFRegistry ff_registry;
+inline FFRegistry & global_ff_registry() { return ff_registry; }
+/*
+extern boost::shared_ptr<FsaFFRegistry> global_fsa_ff_registry;
+extern boost::shared_ptr<FFRegistry> global_ff_registry;
+*/
 #endif
diff --git a/decoder/ff_from_fsa.h b/decoder/ff_from_fsa.h
index 10ccfe6d..2c339aa8 100755
--- a/decoder/ff_from_fsa.h
+++ b/decoder/ff_from_fsa.h
@@ -18,11 +18,8 @@
    uses guarantee about markov order=N to score ASAP
    encoding of state: if less than N-1 (ctxlen) words
 
-   either:
-   struct FF : public FsaImpl,FeatureFunctionFromFsa<FF> (more efficient)
-
-   or:
-   struct FF : public FsaFeatureFunctionDynamic,FeatureFunctionFromFsa<FF> (code sharing, but double dynamic dispatch)
+   usage:
+   typedef FeatureFunctionFromFsa<LanguageModelFsa> LanguageModelFromFsa;
 */
 
 template <class Impl>
@@ -271,7 +268,6 @@ private:
   }
 };
 
-
 #ifdef TEST_FSA
 # include "tdict.cc"
 # include "ff_sample_fsa.h"
diff --git a/decoder/ff_fsa.h b/decoder/ff_fsa.h
index 6c1294f8..e7877dd5 100755
--- a/decoder/ff_fsa.h
+++ b/decoder/ff_fsa.h
@@ -8,6 +8,8 @@
 
   state is some fixed width byte array.  could actually be a void *, WordID sequence, whatever.
 
+  TODO: specify Scan return code or feature value = -inf for failure state (e.g. for hard intersection with desired target lattice?)
+
   TODO: maybe ff that wants to know about SentenceMetadata should store a ref to
   it permanently rather than get passed it for every operation.  we're never
   decoding more than 1 sentence at once and it's annoying to pass it.  same
diff --git a/decoder/ff_fsa_data.h b/decoder/ff_fsa_data.h
index 66d2cca8..3252c5ac 100755
--- a/decoder/ff_fsa_data.h
+++ b/decoder/ff_fsa_data.h
@@ -9,11 +9,19 @@
 
 typedef ValueArray<uint8_t> Bytes;
 
-// stuff I see no reason to have virtual.
+// stuff I see no reason to have virtual.  but there's a diamond inheritance problem to solve now when type erasing the CRTP impl wrapper.  virtual inheritance would slow things?
 struct FsaFeatureFunctionData
 {
+  //HACK for diamond inheritance (w/o costing performance)
+  FsaFeatureFunctionData *sync_to_;
+
+  void sync() const { // call this if you modify any fields after your constructor is done
+    if (sync_to_) *sync_to_=*this;
+  }
+
   FsaFeatureFunctionData(int statesz=0,Sentence const& end_sentence_phrase=Sentence()) : ssz(statesz),start(statesz),h_start(statesz),end_phrase_(end_sentence_phrase) {
     debug_=true;
+    sync_to_=0;
   }
 
   std::string name_;
@@ -65,6 +73,7 @@ protected:
   int ssz; // don't forget to set this. default 0 (it may depend on params of course)
   Bytes start,h_start; // start state and estimated-features (heuristic) start state.  set these.  default empty.
   Sentence end_phrase_; // words appended for final traversal (final state cost is assessed using Scan) e.g. "</s>" for lm.
+  // this can be called instead or after constructor (also set bytes and end_phrase_)
   void set_state_bytes(int sb=0) {
     if (start.size()!=sb) start.resize(sb);
     if (h_start.size()!=sb) h_start.resize(sb);
diff --git a/decoder/ff_fsa_dynamic.h b/decoder/ff_fsa_dynamic.h
index 2703b305..2a26676d 100755
--- a/decoder/ff_fsa_dynamic.h
+++ b/decoder/ff_fsa_dynamic.h
@@ -2,13 +2,14 @@
 #define FF_FSA_DYNAMIC_H
 
 struct SentenceMetadata;
+
 #include "ff_fsa_data.h"
 #include "hg.h" // can't forward declare nested Hypergraph::Edge class
 #include <sstream>
 
-
 // the type-erased interface
 
+//FIXME: diamond inheritance problem.  make a copy of the fixed data?  or else make the dynamic version not wrap but rather be templated CRTP base (yuck)
 struct FsaFeatureFunction : public FsaFeatureFunctionData {
   static const bool simple_phrase_score=false;
   virtual int markov_order() const = 0;
@@ -25,10 +26,11 @@ struct FsaFeatureFunction : public FsaFeatureFunctionData {
   virtual void *ScanPhraseAccumBounce(SentenceMetadata const& smeta,Hypergraph::Edge const& edge,WordID const* i, WordID const* end,void *cs,void *ns,Accum *accum) const = 0;
 
   virtual int early_score_words(SentenceMetadata const& smeta,Hypergraph::Edge const& edge,WordID const* i, WordID const* end,Accum *accum) const { return 0; }
-  virtual std::string usage(bool param,bool verbose) const {
+  virtual std::string usage_v(bool param,bool verbose) const {
     return FeatureFunction::usage_helper("unnamed_dynamic_fsa_feature","","",param,verbose);
   }
 
+
   virtual void print_state(std::ostream &o,void const*state) const {
     FsaFeatureFunctionData::print_state(o,state);
   }
@@ -45,12 +47,12 @@ struct FsaFeatureFunction : public FsaFeatureFunctionData {
 
 // conforming to above interface, type erases FsaImpl
 // you might be wondering: why do this?  answer: it's cool, and it means that the bottom-up ff over ff_fsa wrapper doesn't go through multiple layers of dynamic dispatch
-// usage: struct My : public FsaFeatureFunctionDynamic<My>
+// usage: typedef FsaFeatureFunctionDynamic<MyFsa> MyFsaDyn;
 template <class Impl>
-struct FsaFeatureFunctionDynamic : public FsaFeatureFunction {
+struct FsaFeatureFunctionDynamic : public FsaFeatureFunction,Impl {
   static const bool simple_phrase_score=Impl::simple_phrase_score;
   Impl& d() { return static_cast<Impl&>(*this); }
-  Impl const& d() { return static_cast<Impl const&>(*this); }
+  Impl const& d() const { return static_cast<Impl const&>(*this); }
   int markov_order() const { return d().markov_order(); }
 
   virtual void ScanAccum(SentenceMetadata const& smeta,Hypergraph::Edge const& edge,
@@ -68,6 +70,7 @@ struct FsaFeatureFunctionDynamic : public FsaFeatureFunction {
                            WordID const* i, WordID const* end,
                            void const* state,Accum *a) const {
     return d().ScanPhraseAccumOnly(smeta,edge,i,end,state,a);
+  }
 
   virtual void *ScanPhraseAccumBounce(SentenceMetadata const& smeta,Hypergraph::Edge const& edge,WordID const* i, WordID const* end,void *cs,void *ns,Accum *a) const {
     return d().ScanPhraseAccumBounce(smeta,edge,i,end,cs,ns,a);
@@ -77,15 +80,26 @@ struct FsaFeatureFunctionDynamic : public FsaFeatureFunction {
     return d().early_score_words(smeta,edge,i,end,accum);
   }
 
-  virtual std::string usage(bool param,bool verbose) const {
+  static std::string usage(bool param,bool verbose) {
+    return Impl::usage(param,verbose);
+  }
+
+  std::string usage_v(bool param,bool verbose) const {
     return Impl::usage(param,verbose);
   }
 
   virtual void print_state(std::ostream &o,void const*state) const {
     return d().print_state(o,state);
   }
+
+  FsaFeatureFunctionDynamic(std::string const& param) : Impl(param) {
+    d().sync_to_=(FsaFeatureFunction*)this;
+    d().sync();
+  }
+
 };
 
+
 //TODO: combine 2 (or N) FsaFeatureFunction (type erased)
 
 
diff --git a/decoder/ff_lm_fsa.h b/decoder/ff_lm_fsa.h
index b95fde02..1c8ebdad 100755
--- a/decoder/ff_lm_fsa.h
+++ b/decoder/ff_lm_fsa.h
@@ -4,6 +4,9 @@
 //FIXME: when FSA_LM_PHRASE 1, 3gram fsa has differences, especially with unk words, in about the 4th decimal digit (about .05%), compared to regular ff_lm.  this is USUALLY a bug (there's way more actual precision in there).  this was with #define LM_FSA_SHORTEN_CONTEXT 1 and 0 (so it's not that).  also, LM_FSA_SHORTEN_CONTEXT gives identical scores with FSA_LM_PHRASE 0
 
 // enabling for now - retest unigram+ more, solve above puzzle
+
+// some impls in ff_lm.cc
+
 #define FSA_LM_PHRASE 1
 
 #define FSA_LM_DEBUG 0
@@ -15,8 +18,8 @@
 # define FSALMDBGnl(e)
 #endif
 
+#include "ff_fsa.h"
 #include "ff_lm.h"
-#include "ff_from_fsa.h"
 
 namespace {
 WordID empty_context=TD::none;
@@ -49,17 +52,13 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase<LanguageModelFsa> {
 #endif
     if (!ctxlen_) {
       Add(floored(pimpl_->WordProb(w,&empty_context)),a);
-      return;
-    }
-    //variable length array is in C99, msvc++, if it doesn't support it, #ifdef it or use a stackalloc call (forget the name)
-    if (ctxlen_) {
+    } else {
       WordID ctx[ngram_order_]; //alloca if you don't have C99
       state_copy(ctx,old_st);
-      ctx[ctxlen_]=TD::none; // make this part of state?  wastes space but saves copies.
+      ctx[ctxlen_]=TD::none;
       Featval p=floored(pimpl_->WordProb(w,ctx));
-      FSALMDBG(de,"p("<<TD::Convert(w)<<"|"<<TD::Convert(ctx,ctx+ctxlen_)<<")="<<p);
-      FSALMDBGnl(de);
-// states are sri contexts so are in reverse order (most recent word is first, then 1-back comes next, etc.).
+      FSALMDBG(de,"p("<<TD::Convert(w)<<"|"<<TD::Convert(ctx,ctx+ctxlen_)<<")="<<p);FSALMDBGnl(de);
+      // states are srilm contexts so are in reverse order (most recent word is first, then 1-back comes next, etc.).
       WordID *nst=(WordID *)new_st;
       nst[0]=w; // new most recent word
       to_state(nst+1,ctx,ctxlen_-1); // rotate old words right
@@ -74,9 +73,7 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase<LanguageModelFsa> {
   //FIXME: there is a bug in here somewhere, or else the 3gram LM we use gives different scores for phrases (impossible? BOW nonzero when shortening context past what LM has?)
   template <class Accum>
   void ScanPhraseAccum(SentenceMetadata const& /* smeta */,const Hypergraph::Edge&edge,WordID const* begin,WordID const* end,void const* old_st,void *new_st,Accum *a) const {
-# if USE_INFO_EDGE
-    Hypergraph::Edge &de=(Hypergraph::Edge &)edge;
-# endif
+    Hypergraph::Edge &de=(Hypergraph::Edge &)edge;(void)de;
     if (begin==end) return; // otherwise w/ shortening it's possible to end up with no words at all.
     /* // this is forcing unigram prob always.  we will instead build the phrase
     if (!ctxlen_) {
@@ -98,7 +95,6 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase<LanguageModelFsa> {
     assert(ctx_score_end==ctx+nw);
     // we could just copy the filled state words, but it probably doesn't save much time (and might cost some to scan to find the nones.  most contexts are full except for the shortest source spans.
     FSALMDBG(de," scan.r->l("<<TD::GetString(ctx,ctx_score_end)<<"|"<<TD::GetString(ctx_score_end,ctx+nboth)<<')');
-    FSAFFDBG(de," r->l("<<TD::GetString(ctx,ctx_score_end)<<"|"<<TD::GetString(ctx_score_end,ctx+nboth)<<')');
     Featval p=0;
     FSALMDBGnl(edge);
     for(;ctx_score_end>ctx;--ctx_score_end)
@@ -128,6 +124,5 @@ private:
 
 };
 
-typedef FeatureFunctionFromFsa<LanguageModelFsa> LanguageModelFromFsa;
 
 #endif
diff --git a/decoder/ff_sample_fsa.h b/decoder/ff_sample_fsa.h
index 9f44f1a4..40a32bae 100755
--- a/decoder/ff_sample_fsa.h
+++ b/decoder/ff_sample_fsa.h
@@ -29,7 +29,11 @@ struct WordPenaltyFsa : public FsaFeatureFunctionBase<WordPenaltyFsa> {
 typedef FeatureFunctionFromFsa<WordPenaltyFsa> WordPenaltyFromFsa;
 
 struct SameFirstLetter : public FsaFeatureFunctionBase<SameFirstLetter> {
-  SameFirstLetter(std::string const& param) : FsaFeatureFunctionBase<SameFirstLetter>(1,singleton_sentence("END")) {  start[0]='a';h_start[0]=0; } // 1 byte of state, scan final (single) symbol "END" to get final state cost
+  SameFirstLetter(std::string const& param) : FsaFeatureFunctionBase<SameFirstLetter>(1,singleton_sentence("END"))
+                                               // 1 byte of state, scan final (single) symbol "END" to get final state cost
+  {
+    start[0]='a'; h_start[0]=0; Init();
+  }
   int markov_order() const { return 1; }
   Featval Scan1(WordID w,void const* old_state,void *new_state) const {
     char cw=TD::Convert(w)[0];
@@ -41,12 +45,13 @@ struct SameFirstLetter : public FsaFeatureFunctionBase<SameFirstLetter> {
     o<<*(char const*)st;
   }
   static std::string usage(bool param,bool verbose) {
-    return FeatureFunction::usage_helper("SameFirstLetter","[no args]","1 each time 2 consecutive words start with the same letter",param,verbose);
+    return FeatureFunction::usage_helper("SameFirstLetter",
+                                         "[no args]",
+                                         "1 each time 2 consecutive words start with the same letter",
+                                         param,verbose);
   }
 };
 
-
-// appears to be buggy right now: give it a bonus weight (+)
 struct LongerThanPrev : public FsaFeatureFunctionBase<LongerThanPrev> {
   typedef FsaFeatureFunctionBase<LongerThanPrev> Base;
   static std::string usage(bool param,bool verbose) {
@@ -104,7 +109,10 @@ struct LongerThanPrev : public FsaFeatureFunctionBase<LongerThanPrev> {
 
 // similar example feature; base type exposes stateful type, defines markov_order 1, state size = sizeof(State)
 struct ShorterThanPrev : FsaTypedBase<int,ShorterThanPrev> {
-  typedef FsaTypedBase<int,ShorterThanPrev> Base;
+  ShorterThanPrev(std::string const& param)
+    : FsaTypedBase<int,ShorterThanPrev>(-1,4,singleton_sentence(TD::se)) // start, h_start, end_phrase
+    // h_start estimate state: anything <4 chars is usually shorter than previous
+  { Init(); }
   static std::string usage(bool param,bool verbose) {
     return FeatureFunction::usage_helper(
       "ShorterThanPrev",
@@ -112,28 +120,13 @@ struct ShorterThanPrev : FsaTypedBase<int,ShorterThanPrev> {
       "stupid example stateful (bigram) feature: 1 per target word that's shorter than the previous word (end of sentence considered '</s>')",
       param,verbose);
   }
-
   static inline int wordlen(WordID w) {
     return std::strlen(TD::Convert(w));
   }
-  ShorterThanPrev(std::string const& param)
-    : Base(-1,4,singleton_sentence(TD::se)) // start, h_start, end_phrase
-    // estimate: anything <4 chars is usually shorter than previous
-  {
-    Init();
-  }
-
-
-/*  Featval ScanT1(WordID w,int prevlen,int &len) const;
-    // alternative to below:
-    */
-
-  // evil anti-google int & len out-param:
   Featval ScanT1(SentenceMetadata const& /* smeta */,const Hypergraph::Edge& /* edge */,WordID w,int prevlen,int &len) const {
     len=wordlen(w);
     return (len<prevlen) ? 1 : 0;
   }
-
 };
 
 
diff --git a/decoder/oracle_bleu.h b/decoder/oracle_bleu.h
index 932378e5..c5ca2fcf 100755
--- a/decoder/oracle_bleu.h
+++ b/decoder/oracle_bleu.h
@@ -149,7 +149,7 @@ struct OracleBleu {
     order=(loss==IBM_BLEU_3)?3:4;
     std::ostringstream param;
     param<<"-o "<<order;
-    pff=global_ff_registry->Create("BLEUModel",param.str());
+    pff=ff_registry.Create("BLEUModel",param.str());
   }
 
   bool is_null() const {
diff --git a/decoder/static_utoa.h b/decoder/static_utoa.h
index 0dbe111f..fe5f6d92 100755
--- a/decoder/static_utoa.h
+++ b/decoder/static_utoa.h
@@ -2,6 +2,9 @@
 #define STATIC_UTOA_H
 
 #include "threadlocal.h"
+
+
+#include <string>
 #include <cstring>
 
 #define DIGIT_LOOKUP_TABLE 0
@@ -24,8 +27,7 @@ inline char digit_to_char(int d) {
 #endif
 }
 
-
-// returns n in string [return,num); *num=0 yourself calling if you want a c_str
+// returns n in string [return,num); *num=0 yourself before calling if you want a c_str
 inline char *utoa(char *num,unsigned n) {
   if ( !n ) {
     *--num='0';
@@ -59,5 +61,55 @@ inline char* append_utoa(char *to,unsigned n) {
   return to+ns;
 }
 
+// so named to avoid gcc segfault when named itoa
+inline char *itoa(char *p,int n) {
+  if (n<0) {
+    p=utoa(p,-n); // TODO: check that (unsigned)(-INT_MIN) == 0x1000000 in 2s complement and not == 0
+    *--p='-';
+    return p;
+  } else
+    return utoa(p,n);
+}
+
+inline char *static_itoa(int n) {
+  return itoa(utoa_buf+utoa_bufsizem1,n);
+}
+
+
+inline std::string utos(unsigned n) {
+  const int bufsz=20;
+  char buf[bufsz];
+  char *end=buf+bufsz;
+  char *p=utoa(end,n);
+  return std::string(p,end);
+}
+
+inline std::string itos(int n) {
+  const int bufsz=20;
+  char buf[bufsz];
+  char *end=buf+bufsz;
+  char *p=itoa(end,n);
+  return std::string(p,end);
+}
+
+#ifdef ITOA_SAMPLE
+# include <cstdio>
+# include <sstream>
+# include <iostream>
+using namespace std;
+
+int main(int argc,char *argv[]) {
+  printf("d U d U d U\n");
+  for (int i=1;i<argc;++i) {
+    int n;
+    unsigned un;
+    sscanf(argv[i],"%d",&n);
+    sscanf(argv[i],"%u",&un);
+    printf("%d %u %s",n,un,static_itoa(n));
+    printf(" %s %s %s\n",static_utoa(un),itos(n).c_str(),utos(un).c_str());
+  }
+  return 0;
+}
+#endif
 
 #endif
diff --git a/decoder/value_array.h b/decoder/value_array.h
index 1a0d5f95..cdf1d697 100755
--- a/decoder/value_array.h
+++ b/decoder/value_array.h
@@ -150,8 +150,10 @@ public:
   }
 
   ValueArray(ValueArray const& other)
-    : sz(other.sz)
+    : A(other)
+    , sz(other.sz)
     , array(A::allocate(sz))
+
   {
     copy_construct(other.begin(),other.end(),array);
   }