Add just enough jam to compile the decoder.

8 files changed, 353 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 638ad83f..fcacb135 100644
--- a/.gitignore
+++ b/.gitignore
@@ -197,3 +197,5 @@ mteval/scorer_test
 phrasinator/gibbs_train_plm
 phrasinator/gibbs_train_plm_notables
 .*
+previous.sh
+dist
diff --git a/Jamroot b/Jamroot
new file mode 100644
index 00000000..ad09509d
--- /dev/null
+++ b/Jamroot
@@ -0,0 +1,29 @@
+path-constant TOP : . ;
+include $(TOP)/sanity.jam ;
+boost 104400 ;
+external-lib z ;
+
+if [ test_header google/dense_hash_map ] {
+  requirements += <define>HAVE_SPARSEHASH ;
+}
+
+if [ test_header boost/serialization/map.hpp ] && [ test_library boost_serialization ] {
+  requirements += <define>HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP ;
+}
+
+project : requirements $(requirements) ;
+project : default-build <threading>single <warnings>on <variant>release ;
+
+install-bin-libs utils//programs mteval//programs klm/lm//programs decoder//cdec ;
+
+build-project mteval ;
+build-project decoder ;
+
+#Compile everything ending with _test.cc into a test and run it.
+rule all_tests ( targets * : dependencies ) {
+  targets ?= [ glob *_test.cc ] ;
+  for t in $(targets) {
+    local base = [ MATCH "^(.*).cc$" : $(t) ] ;
+    unit-test $(base) : $(t) $(dependencies) ..//boost_unit_test_framework ;
+  }
+}
diff --git a/decoder/Jamfile b/decoder/Jamfile
new file mode 100644
index 00000000..d7d42f28
--- /dev/null
+++ b/decoder/Jamfile
@@ -0,0 +1,72 @@
+import testing ;
+import lex ;
+import option ;
+
+if [ option.get "with-glc" ] {
+  glc = ff_glc.cc string_util.cc feature-factory.cc ;
+}
+
+lib cdec_lib : 
+  forest_writer.cc
+  maxtrans_blunsom.cc
+  cdec_ff.cc
+  cfg.cc
+  dwarf.cc
+  ff_dwarf.cc
+  rule_lexer.ll
+  fst_translator.cc
+  csplit.cc
+  translator.cc
+  scfg_translator.cc
+  hg.cc
+  hg_io.cc
+  decoder.cc
+  hg_intersect.cc
+  hg_sampler.cc
+  factored_lexicon_helper.cc
+  viterbi.cc
+  lattice.cc
+  aligner.cc
+  apply_models.cc
+  earley_composer.cc
+  phrasetable_fst.cc
+  trule.cc
+  ff.cc
+  ff_rules.cc
+  ff_wordset.cc
+  ff_context.cc
+  ff_charset.cc
+  ff_lm.cc
+  ff_klm.cc
+  ff_ngrams.cc
+  ff_spans.cc
+  ff_ruleshape.cc
+  ff_wordalign.cc
+  ff_csplit.cc
+  ff_tagger.cc
+  ff_source_syntax.cc
+  ff_bleu.cc
+  ff_factory.cc
+  lexalign.cc
+  lextrans.cc
+  tagger.cc
+  bottom_up_parser.cc
+  phrasebased_translator.cc
+  JSON_parser.c
+  json_parse.cc
+  grammar.cc
+  $(glc)
+  ..//utils
+  ..//mteval
+  ../klm/lm//kenlm
+  ..//boost_program_options
+  : <include>.
+  ;
+
+exe cdec : cdec.cc cdec_lib ;
+
+all_tests [ glob *_test.cc : cfg_test.cc ] : cdec_lib ;
+
+install legacy : cdec
+  : <location>$(TOP)/cdec <install-type>EXE <install-dependencies>on <link>shared:<dll-path>$(TOP)/cdec <link>shared:<install-type>LIB ;
+
diff --git a/klm/lm/Jamfile b/klm/lm/Jamfile
new file mode 100644
index 00000000..b84dbb35
--- /dev/null
+++ b/klm/lm/Jamfile
@@ -0,0 +1,14 @@
+lib kenlm : bhiksha.cc binary_format.cc config.cc lm_exception.cc model.cc quantize.cc read_arpa.cc search_hashed.cc search_trie.cc trie.cc trie_sort.cc virtual_interface.cc vocab.cc ../util//kenutil : <include>.. : : <include>.. <library>../util//kenutil ;
+
+import testing ;
+
+run left_test.cc ../util//kenutil kenlm ../..//boost_unit_test_framework : : test.arpa ;
+run model_test.cc ../util//kenutil kenlm ../..//boost_unit_test_framework : : test.arpa test_nounk.arpa ;
+
+exe query : ngram_query.cc kenlm ../util//kenutil ;
+exe build_binary : build_binary.cc kenlm ../util//kenutil ;
+
+install legacy : build_binary query 
+  : <location>$(TOP)/klm/lm <install-type>EXE <install-dependencies>on <link>shared:<dll-path>$(TOP)/klm/lm <link>shared:<install-type>LIB ;
+
+alias programs : build_binary query ;
diff --git a/klm/util/Jamfile b/klm/util/Jamfile
new file mode 100644
index 00000000..00eefc22
--- /dev/null
+++ b/klm/util/Jamfile
@@ -0,0 +1,10 @@
+lib kenutil : bit_packing.cc ersatz_progress.cc exception.cc file.cc file_piece.cc mmap.cc murmur_hash.cc ../..///z : <include>.. : : <include>.. ;
+
+import testing ;
+
+unit-test bit_packing_test : bit_packing_test.cc kenutil ../..///boost_unit_test_framework ;
+run file_piece_test.cc kenutil ../..///boost_unit_test_framework : : file_piece.cc ;
+unit-test joint_sort_test : joint_sort_test.cc kenutil ../..///boost_unit_test_framework ;
+unit-test probing_hash_table_test : probing_hash_table_test.cc kenutil ../..///boost_unit_test_framework ;
+unit-test sorted_uniform_test : sorted_uniform_test.cc kenutil ../..///boost_unit_test_framework ;
+unit-test tokenize_piece_test : tokenize_piece_test.cc kenutil ../..///boost_unit_test_framework ;
diff --git a/mteval/Jamfile b/mteval/Jamfile
new file mode 100644
index 00000000..f671134a
--- /dev/null
+++ b/mteval/Jamfile
@@ -0,0 +1,8 @@
+import testing ;
+
+lib mteval : ter.cc comb_scorer.cc aer_scorer.cc scorer.cc external_scorer.cc ns.cc ns_ter.cc ns_ext.cc ns_comb.cc ns_docscorer.cc ..//utils : <include>. : : <include>. ;
+exe fast_score : fast_score.cc mteval ..//utils ..//boost_program_options ;
+exe mbr_kbest : mbr_kbest.cc mteval ..//utils ..//boost_program_options ;
+alias programs : fast_score mbr_kbest ;
+
+unit-test scorer_test : scorer_test.cc mteval ..//utils ..//z ..//boost_unit_test_framework ;
diff --git a/sanity.jam b/sanity.jam
new file mode 100644
index 00000000..1b37a4e7
--- /dev/null
+++ b/sanity.jam
@@ -0,0 +1,184 @@
+import modules ;
+import option ;
+import os ;
+import path ;
+import project ;
+
+#Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php
+rule trim-nl ( str extras * ) {
+return [ MATCH "([^
+]*)" : $(str) ] $(extras) ;
+}
+rule _shell ( cmd : extras * ) {
+  return [ trim-nl [ SHELL $(cmd) : $(extras) ] ] ;
+}
+
+#Run g++ with empty main and these arguments to see if it passes.  
+rule test_flags ( flags * ) {
+  local cmd = "bash -c \"g++ "$(flags:J=" ")" -x c++ - <<<'int main() {}' -o /dev/null >/dev/null 2>/dev/null\"" ;
+  local ret = [ SHELL $(cmd) : exit-status ] ;
+  if --debug-configuration in [ modules.peek : ARGV ] {
+    echo $(cmd) ;
+    echo $(ret) ;
+  }
+  if $(ret[2]) = 0 {
+    return true ;
+  } else {
+    return ;
+  }
+}
+
+rule test_header ( name ) {
+  return [ test_flags "-include $(name)" ] ;
+}
+
+rule test_library ( name ) {
+  return [ test_flags "-l$(name)" ] ;
+}
+
+{
+  local cleaning = [ option.get "clean" : : yes ] ;
+  cleaning ?= [ option.get "clean-all" : no : yes ] ;
+  if "clean" in [ modules.peek : ARGV ] {
+    cleaning = yes ;
+  }
+  constant CLEANING : $(cleaning) ;
+}
+
+#Determine if a library can be compiled statically.  
+rule auto-shared ( name : additional * ) {
+  additional ?= "" ;
+  if [ test_flags $(additional)" -static -l"$(name) ] {
+    return ;
+  } else {
+    return "<link>shared" ;
+  }
+}
+
+# MacPorts' default location is /opt/local -- use this if no path is given.
+with-macports = [ option.get "with-macports" : : "/opt/local" ] ;
+if $(with-macports) {
+  using darwin ;
+  ECHO "Using --with-macports=$(with-macports), implying use of darwin GCC" ;
+
+  L-boost-search = -L$(with-macports)/lib ;
+  boost-search = <search>$(with-macports)/lib ;
+  I-boost-include = -I$(with-macports)/include ;
+  boost-include = <include>$(with-macports)/include ;
+}
+else {
+  with-boost = [ option.get "with-boost" ] ;
+  if $(with-boost) {
+    L-boost-search = -L$(with-boost)/lib" "-L$(with-boost)/lib64 ;
+    boost-search = <search>$(with-boost)/lib <search>$(with-boost)/lib64 ;
+    I-boost-include = -I$(with-boost)/include ;
+    boost-include = <include>$(with-boost)/include ;
+  } else {
+    L-boost-search = "" ;
+    boost-search = ;
+    I-boost-include = "" ;
+    boost-include = ;
+  }
+}
+ 
+#Are we linking static binaries against shared boost?
+boost-auto-shared = [ auto-shared "boost_program_options" : $(L-boost-search) ] ;
+#Convenience rule for boost libraries.  Defines library boost_$(name).  
+rule boost-lib ( name macro ) {
+  #Link multi-threaded programs against the -mt version if available.  Old 
+  #versions of boost do not have -mt tagged versions of all libraries.   Sadly,
+  #boost.jam does not handle this correctly.  
+  if [ test_flags $(L-boost-search)" -lboost_"$(name)"-mt" ] {
+    lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name) ;
+    lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt ;
+  } else {
+    lib inner_boost_$(name) : : $(boost-search) <name>boost_$(name) ;
+  }
+
+  alias boost_$(name) : inner_boost_$(name) : $(boost-auto-shared) : : <link>shared:<define>BOOST_$(macro) $(boost-include) ;
+}
+
+#Argument is e.g. 103600
+rule boost ( min-version ) {
+  local cmd = "bash -c \"g++ "$(I-boost-include)" -dM -x c++ -E /dev/null -include boost/version.hpp 2>/dev/null |grep '#define BOOST_VERSION '\"" ;
+  local boost-shell = [ SHELL "$(cmd)" : exit-status ] ;
+  if $(boost-shell[2]) != 0 && $(CLEANING) = no {
+    echo Failed to run "$(cmd)" ;
+    exit Boost does not seem to be installed or g++ is confused. : 1 ;
+  }
+  boost-version = [ MATCH "#define BOOST_VERSION ([0-9]*)" : $(boost-shell[1]) ] ;
+  if $(boost-version) < $(min-version) && $(CLEANING) = no {
+    exit You have Boost $(boost-version).  This package requires Boost at least $(min-version) (and preferably newer). : 1 ;
+  }
+  #See tools/build/v2/contrib/boost.jam in a boost distribution for a table of macros to define.   
+  boost-lib thread THREAD_DYN_DLL ;
+  boost-lib program_options PROGRAM_OPTIONS_DYN_LINK ;
+  boost-lib unit_test_framework TEST_DYN_LINK ;
+  boost-lib iostreams IOSTREAMS_DYN_LINK ;
+  boost-lib serialization SERIALIZATION_DYN_LINK ;
+  boost-lib mpi MPI_DYN_LINK ;
+}
+ 
+#Link normally to a library, but sometimes static isn't installed so fall back to dynamic.
+rule external-lib ( name : search-path * ) {
+  lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) ;
+}
+
+#Write the current command line to previous.sh.  This does not do shell escaping.  
+{
+  local build-log = $(TOP)/previous.sh ;
+  if ! [ path.exists $(build-log) ] {
+    SHELL "touch $(build-log) && chmod +x $(build-log)" ;
+  }
+  local script = [ modules.peek : ARGV ] ;
+  if $(script[1]) = "./jam-files/bjam" {
+    #The ./bjam shell script calls ./jam-files/bjam so that appears in argv but
+    #we want ./bjam to appear so the environment variables are set correctly.  
+    script = "./bjam "$(script[2-]:J=" ") ;
+  } else {
+    script = $(script:J=" ") ;
+  }
+  script = "#!/bin/sh\n$(script)\n" ;
+  local ignored = @($(build-log):E=$(script)) ;
+}
+
+requirements = ;
+{
+  local cxxflags = [ os.environ "CXXFLAGS" ] ;
+  local cflags = [ os.environ "CFLAGS" ] ;
+  local ldflags = [ os.environ "LDFLAGS" ] ;
+
+  #Boost jam's static clang is buggy.  
+  requirements += <cxxflags>$(cxxflags) <cflags>$(cflags) <linkflags>$(ldflags) <toolset>clang:<link>shared ;
+
+  #libSegFault prints a stack trace on segfault.  Link against it if available.  
+  if [ test_flags "-lSegFault" ] {
+    external-lib SegFault ;
+    requirements += <library>SegFault ;
+  }
+}
+
+if [ option.get "git" : : "yes" ] {
+  local revision = [ _shell "git rev-parse --verify HEAD |head -c 7" ] ;
+  constant GITTAG : "/"$(revision) ;
+} else {
+  constant GITTAG : "" ;
+}
+
+prefix = [ option.get "prefix" ] ;
+if $(prefix) {
+  prefix = [ path.root $(prefix) [ path.pwd ] ] ;
+} else {
+  prefix = $(TOP)/dist$(GITTAG) ;
+}
+rule install-bin-libs ( deps * ) {
+  local bindir = [ option.get "bindir" : $(prefix)/bin ] ;
+  local libdir = [ option.get "libdir" : $(prefix)/lib ] ;
+  install prefix-bin : $(deps) : <location>$(bindir) <install-dependencies>on <install-type>EXE <link>shared:<dll-path>$(libdir) ;
+  install prefix-lib : $(deps) : <location>$(libdir) <install-dependencies>on <install-type>LIB <link>shared:<dll-path>$(libdir) ;
+}
+rule install-headers ( name : list * : source-root ? ) {
+  local includedir = [ option.get "includedir" : $(prefix)/include ] ;
+  source-root ?= "." ;
+  install $(name) : $(list) : <location>$(includedir) <install-source-root>$(source-root) ;
+}
diff --git a/utils/Jamfile b/utils/Jamfile
new file mode 100644
index 00000000..53a51277
--- /dev/null
+++ b/utils/Jamfile
@@ -0,0 +1,34 @@
+import testing ;
+import option ;
+
+additional = ;
+if [ option.get with-cmph ] {
+  additional += perfect_hash.cc ;
+}
+
+lib utils :
+  alignment_io.cc 
+  b64tools.cc 
+  corpus_tools.cc 
+  dict.cc 
+  tdict.cc 
+  fdict.cc 
+  gzstream.cc 
+  filelib.cc 
+  stringlib.cc 
+  sparse_vector.cc 
+  timing_stats.cc 
+  verbose.cc 
+  weights.cc
+  $(additional)
+  ..//z
+  : <include>.. <include>. : : <include>.. <include>. ;
+
+exe ts : ts.cc utils ;
+exe atools : atools.cc utils ..//boost_program_options ;
+exe phmt : phmt.cc utils ;
+exe reconstruct_weights : reconstruct_weights.cc utils ..//boost_program_options ;
+
+alias programs : reconstruct_weights atools ;
+
+all_tests : utils ;