diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Jamroot | 29 | ||||
| -rw-r--r-- | decoder/Jamfile | 72 | ||||
| -rw-r--r-- | klm/lm/Jamfile | 14 | ||||
| -rw-r--r-- | klm/util/Jamfile | 10 | ||||
| -rw-r--r-- | mteval/Jamfile | 8 | ||||
| -rw-r--r-- | sanity.jam | 184 | ||||
| -rw-r--r-- | utils/Jamfile | 34 |
8 files changed, 353 insertions, 0 deletions
@@ -197,3 +197,5 @@ mteval/scorer_test phrasinator/gibbs_train_plm phrasinator/gibbs_train_plm_notables .* +previous.sh +dist diff --git a/Jamroot b/Jamroot new file mode 100644 index 00000000..ad09509d --- /dev/null +++ b/Jamroot @@ -0,0 +1,29 @@ +path-constant TOP : . ; +include $(TOP)/sanity.jam ; +boost 104400 ; +external-lib z ; + +if [ test_header google/dense_hash_map ] { + requirements += <define>HAVE_SPARSEHASH ; +} + +if [ test_header boost/serialization/map.hpp ] && [ test_library boost_serialization ] { + requirements += <define>HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP ; +} + +project : requirements $(requirements) ; +project : default-build <threading>single <warnings>on <variant>release ; + +install-bin-libs utils//programs mteval//programs klm/lm//programs decoder//cdec ; + +build-project mteval ; +build-project decoder ; + +#Compile everything ending with _test.cc into a test and run it. +rule all_tests ( targets * : dependencies ) { + targets ?= [ glob *_test.cc ] ; + for t in $(targets) { + local base = [ MATCH "^(.*).cc$" : $(t) ] ; + unit-test $(base) : $(t) $(dependencies) ..//boost_unit_test_framework ; + } +} diff --git a/decoder/Jamfile b/decoder/Jamfile new file mode 100644 index 00000000..d7d42f28 --- /dev/null +++ b/decoder/Jamfile @@ -0,0 +1,72 @@ +import testing ; +import lex ; +import option ; + +if [ option.get "with-glc" ] { + glc = ff_glc.cc string_util.cc feature-factory.cc ; +} + +lib cdec_lib : + forest_writer.cc + maxtrans_blunsom.cc + cdec_ff.cc + cfg.cc + dwarf.cc + ff_dwarf.cc + rule_lexer.ll + fst_translator.cc + csplit.cc + translator.cc + scfg_translator.cc + hg.cc + hg_io.cc + decoder.cc + hg_intersect.cc + hg_sampler.cc + factored_lexicon_helper.cc + viterbi.cc + lattice.cc + aligner.cc + apply_models.cc + earley_composer.cc + phrasetable_fst.cc + trule.cc + ff.cc + ff_rules.cc + ff_wordset.cc + ff_context.cc + ff_charset.cc + ff_lm.cc + ff_klm.cc + ff_ngrams.cc + ff_spans.cc + ff_ruleshape.cc + ff_wordalign.cc + ff_csplit.cc + ff_tagger.cc + ff_source_syntax.cc + ff_bleu.cc + ff_factory.cc + lexalign.cc + lextrans.cc + tagger.cc + bottom_up_parser.cc + phrasebased_translator.cc + JSON_parser.c + json_parse.cc + grammar.cc + $(glc) + ..//utils + ..//mteval + ../klm/lm//kenlm + ..//boost_program_options + : <include>. + ; + +exe cdec : cdec.cc cdec_lib ; + +all_tests [ glob *_test.cc : cfg_test.cc ] : cdec_lib ; + +install legacy : cdec + : <location>$(TOP)/cdec <install-type>EXE <install-dependencies>on <link>shared:<dll-path>$(TOP)/cdec <link>shared:<install-type>LIB ; + diff --git a/klm/lm/Jamfile b/klm/lm/Jamfile new file mode 100644 index 00000000..b84dbb35 --- /dev/null +++ b/klm/lm/Jamfile @@ -0,0 +1,14 @@ +lib kenlm : bhiksha.cc binary_format.cc config.cc lm_exception.cc model.cc quantize.cc read_arpa.cc search_hashed.cc search_trie.cc trie.cc trie_sort.cc virtual_interface.cc vocab.cc ../util//kenutil : <include>.. : : <include>.. <library>../util//kenutil ; + +import testing ; + +run left_test.cc ../util//kenutil kenlm ../..//boost_unit_test_framework : : test.arpa ; +run model_test.cc ../util//kenutil kenlm ../..//boost_unit_test_framework : : test.arpa test_nounk.arpa ; + +exe query : ngram_query.cc kenlm ../util//kenutil ; +exe build_binary : build_binary.cc kenlm ../util//kenutil ; + +install legacy : build_binary query + : <location>$(TOP)/klm/lm <install-type>EXE <install-dependencies>on <link>shared:<dll-path>$(TOP)/klm/lm <link>shared:<install-type>LIB ; + +alias programs : build_binary query ; diff --git a/klm/util/Jamfile b/klm/util/Jamfile new file mode 100644 index 00000000..00eefc22 --- /dev/null +++ b/klm/util/Jamfile @@ -0,0 +1,10 @@ +lib kenutil : bit_packing.cc ersatz_progress.cc exception.cc file.cc file_piece.cc mmap.cc murmur_hash.cc ../..///z : <include>.. : : <include>.. ; + +import testing ; + +unit-test bit_packing_test : bit_packing_test.cc kenutil ../..///boost_unit_test_framework ; +run file_piece_test.cc kenutil ../..///boost_unit_test_framework : : file_piece.cc ; +unit-test joint_sort_test : joint_sort_test.cc kenutil ../..///boost_unit_test_framework ; +unit-test probing_hash_table_test : probing_hash_table_test.cc kenutil ../..///boost_unit_test_framework ; +unit-test sorted_uniform_test : sorted_uniform_test.cc kenutil ../..///boost_unit_test_framework ; +unit-test tokenize_piece_test : tokenize_piece_test.cc kenutil ../..///boost_unit_test_framework ; diff --git a/mteval/Jamfile b/mteval/Jamfile new file mode 100644 index 00000000..f671134a --- /dev/null +++ b/mteval/Jamfile @@ -0,0 +1,8 @@ +import testing ; + +lib mteval : ter.cc comb_scorer.cc aer_scorer.cc scorer.cc external_scorer.cc ns.cc ns_ter.cc ns_ext.cc ns_comb.cc ns_docscorer.cc ..//utils : <include>. : : <include>. ; +exe fast_score : fast_score.cc mteval ..//utils ..//boost_program_options ; +exe mbr_kbest : mbr_kbest.cc mteval ..//utils ..//boost_program_options ; +alias programs : fast_score mbr_kbest ; + +unit-test scorer_test : scorer_test.cc mteval ..//utils ..//z ..//boost_unit_test_framework ; diff --git a/sanity.jam b/sanity.jam new file mode 100644 index 00000000..1b37a4e7 --- /dev/null +++ b/sanity.jam @@ -0,0 +1,184 @@ +import modules ; +import option ; +import os ; +import path ; +import project ; + +#Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php +rule trim-nl ( str extras * ) { +return [ MATCH "([^ +]*)" : $(str) ] $(extras) ; +} +rule _shell ( cmd : extras * ) { + return [ trim-nl [ SHELL $(cmd) : $(extras) ] ] ; +} + +#Run g++ with empty main and these arguments to see if it passes. +rule test_flags ( flags * ) { + local cmd = "bash -c \"g++ "$(flags:J=" ")" -x c++ - <<<'int main() {}' -o /dev/null >/dev/null 2>/dev/null\"" ; + local ret = [ SHELL $(cmd) : exit-status ] ; + if --debug-configuration in [ modules.peek : ARGV ] { + echo $(cmd) ; + echo $(ret) ; + } + if $(ret[2]) = 0 { + return true ; + } else { + return ; + } +} + +rule test_header ( name ) { + return [ test_flags "-include $(name)" ] ; +} + +rule test_library ( name ) { + return [ test_flags "-l$(name)" ] ; +} + +{ + local cleaning = [ option.get "clean" : : yes ] ; + cleaning ?= [ option.get "clean-all" : no : yes ] ; + if "clean" in [ modules.peek : ARGV ] { + cleaning = yes ; + } + constant CLEANING : $(cleaning) ; +} + +#Determine if a library can be compiled statically. +rule auto-shared ( name : additional * ) { + additional ?= "" ; + if [ test_flags $(additional)" -static -l"$(name) ] { + return ; + } else { + return "<link>shared" ; + } +} + +# MacPorts' default location is /opt/local -- use this if no path is given. +with-macports = [ option.get "with-macports" : : "/opt/local" ] ; +if $(with-macports) { + using darwin ; + ECHO "Using --with-macports=$(with-macports), implying use of darwin GCC" ; + + L-boost-search = -L$(with-macports)/lib ; + boost-search = <search>$(with-macports)/lib ; + I-boost-include = -I$(with-macports)/include ; + boost-include = <include>$(with-macports)/include ; +} +else { + with-boost = [ option.get "with-boost" ] ; + if $(with-boost) { + L-boost-search = -L$(with-boost)/lib" "-L$(with-boost)/lib64 ; + boost-search = <search>$(with-boost)/lib <search>$(with-boost)/lib64 ; + I-boost-include = -I$(with-boost)/include ; + boost-include = <include>$(with-boost)/include ; + } else { + L-boost-search = "" ; + boost-search = ; + I-boost-include = "" ; + boost-include = ; + } +} + +#Are we linking static binaries against shared boost? +boost-auto-shared = [ auto-shared "boost_program_options" : $(L-boost-search) ] ; +#Convenience rule for boost libraries. Defines library boost_$(name). +rule boost-lib ( name macro ) { + #Link multi-threaded programs against the -mt version if available. Old + #versions of boost do not have -mt tagged versions of all libraries. Sadly, + #boost.jam does not handle this correctly. + if [ test_flags $(L-boost-search)" -lboost_"$(name)"-mt" ] { + lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name) ; + lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt ; + } else { + lib inner_boost_$(name) : : $(boost-search) <name>boost_$(name) ; + } + + alias boost_$(name) : inner_boost_$(name) : $(boost-auto-shared) : : <link>shared:<define>BOOST_$(macro) $(boost-include) ; +} + +#Argument is e.g. 103600 +rule boost ( min-version ) { + local cmd = "bash -c \"g++ "$(I-boost-include)" -dM -x c++ -E /dev/null -include boost/version.hpp 2>/dev/null |grep '#define BOOST_VERSION '\"" ; + local boost-shell = [ SHELL "$(cmd)" : exit-status ] ; + if $(boost-shell[2]) != 0 && $(CLEANING) = no { + echo Failed to run "$(cmd)" ; + exit Boost does not seem to be installed or g++ is confused. : 1 ; + } + boost-version = [ MATCH "#define BOOST_VERSION ([0-9]*)" : $(boost-shell[1]) ] ; + if $(boost-version) < $(min-version) && $(CLEANING) = no { + exit You have Boost $(boost-version). This package requires Boost at least $(min-version) (and preferably newer). : 1 ; + } + #See tools/build/v2/contrib/boost.jam in a boost distribution for a table of macros to define. + boost-lib thread THREAD_DYN_DLL ; + boost-lib program_options PROGRAM_OPTIONS_DYN_LINK ; + boost-lib unit_test_framework TEST_DYN_LINK ; + boost-lib iostreams IOSTREAMS_DYN_LINK ; + boost-lib serialization SERIALIZATION_DYN_LINK ; + boost-lib mpi MPI_DYN_LINK ; +} + +#Link normally to a library, but sometimes static isn't installed so fall back to dynamic. +rule external-lib ( name : search-path * ) { + lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) ; +} + +#Write the current command line to previous.sh. This does not do shell escaping. +{ + local build-log = $(TOP)/previous.sh ; + if ! [ path.exists $(build-log) ] { + SHELL "touch $(build-log) && chmod +x $(build-log)" ; + } + local script = [ modules.peek : ARGV ] ; + if $(script[1]) = "./jam-files/bjam" { + #The ./bjam shell script calls ./jam-files/bjam so that appears in argv but + #we want ./bjam to appear so the environment variables are set correctly. + script = "./bjam "$(script[2-]:J=" ") ; + } else { + script = $(script:J=" ") ; + } + script = "#!/bin/sh\n$(script)\n" ; + local ignored = @($(build-log):E=$(script)) ; +} + +requirements = ; +{ + local cxxflags = [ os.environ "CXXFLAGS" ] ; + local cflags = [ os.environ "CFLAGS" ] ; + local ldflags = [ os.environ "LDFLAGS" ] ; + + #Boost jam's static clang is buggy. + requirements += <cxxflags>$(cxxflags) <cflags>$(cflags) <linkflags>$(ldflags) <toolset>clang:<link>shared ; + + #libSegFault prints a stack trace on segfault. Link against it if available. + if [ test_flags "-lSegFault" ] { + external-lib SegFault ; + requirements += <library>SegFault ; + } +} + +if [ option.get "git" : : "yes" ] { + local revision = [ _shell "git rev-parse --verify HEAD |head -c 7" ] ; + constant GITTAG : "/"$(revision) ; +} else { + constant GITTAG : "" ; +} + +prefix = [ option.get "prefix" ] ; +if $(prefix) { + prefix = [ path.root $(prefix) [ path.pwd ] ] ; +} else { + prefix = $(TOP)/dist$(GITTAG) ; +} +rule install-bin-libs ( deps * ) { + local bindir = [ option.get "bindir" : $(prefix)/bin ] ; + local libdir = [ option.get "libdir" : $(prefix)/lib ] ; + install prefix-bin : $(deps) : <location>$(bindir) <install-dependencies>on <install-type>EXE <link>shared:<dll-path>$(libdir) ; + install prefix-lib : $(deps) : <location>$(libdir) <install-dependencies>on <install-type>LIB <link>shared:<dll-path>$(libdir) ; +} +rule install-headers ( name : list * : source-root ? ) { + local includedir = [ option.get "includedir" : $(prefix)/include ] ; + source-root ?= "." ; + install $(name) : $(list) : <location>$(includedir) <install-source-root>$(source-root) ; +} diff --git a/utils/Jamfile b/utils/Jamfile new file mode 100644 index 00000000..53a51277 --- /dev/null +++ b/utils/Jamfile @@ -0,0 +1,34 @@ +import testing ; +import option ; + +additional = ; +if [ option.get with-cmph ] { + additional += perfect_hash.cc ; +} + +lib utils : + alignment_io.cc + b64tools.cc + corpus_tools.cc + dict.cc + tdict.cc + fdict.cc + gzstream.cc + filelib.cc + stringlib.cc + sparse_vector.cc + timing_stats.cc + verbose.cc + weights.cc + $(additional) + ..//z + : <include>.. <include>. : : <include>.. <include>. ; + +exe ts : ts.cc utils ; +exe atools : atools.cc utils ..//boost_program_options ; +exe phmt : phmt.cc utils ; +exe reconstruct_weights : reconstruct_weights.cc utils ..//boost_program_options ; + +alias programs : reconstruct_weights atools ; + +all_tests : utils ; |