From 95183b5760d7f168ae093ae8f9b29740628a278f Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 3 Mar 2015 01:14:07 -0500 Subject: migration to cmake --- training/CMakeLists.txt | 14 +++++++ training/Makefile.am | 12 ------ training/const_reorder/CMakeLists.txt | 9 ++++ training/const_reorder/Makefile.am | 8 ---- training/crf/CMakeLists.txt | 78 +++++++++++++++++++++++++++++++++++ training/crf/Makefile.am | 36 ---------------- training/dpmert/CMakeLists.txt | 59 ++++++++++++++++++++++++++ training/dpmert/Makefile.am | 27 ------------ training/dpmert/ces.h | 4 +- training/dtrain/CMakeLists.txt | 15 +++++++ training/dtrain/Makefile.am | 7 ---- training/latent_svm/CMakeLists.txt | 8 ++++ training/latent_svm/Makefile.am | 6 --- training/liblbfgs/CMakeLists.txt | 37 +++++++++++++++++ training/liblbfgs/Makefile.am | 21 ---------- training/minrisk/CMakeLists.txt | 9 ++++ training/minrisk/Makefile.am | 8 ---- training/minrisk/minrisk_optimize.cc | 2 +- training/mira/CMakeLists.txt | 17 ++++++++ training/mira/Makefile.am | 20 --------- training/pro/CMakeLists.txt | 18 ++++++++ training/pro/Makefile.am | 13 ------ training/rampion/CMakeLists.txt | 8 ++++ training/rampion/Makefile.am | 8 ---- training/utils/CMakeLists.txt | 32 ++++++++++++++ training/utils/Makefile.am | 48 --------------------- 26 files changed, 307 insertions(+), 217 deletions(-) create mode 100644 training/CMakeLists.txt delete mode 100644 training/Makefile.am create mode 100644 training/const_reorder/CMakeLists.txt delete mode 100644 training/const_reorder/Makefile.am create mode 100644 training/crf/CMakeLists.txt delete mode 100644 training/crf/Makefile.am create mode 100644 training/dpmert/CMakeLists.txt delete mode 100644 training/dpmert/Makefile.am create mode 100644 training/dtrain/CMakeLists.txt delete mode 100644 training/dtrain/Makefile.am create mode 100644 training/latent_svm/CMakeLists.txt delete mode 100644 training/latent_svm/Makefile.am create mode 100644 training/liblbfgs/CMakeLists.txt delete mode 100644 training/liblbfgs/Makefile.am create mode 100644 training/minrisk/CMakeLists.txt delete mode 100644 training/minrisk/Makefile.am create mode 100644 training/mira/CMakeLists.txt delete mode 100644 training/mira/Makefile.am create mode 100644 training/pro/CMakeLists.txt delete mode 100644 training/pro/Makefile.am create mode 100644 training/rampion/CMakeLists.txt delete mode 100644 training/rampion/Makefile.am create mode 100644 training/utils/CMakeLists.txt delete mode 100644 training/utils/Makefile.am (limited to 'training') diff --git a/training/CMakeLists.txt b/training/CMakeLists.txt new file mode 100644 index 00000000..c0ae42e8 --- /dev/null +++ b/training/CMakeLists.txt @@ -0,0 +1,14 @@ + +add_subdirectory(liblbfgs) +add_subdirectory(utils) +add_subdirectory(crf) +add_subdirectory(minrisk) +add_subdirectory(dpmert) +add_subdirectory(pro) +add_subdirectory(dtrain) +add_subdirectory(latent_svm) +add_subdirectory(mira) +add_subdirectory(rampion) +add_subdirectory(const_reorder) + + diff --git a/training/Makefile.am b/training/Makefile.am deleted file mode 100644 index 2812a9be..00000000 --- a/training/Makefile.am +++ /dev/null @@ -1,12 +0,0 @@ -SUBDIRS = \ - liblbfgs \ - utils \ - crf \ - minrisk \ - dpmert \ - pro \ - dtrain \ - latent_svm \ - mira \ - rampion \ - const_reorder diff --git a/training/const_reorder/CMakeLists.txt b/training/const_reorder/CMakeLists.txt new file mode 100644 index 00000000..da074a23 --- /dev/null +++ b/training/const_reorder/CMakeLists.txt @@ -0,0 +1,9 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) + +set(const_reorder_model_trainer_SRCS constituent_reorder_model.cc trainer.h trainer.cc) +add_executable(const_reorder_model_trainer ${const_reorder_model_trainer_SRCS}) +target_link_libraries(const_reorder_model_trainer utils ${Boost_LIBRARIES} z) + +set(argument_reorder_model_trainer_SRCS argument_reorder_model.cc trainer.h trainer.cc) +add_executable(argument_reorder_model_trainer ${argument_reorder_model_trainer_SRCS}) +target_link_libraries(argument_reorder_model_trainer utils ${Boost_LIBRARIES} z) diff --git a/training/const_reorder/Makefile.am b/training/const_reorder/Makefile.am deleted file mode 100644 index 2c681398..00000000 --- a/training/const_reorder/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -bin_PROGRAMS = const_reorder_model_trainer argument_reorder_model_trainer - -AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder - -const_reorder_model_trainer_SOURCES = constituent_reorder_model.cc trainer.h trainer.cc -const_reorder_model_trainer_LDADD = ../../utils/libutils.a -argument_reorder_model_trainer_SOURCES = argument_reorder_model.cc trainer.h trainer.cc -argument_reorder_model_trainer_LDADD = ../../utils/libutils.a diff --git a/training/crf/CMakeLists.txt b/training/crf/CMakeLists.txt new file mode 100644 index 00000000..4792983e --- /dev/null +++ b/training/crf/CMakeLists.txt @@ -0,0 +1,78 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + + + +########### next target ############### + +set(mpi_batch_optimize_SRCS mpi_batch_optimize.cc cllh_observer.cc cllh_observer.h) +add_executable(mpi_batch_optimize ${mpi_batch_optimize_SRCS}) +target_link_libraries(mpi_batch_optimize training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +########### next target ############### + +set(mpi_adagrad_optimize_SRCS mpi_adagrad_optimize.cc cllh_observer.cc cllh_observer.h) + +add_executable(mpi_adagrad_optimize ${mpi_adagrad_optimize_SRCS}) + +target_link_libraries(mpi_adagrad_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + + +########### next target ############### + +set(mpi_compute_cllh_SRCS mpi_compute_cllh.cc cllh_observer.cc cllh_observer.h) + +add_executable(mpi_compute_cllh ${mpi_compute_cllh_SRCS}) + +target_link_libraries(mpi_compute_cllh ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + + +########### next target ############### + +set(mpi_extract_features_SRCS mpi_extract_features.cc) + +add_executable(mpi_extract_features ${mpi_extract_features_SRCS}) + +target_link_libraries(mpi_extract_features ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + + +########### next target ############### + +set(mpi_extract_reachable_SRCS mpi_extract_reachable.cc) + +add_executable(mpi_extract_reachable ${mpi_extract_reachable_SRCS}) + +target_link_libraries(mpi_extract_reachable ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + +########### next target ############### + +set(mpi_flex_optimize_SRCS mpi_flex_optimize.cc) + +add_executable(mpi_flex_optimize ${mpi_flex_optimize_SRCS}) + +target_link_libraries(mpi_flex_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + +########### next target ############### + +set(mpi_online_optimize_SRCS mpi_online_optimize.cc) + +add_executable(mpi_online_optimize ${mpi_online_optimize_SRCS}) + +target_link_libraries(mpi_online_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +########### next target ############### + +set(mpi_baum_welch_SRCS mpi_baum_welch.cc) + +add_executable(mpi_baum_welch ${mpi_baum_welch_SRCS}) + +target_link_libraries(mpi_baum_welch ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + diff --git a/training/crf/Makefile.am b/training/crf/Makefile.am deleted file mode 100644 index 87d9f838..00000000 --- a/training/crf/Makefile.am +++ /dev/null @@ -1,36 +0,0 @@ -bin_PROGRAMS = \ - mpi_batch_optimize \ - mpi_adagrad_optimize \ - mpi_compute_cllh \ - mpi_extract_features \ - mpi_extract_reachable \ - mpi_flex_optimize \ - mpi_online_optimize \ - mpi_baum_welch - -mpi_baum_welch_SOURCES = mpi_baum_welch.cc -mpi_baum_welch_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_adagrad_optimize_SOURCES = mpi_adagrad_optimize.cc cllh_observer.cc cllh_observer.h -mpi_adagrad_optimize_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_online_optimize_SOURCES = mpi_online_optimize.cc -mpi_online_optimize_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_flex_optimize_SOURCES = mpi_flex_optimize.cc -mpi_flex_optimize_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_extract_reachable_SOURCES = mpi_extract_reachable.cc -mpi_extract_reachable_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_extract_features_SOURCES = mpi_extract_features.cc -mpi_extract_features_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_batch_optimize_SOURCES = mpi_batch_optimize.cc cllh_observer.cc cllh_observer.h -mpi_batch_optimize_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -mpi_compute_cllh_SOURCES = mpi_compute_cllh.cc cllh_observer.cc cllh_observer.h -mpi_compute_cllh_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a $(BOOST_MPI_LDFLAGS) $(BOOST_MPI_LIBS) -lz - -AM_CPPFLAGS = -DBOOST_TEST_DYN_LINK -W -Wall -Wno-sign-compare -I$(top_srcdir)/training -I$(top_srcdir)/training/utils -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval - diff --git a/training/dpmert/CMakeLists.txt b/training/dpmert/CMakeLists.txt new file mode 100644 index 00000000..382af070 --- /dev/null +++ b/training/dpmert/CMakeLists.txt @@ -0,0 +1,59 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(dpmert_SRCS + mert_geometry.cc + ces.cc + error_surface.cc + mr_dpmert_map.cc + line_optimizer.cc + ces.h + error_surface.h + line_optimizer.h + mert_geometry.h) +add_library(dpmert ${dpmert_SRCS}) + +set(mr_dpmert_map_SRCS mr_dpmert_map.cc) + +add_executable(mr_dpmert_map ${mr_dpmert_map_SRCS}) +target_link_libraries(mr_dpmert_map dpmert training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +########### next target ############### + +set(mr_dpmert_reduce_SRCS + mr_dpmert_reduce.cc) + +add_executable(mr_dpmert_reduce ${mr_dpmert_reduce_SRCS}) +target_link_libraries(mr_dpmert_reduce dpmert training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + + +set(mr_dpmert_generate_mapper_input_SRCS mr_dpmert_generate_mapper_input.cc) +add_executable(mr_dpmert_generate_mapper_input ${mr_dpmert_generate_mapper_input_SRCS}) +target_link_libraries(mr_dpmert_generate_mapper_input dpmert training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +set(TEST_SRCS lo_test.cc) +foreach(testSrc ${TEST_SRCS}) + #Extract the filename without an extension (NAME_WE) + get_filename_component(testName ${testSrc} NAME_WE) + + #Add compile target + set_source_files_properties(${testSrc} PROPERTIES COMPILE_FLAGS "-DBOOST_TEST_DYN_LINK -DTEST_DATA=\\\"test_data/\\\"") + add_executable(${testName} ${testSrc}) + + #link to Boost libraries AND your targets and dependencies + target_link_libraries(${testName} dpmert training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES}) + + #I like to move testing binaries into a testBin directory + set_target_properties(${testName} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + + #Finally add it to test execution - + #Notice the WORKING_DIRECTORY and COMMAND + add_test(NAME ${testName} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${testName} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +endforeach(testSrc) + + + diff --git a/training/dpmert/Makefile.am b/training/dpmert/Makefile.am deleted file mode 100644 index b85bb275..00000000 --- a/training/dpmert/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -bin_PROGRAMS = \ - mr_dpmert_map \ - mr_dpmert_reduce \ - mr_dpmert_generate_mapper_input - -noinst_PROGRAMS = \ - lo_test -TESTS = lo_test - -mr_dpmert_generate_mapper_input_SOURCES = mr_dpmert_generate_mapper_input.cc line_optimizer.cc -mr_dpmert_generate_mapper_input_LDADD = ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -# nbest2hg_SOURCES = nbest2hg.cc -# nbest2hg_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lfst - -mr_dpmert_map_SOURCES = mert_geometry.cc ces.cc error_surface.cc mr_dpmert_map.cc line_optimizer.cc ces.h error_surface.h line_optimizer.h mert_geometry.h -mr_dpmert_map_LDADD = ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -mr_dpmert_reduce_SOURCES = error_surface.cc ces.cc mr_dpmert_reduce.cc line_optimizer.cc mert_geometry.cc ces.h error_surface.h line_optimizer.h mert_geometry.h -mr_dpmert_reduce_LDADD = ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -lo_test_SOURCES = lo_test.cc ces.cc mert_geometry.cc error_surface.cc line_optimizer.cc ces.h error_surface.h line_optimizer.h mert_geometry.h -lo_test_LDADD = $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_UNIT_TEST_FRAMEWORK_LIBS) ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -EXTRA_DIST = test_data dpmert.pl - -AM_CPPFLAGS = -DTEST_DATA=\"$(top_srcdir)/training/dpmert/test_data\" -DBOOST_TEST_DYN_LINK -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval diff --git a/training/dpmert/ces.h b/training/dpmert/ces.h index e4fa2080..b179499d 100644 --- a/training/dpmert/ces.h +++ b/training/dpmert/ces.h @@ -1,9 +1,9 @@ #ifndef _CES_H_ #define _CES_H_ -class ConvexHull; +struct ConvexHull; class Hypergraph; -class SegmentEvaluator; +struct SegmentEvaluator; class ErrorSurface; class EvaluationMetric; diff --git a/training/dtrain/CMakeLists.txt b/training/dtrain/CMakeLists.txt new file mode 100644 index 00000000..653cfe2b --- /dev/null +++ b/training/dtrain/CMakeLists.txt @@ -0,0 +1,15 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(dtrain_SRCS + dtrain.cc + score.cc + dtrain.h + kbestget.h + ksampler.h + pairsampling.h + score.h) +add_executable(dtrain ${dtrain_SRCS}) +target_link_libraries(dtrain libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) diff --git a/training/dtrain/Makefile.am b/training/dtrain/Makefile.am deleted file mode 100644 index 844c790d..00000000 --- a/training/dtrain/Makefile.am +++ /dev/null @@ -1,7 +0,0 @@ -bin_PROGRAMS = dtrain - -dtrain_SOURCES = dtrain.cc score.cc dtrain.h kbestget.h ksampler.h pairsampling.h score.h -dtrain_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval - diff --git a/training/latent_svm/CMakeLists.txt b/training/latent_svm/CMakeLists.txt new file mode 100644 index 00000000..ec48e02f --- /dev/null +++ b/training/latent_svm/CMakeLists.txt @@ -0,0 +1,8 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(latent_svm_SRCS latent_svm.cc) +add_executable(latent_svm ${latent_svm_SRCS}) +target_link_libraries(latent_svm libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) diff --git a/training/latent_svm/Makefile.am b/training/latent_svm/Makefile.am deleted file mode 100644 index 65c5e038..00000000 --- a/training/latent_svm/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -bin_PROGRAMS = latent_svm - -latent_svm_SOURCES = latent_svm.cc -latent_svm_LDADD = ../..//decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval diff --git a/training/liblbfgs/CMakeLists.txt b/training/liblbfgs/CMakeLists.txt new file mode 100644 index 00000000..d936c0ab --- /dev/null +++ b/training/liblbfgs/CMakeLists.txt @@ -0,0 +1,37 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) + + +set(lbfgs_STAT_SRCS + lbfgs.c + arithmetic_ansi.h + arithmetic_sse_double.h + arithmetic_sse_float.h + lbfgs++.h + lbfgs.h) + +add_library(lbfgs STATIC ${lbfgs_STAT_SRCS}) + +set(TEST_SRCS ll_test.cc) + +foreach(testSrc ${TEST_SRCS}) + #Extract the filename without an extension (NAME_WE) + get_filename_component(testName ${testSrc} NAME_WE) + + #Add compile target + set_source_files_properties(${testSrc} PROPERTIES COMPILE_FLAGS "-DBOOST_TEST_DYN_LINK -DTEST_DATA=\\\"test_data/\\\"") + add_executable(${testName} ${testSrc}) + + #link to Boost libraries AND your targets and dependencies + target_link_libraries(${testName} lbfgs ${Boost_LIBRARIES} ${ZLIB_LIBRARIES}) + + #I like to move testing binaries into a testBin directory + set_target_properties(${testName} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + + #Finally add it to test execution - + #Notice the WORKING_DIRECTORY and COMMAND + add_test(NAME ${testName} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${testName} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +endforeach(testSrc) + + diff --git a/training/liblbfgs/Makefile.am b/training/liblbfgs/Makefile.am deleted file mode 100644 index aa3af377..00000000 --- a/training/liblbfgs/Makefile.am +++ /dev/null @@ -1,21 +0,0 @@ -noinst_LIBRARIES = liblbfgs.a - -noinst_PROGRAMS = ll_test -ll_test_SOURCES = ll_test.cc -ll_test_LDADD = liblbfgs.a -lz -TESTS = ll_test - -liblbfgs_a_SOURCES = \ - lbfgs.c \ - arithmetic_ansi.h \ - arithmetic_sse_double.h \ - arithmetic_sse_float.h \ - lbfgs++.h \ - lbfgs.h - -################################################################ -# do NOT NOT NOT add any other -I includes NO NO NO NO NO ###### -AM_LDFLAGS = liblbfgs.a -lz -AM_CPPFLAGS = -DBOOST_TEST_DYN_LINK -W -Wall -I$(top_srcdir)/training -I$(top_srcdir)/training/liblbfgs -################################################################ - diff --git a/training/minrisk/CMakeLists.txt b/training/minrisk/CMakeLists.txt new file mode 100644 index 00000000..0795bc54 --- /dev/null +++ b/training/minrisk/CMakeLists.txt @@ -0,0 +1,9 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(minrisk_optimize_SRCS minrisk_optimize.cc) +add_executable(minrisk_optimize ${minrisk_optimize_SRCS}) +target_link_libraries(minrisk_optimize training_utils libcdec mteval utils lbfgs ${Boost_LIBRARIES} z) diff --git a/training/minrisk/Makefile.am b/training/minrisk/Makefile.am deleted file mode 100644 index ebf6fa91..00000000 --- a/training/minrisk/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -bin_PROGRAMS = minrisk_optimize - -minrisk_optimize_SOURCES = minrisk_optimize.cc -minrisk_optimize_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a ../../training/liblbfgs/liblbfgs.a - -EXTRA_DIST = minrisk.pl minrisk_generate_input.pl - -AM_CPPFLAGS = -W -Wall -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training -I$(top_srcdir)/training/utils diff --git a/training/minrisk/minrisk_optimize.cc b/training/minrisk/minrisk_optimize.cc index a2938fb0..f90c7118 100644 --- a/training/minrisk/minrisk_optimize.cc +++ b/training/minrisk/minrisk_optimize.cc @@ -163,7 +163,7 @@ int main(int argc, char** argv) { cerr << "Loading hypergraphs...\n"; while(getline(in, line)) { istringstream is(line); - int sent_id; + int sent_id = 0; kis.resize(kis.size() + 1); training::CandidateSet& curkbest = kis.back(); string kbest_file; diff --git a/training/mira/CMakeLists.txt b/training/mira/CMakeLists.txt new file mode 100644 index 00000000..3a8fa516 --- /dev/null +++ b/training/mira/CMakeLists.txt @@ -0,0 +1,17 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(kbest_mira_SRCS kbest_mira.cc) +add_executable(kbest_mira ${kbest_mira_SRCS}) +target_link_libraries(kbest_mira ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +set(kbest_cut_mira_SRCS kbest_cut_mira.cc) +add_executable(kbest_cut_mira ${kbest_cut_mira_SRCS}) +target_link_libraries(kbest_cut_mira ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +set(ada_opt_sm_SRCS ada_opt_sm.cc) +add_executable(ada_opt_sm ${ada_opt_sm_SRCS}) +target_link_libraries(ada_opt_sm ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + diff --git a/training/mira/Makefile.am b/training/mira/Makefile.am deleted file mode 100644 index a318cf6e..00000000 --- a/training/mira/Makefile.am +++ /dev/null @@ -1,20 +0,0 @@ -bin_PROGRAMS = \ - kbest_mira \ - kbest_cut_mira \ - ada_opt_sm - -EXTRA_DIST = mira.py - -ada_opt_sm_SOURCES = ada_opt_sm.cc -ada_opt_sm_LDFLAGS= -rdynamic -ada_opt_sm_LDADD = ../utils/libtraining_utils.a ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a - -kbest_mira_SOURCES = kbest_mira.cc -kbest_mira_LDFLAGS= -rdynamic -kbest_mira_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a - -kbest_cut_mira_SOURCES = kbest_cut_mira.cc -kbest_cut_mira_LDFLAGS= -rdynamic -kbest_cut_mira_LDADD = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training/utils diff --git a/training/pro/CMakeLists.txt b/training/pro/CMakeLists.txt new file mode 100644 index 00000000..5ee09a74 --- /dev/null +++ b/training/pro/CMakeLists.txt @@ -0,0 +1,18 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +########### next target ############### + +set(mr_pro_map_SRCS mr_pro_map.cc) +add_executable(mr_pro_map ${mr_pro_map_SRCS}) +target_link_libraries(mr_pro_map training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) + +########### next target ############### + +set(mr_pro_reduce_SRCS mr_pro_reduce.cc) +add_executable(mr_pro_reduce ${mr_pro_reduce_SRCS}) +target_link_libraries(mr_pro_reduce lbfgs utils ${Boost_LIBRARIES} z) + diff --git a/training/pro/Makefile.am b/training/pro/Makefile.am deleted file mode 100644 index 09364804..00000000 --- a/training/pro/Makefile.am +++ /dev/null @@ -1,13 +0,0 @@ -bin_PROGRAMS = \ - mr_pro_map \ - mr_pro_reduce - -mr_pro_map_SOURCES = mr_pro_map.cc -mr_pro_map_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -mr_pro_reduce_SOURCES = mr_pro_reduce.cc -mr_pro_reduce_LDADD = ../../training/liblbfgs/liblbfgs.a ../../utils/libutils.a - -EXTRA_DIST = mr_pro_generate_mapper_input.pl pro.pl - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training/utils -I$(top_srcdir)/training diff --git a/training/rampion/CMakeLists.txt b/training/rampion/CMakeLists.txt new file mode 100644 index 00000000..8a60b5ff --- /dev/null +++ b/training/rampion/CMakeLists.txt @@ -0,0 +1,8 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(rampion_cccp_SRCS rampion_cccp.cc) +add_executable(rampion_cccp ${rampion_cccp_SRCS}) +target_link_libraries(rampion_cccp training_utils libcdec mteval utils ${Boost_LIBRARIES} z) diff --git a/training/rampion/Makefile.am b/training/rampion/Makefile.am deleted file mode 100644 index c72283cd..00000000 --- a/training/rampion/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -bin_PROGRAMS = rampion_cccp - -rampion_cccp_SOURCES = rampion_cccp.cc -rampion_cccp_LDADD = ../../training/utils/libtraining_utils.a ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -EXTRA_DIST = rampion.pl rampion_generate_input.pl - -AM_CPPFLAGS = -W -Wall $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training/utils diff --git a/training/utils/CMakeLists.txt b/training/utils/CMakeLists.txt new file mode 100644 index 00000000..1734ee73 --- /dev/null +++ b/training/utils/CMakeLists.txt @@ -0,0 +1,32 @@ +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) + +set(training_utils_STAT_SRCS + candidate_set.h + entropy.h + lbfgs.h + online_optimizer.h + optimize.h + risk.h + sentserver.h + candidate_set.cc + entropy.cc + optimize.cc + online_optimizer.cc + risk.cc) + +add_library(training_utils STATIC ${training_utils_STAT_SRCS}) + + +set(sentserver_SRCS sentserver.cc) +add_executable(sentserver ${sentserver_SRCS}) +target_link_libraries(sentserver) + +set(sentclient_SRCS sentclient.cc) +add_executable(sentclient ${sentclient_SRCS}) +target_link_libraries(sentclient) + +set(grammar_convert_SRCS grammar_convert.cc) +add_executable(grammar_convert ${grammar_convert_SRCS}) +target_link_libraries(grammar_convert libcdec mteval utils ${Boost_LIBRARIES} z) diff --git a/training/utils/Makefile.am b/training/utils/Makefile.am deleted file mode 100644 index edaaf3d4..00000000 --- a/training/utils/Makefile.am +++ /dev/null @@ -1,48 +0,0 @@ -noinst_LIBRARIES = libtraining_utils.a - -bin_PROGRAMS = \ - sentserver \ - sentclient \ - grammar_convert - -noinst_PROGRAMS = \ - lbfgs_test \ - optimize_test - -EXTRA_DIST = decode-and-evaluate.pl libcall.pl parallelize.pl - -sentserver_SOURCES = sentserver.cc -sentserver_LDFLAGS = $(PTHREAD_LIBS) -sentserver_CXXFLAGS = $(PTHREAD_CFLAGS) - -sentclient_SOURCES = sentclient.cc -sentclient_LDFLAGS = $(PTHREAD_LIBS) -sentclient_CXXFLAGS = $(PTHREAD_CFLAGS) - -TESTS = lbfgs_test optimize_test - -libtraining_utils_a_SOURCES = \ - candidate_set.h \ - entropy.h \ - lbfgs.h \ - online_optimizer.h \ - optimize.h \ - risk.h \ - sentserver.h \ - candidate_set.cc \ - entropy.cc \ - optimize.cc \ - online_optimizer.cc \ - risk.cc - -optimize_test_SOURCES = optimize_test.cc -optimize_test_LDADD = libtraining_utils.a ../../utils/libutils.a - -grammar_convert_SOURCES = grammar_convert.cc -grammar_convert_LDADD = ../../decoder/libcdec.a ../../mteval/libmteval.a ../../utils/libutils.a - -lbfgs_test_SOURCES = lbfgs_test.cc -lbfgs_test_LDADD = ../../utils/libutils.a - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/decoder -I$(top_srcdir)/utils -I$(top_srcdir)/mteval -I$(top_srcdir)/klm - -- cgit v1.2.3 From 06f1b6aedbd96d652d5337cf1b93b51e4dd9a620 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 4 Mar 2015 21:36:51 -0500 Subject: remove perfect hash function stuff, add zip option to extract.cc --- extractor/CMakeLists.txt | 17 ++++++++++---- extractor/extract.cc | 14 ++++++++---- training/mira/mira.py | 59 +++++++++++++++++++++++------------------------- utils/CMakeLists.txt | 1 - utils/perfect_hash.cc | 37 ------------------------------ utils/perfect_hash.h | 27 ---------------------- utils/phmt.cc | 42 ---------------------------------- 7 files changed, 49 insertions(+), 148 deletions(-) delete mode 100644 utils/perfect_hash.cc delete mode 100644 utils/perfect_hash.h delete mode 100644 utils/phmt.cc (limited to 'training') diff --git a/extractor/CMakeLists.txt b/extractor/CMakeLists.txt index d37c6b95..1cf8533b 100644 --- a/extractor/CMakeLists.txt +++ b/extractor/CMakeLists.txt @@ -1,9 +1,17 @@ -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../utils) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/features) -find_package(GTest) -find_package(GMock) +find_package(OpenMP) +if (OPENMP_FOUND) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") +endif() + +find_package(GTest REQUIRED) +find_package(GMock REQUIRED) if(GMOCK_FOUND) + #rule_factory_test.cc set(TEST_SRCS alignment_test.cc data_array_test.cc fast_intersector_test.cc @@ -15,7 +23,6 @@ if(GMOCK_FOUND) precomputation_test.cc rule_extractor_helper_test.cc rule_extractor_test.cc - rule_factory_test.cc scorer2_test.cc suffix_array_sampler_test.cc suffix_array_test.cc @@ -56,7 +63,7 @@ target_link_libraries(run_extractor extractor ${Boost_LIBRARIES} z) set(extract_SRCS extract.cc) add_executable(extract ${extract_SRCS}) -target_link_libraries(extract extractor ${Boost_LIBRARIES} z) +target_link_libraries(extract extractor utils ${Boost_LIBRARIES} z) set(extractor_STAT_SRCS diff --git a/extractor/extract.cc b/extractor/extract.cc index e5b6f6ff..08f209cc 100644 --- a/extractor/extract.cc +++ b/extractor/extract.cc @@ -14,6 +14,7 @@ const unsigned omp_get_num_threads() { return 1; } #endif +#include "filelib.h" #include "alignment.h" #include "data_array.h" #include "features/count_source_target.h" @@ -42,8 +43,8 @@ using namespace features; using namespace std; // Returns the file path in which a given grammar should be written. -fs::path GetGrammarFilePath(const fs::path& grammar_path, int file_number) { - string file_name = "grammar." + to_string(file_number); +fs::path GetGrammarFilePath(const fs::path& grammar_path, int file_number, bool use_zip) { + string file_name = "grammar." + to_string(file_number) + (use_zip ? ".gz" : ""); return grammar_path / file_name; } @@ -58,6 +59,7 @@ int main(int argc, char** argv) { ("threads,t", po::value()->required()->default_value(1), threads_option.c_str()) ("grammars,g", po::value()->required(), "Grammars output path") + ("gzip,z", "Gzip grammars") ("max_rule_span", po::value()->default_value(15), "Maximum rule span") ("max_rule_symbols", po::value()->default_value(5), @@ -205,12 +207,14 @@ int main(int argc, char** argv) { vm["max_rule_symbols"].as(), vm["max_samples"].as(), vm["tight_phrases"].as()); + const bool use_zip = vm.count("gzip"); // Creates the grammars directory if it doesn't exist. fs::path grammar_path = vm["grammars"].as(); if (!fs::is_directory(grammar_path)) { fs::create_directory(grammar_path); } + grammar_path = fs::canonical(grammar_path); // Reads all sentences for which we extract grammar rules (the paralellization // is simplified if we read all sentences upfront). @@ -239,12 +243,12 @@ int main(int argc, char** argv) { } Grammar grammar = extractor.GetGrammar( sentences[i], blacklisted_sentence_ids); - ofstream output(GetGrammarFilePath(grammar_path, i).c_str()); - output << grammar; + WriteFile wf(GetGrammarFilePath(grammar_path, i, use_zip).c_str()); + *wf.stream() << grammar; } for (size_t i = 0; i < sentences.size(); ++i) { - cout << " " << sentences[i] << " " << suffixes[i] << endl; } diff --git a/training/mira/mira.py b/training/mira/mira.py index ec9c2d64..ccecb10e 100755 --- a/training/mira/mira.py +++ b/training/mira/mira.py @@ -5,12 +5,6 @@ import argparse import logging import random, time import gzip, itertools -try: - import cdec.score -except ImportError: - sys.stderr.write('Could not import pycdec, see cdec/python/README.md for details\n') - sys.exit(1) -have_mpl = True try: import matplotlib matplotlib.use('Agg') @@ -19,26 +13,33 @@ except ImportError: have_mpl = False #mira run script -#requires pycdec to be built, since it is used for scoring hypothesis -#translations. #matplotlib must be installed for graphing to work #email option requires mail +script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) +fast_score_binary = script_dir+'/../../mteval/fast_score' +dlog = None + #scoring function using pycdec scoring def fast_score(hyps, refs, metric): - scorer = cdec.score.Scorer(metric) - logging.info('loaded {0} references for scoring with {1}'.format( - len(refs), metric)) - if metric=='BLEU': - logging.warning('BLEU is ambiguous, assuming IBM_BLEU\n') - metric = 'IBM_BLEU' - elif metric=='COMBI': - logging.warning('COMBI metric is no longer supported, switching to ' - 'COMB:TER=-0.5;BLEU=0.5') - metric = 'COMB:TER=-0.5;BLEU=0.5' - stats = sum(scorer(r).evaluate(h) for h,r in itertools.izip(hyps,refs)) - logging.info('Score={} ({})'.format(stats.score, stats.detail)) - return stats.score + #scorer = cdec.score.Scorer(metric) + #logging.info('loaded {0} references for scoring with {1}'.format( + # len(refs), metric)) + #if metric=='BLEU': + # logging.warning('BLEU is ambiguous, assuming IBM_BLEU\n') + # metric = 'IBM_BLEU' + #elif metric=='COMBI': + # logging.warning('COMBI metric is no longer supported, switching to ' + # 'COMB:TER=-0.5;BLEU=0.5') + # metric = 'COMB:TER=-0.5;BLEU=0.5' + #stats = sum(scorer(r).evaluate(h) for h,r in itertools.izip(hyps,refs)) + #logging.info('Score={} ({})'.format(stats.score, stats.detail)) + #return stats.score + cmd = ('{0} -r{1} -i {2} -m {3}').format(fast_score_binary, refs, hyps, metric) + proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + o = proc.stdout.readline().strip() + print 'res: ', o + return float(o) #create new parallel input file in output directory in sgml format def enseg(devfile, newfile, gprefix): @@ -81,7 +82,6 @@ def enseg(devfile, newfile, gprefix): def main(): #set logging to write all info messages to stderr logging.basicConfig(level=logging.INFO) - script_dir = os.path.dirname(os.path.abspath(sys.argv[0])) if not have_mpl: logging.warning('Failed to import matplotlib, graphs will not be generated.') @@ -373,7 +373,8 @@ def optimize(args, script_dir, dev_size): cmd = parallel_cmd + ' ' + decoder_cmd logging.info('OPTIMIZATION COMMAND: {}'.format(cmd)) - + + global dlog dlog = open(decoderlog,'w') runf = open(runfile,'w') retries = 0 @@ -420,7 +421,7 @@ def optimize(args, script_dir, dev_size): bests = [] fears = [] for line in run: - hope, best, fear = line.split(' ||| ') + hope, best, fear = line.strip().split(' ||| ') hopes.append(hope) bests.append(best) fears.append(fear) @@ -436,14 +437,10 @@ def optimize(args, script_dir, dev_size): gzip_file(runfile) gzip_file(decoderlog) - ref_file = open(refs) - references = [line.split(' ||| ') for line in - ref_file.read().strip().split('\n')] - ref_file.close() #get score for best hypothesis translations, hope and fear translations - dec_score = fast_score(bests, references, args.metric) - dec_score_h = fast_score(hopes, references, args.metric) - dec_score_f = fast_score(fears, references, args.metric) + dec_score = fast_score(runfile+'.B', refs, args.metric) + dec_score_h = fast_score(runfile+'.H', refs, args.metric) + dec_score_f = fast_score(runfile+'.F', refs, args.metric) hope_best_fear['hope'].append(dec_score) hope_best_fear['best'].append(dec_score_h) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 17436263..59fb644d 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -70,7 +70,6 @@ set(utils_STAT_SRCS named_enum.h null_deleter.h null_traits.h - perfect_hash.h prob.h sampler.h semiring.h diff --git a/utils/perfect_hash.cc b/utils/perfect_hash.cc deleted file mode 100644 index 706e2741..00000000 --- a/utils/perfect_hash.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "config.h" - -#ifdef HAVE_CMPH - -#include "perfect_hash.h" - -#include -#include - -using namespace std; - -PerfectHashFunction::~PerfectHashFunction() { - cmph_destroy(mphf_); -} - -PerfectHashFunction::PerfectHashFunction(const string& fname) { - FILE* f = fopen(fname.c_str(), "r"); - if (!f) { - cerr << "Failed to open file " << fname << " for reading: cannot load hash function.\n"; - abort(); - } - mphf_ = cmph_load(f); - if (!mphf_) { - cerr << "cmph_load failed on " << fname << "!\n"; - abort(); - } -} - -size_t PerfectHashFunction::operator()(const string& key) const { - return cmph_search(mphf_, &key[0], key.size()); -} - -size_t PerfectHashFunction::number_of_keys() const { - return cmph_size(mphf_); -} - -#endif diff --git a/utils/perfect_hash.h b/utils/perfect_hash.h deleted file mode 100644 index 8c12c9f0..00000000 --- a/utils/perfect_hash.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef PERFECT_HASH_MAP_H_ -#define PERFECT_HASH_MAP_H_ - -#include -#include - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifdef HAVE_CMPH -#include "cmph.h" -#endif - -class PerfectHashFunction : boost::noncopyable { - public: - explicit PerfectHashFunction(const std::string& fname); - ~PerfectHashFunction(); - size_t operator()(const std::string& key) const; - size_t number_of_keys() const; - private: -#ifdef HAVE_CMPH - cmph_t *mphf_; -#endif -}; - -#endif diff --git a/utils/phmt.cc b/utils/phmt.cc deleted file mode 100644 index b17febf6..00000000 --- a/utils/phmt.cc +++ /dev/null @@ -1,42 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifndef HAVE_CMPH -int main() { - return 0; -} -#else - -#include -#include "weights.h" -#include "fdict.h" - -using namespace std; - -int main(int argc, char** argv) { - if (argc != 2) { cerr << "Usage: " << argv[0] << " file.mphf\n"; return 1; } - FD::EnableHash(argv[1]); - cerr << "Number of keys: " << FD::NumFeats() << endl; - cerr << "LexFE = " << FD::Convert("LexFE") << endl; - cerr << "LexEF = " << FD::Convert("LexEF") << endl; - { - vector v(FD::NumFeats()); - v[FD::Convert("LexFE")] = 1.0; - v[FD::Convert("LexEF")] = 0.5; - cerr << "Writing...\n"; - Weights::WriteToFile("weights.bin", v); - cerr << "Done.\n"; - } - { - vector v(FD::NumFeats()); - cerr << "Reading...\n"; - Weights::InitFromFile("weights.bin", &v); - cerr << "Done.\n"; - assert(v[FD::Convert("LexFE")] == 1.0); - assert(v[FD::Convert("LexEF")] == 0.5); - } -} - -#endif - -- cgit v1.2.3 From 3d2a60701f053fc35130711e88d9fcb29f29e274 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 4 Mar 2015 21:42:29 -0500 Subject: remove warning --- training/pro/mr_pro_reduce.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'training') diff --git a/training/pro/mr_pro_reduce.cc b/training/pro/mr_pro_reduce.cc index 5ef9b470..a61a3a5f 100644 --- a/training/pro/mr_pro_reduce.cc +++ b/training/pro/mr_pro_reduce.cc @@ -18,7 +18,6 @@ namespace po = boost::program_options; // since this is a ranking model, there should be equal numbers of // positive and negative examples, so the bias should be 0 -static const double MAX_BIAS = 1e-10; void InitCommandLine(int argc, char** argv, po::variables_map* conf) { po::options_description opts("Configuration options"); -- cgit v1.2.3 From 42e6a2888f986b062c6391b6bef6ef817c6b8a68 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 4 Mar 2015 22:46:33 -0500 Subject: deal with libdl --- .gitignore | 1 + CMakeLists.txt | 1 + cmake/FindLibDL.cmake | 30 ++++++++++++++++++++++++++++++ decoder/CMakeLists.txt | 4 +++- extractor/CMakeLists.txt | 10 ++++++---- training/crf/CMakeLists.txt | 22 ++++++++-------------- training/dtrain/CMakeLists.txt | 2 +- training/latent_svm/CMakeLists.txt | 2 +- training/mira/CMakeLists.txt | 7 +++---- training/utils/CMakeLists.txt | 6 ++++-- utils/CMakeLists.txt | 2 +- 11 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 cmake/FindLibDL.cmake (limited to 'training') diff --git a/.gitignore b/.gitignore index 81ede92e..07145db9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +example_extff/libff_example.so* Testing/ */Testing/ training/Testing/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 21c2a230..934c69e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) #### packages find_package(ZLIB REQUIRED) find_package(BZip2 REQUIRED) +find_package(LibDL REQUIRED) # for pycdec find_package(PythonInterp 2.7 REQUIRED) diff --git a/cmake/FindLibDL.cmake b/cmake/FindLibDL.cmake new file mode 100644 index 00000000..1689e4c7 --- /dev/null +++ b/cmake/FindLibDL.cmake @@ -0,0 +1,30 @@ +# - Find libdl +# Find the native LIBDL includes and library +# +# LIBDL_INCLUDE_DIR - where to find dlfcn.h, etc. +# LIBDL_LIBRARIES - List of libraries when using libdl. +# LIBDL_FOUND - True if libdl found. + + +IF (LIBDL_INCLUDE_DIR) + # Already in cache, be silent + SET(LIBDL_FIND_QUIETLY TRUE) +ENDIF (LIBDL_INCLUDE_DIR) + +FIND_PATH(LIBDL_INCLUDE_DIR dlfcn.h) + +SET(LIBDL_NAMES dl libdl ltdl libltdl) +FIND_LIBRARY(LIBDL_LIBRARY NAMES ${LIBDL_NAMES} ) + +# handle the QUIETLY and REQUIRED arguments and set LIBDL_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibDL DEFAULT_MSG LIBDL_LIBRARY LIBDL_INCLUDE_DIR) + +IF(LIBDL_FOUND) + SET( LIBDL_LIBRARIES ${LIBDL_LIBRARY} ) +ELSE(LIBDL_FOUND) + SET( LIBDL_LIBRARIES ) +ENDIF(LIBDL_FOUND) + +MARK_AS_ADVANCED( LIBDL_LIBRARY LIBDL_INCLUDE_DIR ) diff --git a/decoder/CMakeLists.txt b/decoder/CMakeLists.txt index 591ab7b3..f7683887 100644 --- a/decoder/CMakeLists.txt +++ b/decoder/CMakeLists.txt @@ -3,6 +3,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../mteval) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../klm) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + PROJECT(decoder C CXX) find_package(FLEX REQUIRED) @@ -145,7 +147,7 @@ add_library(libcdec STATIC ${libcdec_SRCS}) set(cdec_SRCS cdec.cc) add_executable(cdec ${cdec_SRCS}) -target_link_libraries(cdec libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES}) +target_link_libraries(cdec libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${LIBDL_LIBRARIES}) set(TEST_SRCS grammar_test.cc diff --git a/extractor/CMakeLists.txt b/extractor/CMakeLists.txt index 1cf8533b..93a524cc 100644 --- a/extractor/CMakeLists.txt +++ b/extractor/CMakeLists.txt @@ -8,9 +8,10 @@ if (OPENMP_FOUND) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") endif() -find_package(GTest REQUIRED) -find_package(GMock REQUIRED) -if(GMOCK_FOUND) +find_package(GTest) +find_package(GMock) +if(GTEST_FOUND) + if(GMOCK_FOUND) #rule_factory_test.cc set(TEST_SRCS alignment_test.cc data_array_test.cc @@ -50,7 +51,8 @@ if(GMOCK_FOUND) add_test(NAME ${testName} COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${testName} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endforeach(testSrc) -endif(GMOCK_FOUND) + endif(GMOCK_FOUND) +endif(GTEST_FOUND) set(sacompile_SRCS sacompile.cc) add_executable(sacompile ${sacompile_SRCS}) diff --git a/training/crf/CMakeLists.txt b/training/crf/CMakeLists.txt index 4792983e..85fab7a0 100644 --- a/training/crf/CMakeLists.txt +++ b/training/crf/CMakeLists.txt @@ -9,7 +9,7 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) set(mpi_batch_optimize_SRCS mpi_batch_optimize.cc cllh_observer.cc cllh_observer.h) add_executable(mpi_batch_optimize ${mpi_batch_optimize_SRCS}) -target_link_libraries(mpi_batch_optimize training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(mpi_batch_optimize training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -17,8 +17,7 @@ set(mpi_adagrad_optimize_SRCS mpi_adagrad_optimize.cc cllh_observer.cc cllh_obse add_executable(mpi_adagrad_optimize ${mpi_adagrad_optimize_SRCS}) -target_link_libraries(mpi_adagrad_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_adagrad_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -27,8 +26,7 @@ set(mpi_compute_cllh_SRCS mpi_compute_cllh.cc cllh_observer.cc cllh_observer.h) add_executable(mpi_compute_cllh ${mpi_compute_cllh_SRCS}) -target_link_libraries(mpi_compute_cllh ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_compute_cllh ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -37,8 +35,7 @@ set(mpi_extract_features_SRCS mpi_extract_features.cc) add_executable(mpi_extract_features ${mpi_extract_features_SRCS}) -target_link_libraries(mpi_extract_features ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_extract_features ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -47,8 +44,7 @@ set(mpi_extract_reachable_SRCS mpi_extract_reachable.cc) add_executable(mpi_extract_reachable ${mpi_extract_reachable_SRCS}) -target_link_libraries(mpi_extract_reachable ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_extract_reachable ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -56,8 +52,7 @@ set(mpi_flex_optimize_SRCS mpi_flex_optimize.cc) add_executable(mpi_flex_optimize ${mpi_flex_optimize_SRCS}) -target_link_libraries(mpi_flex_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_flex_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -65,7 +60,7 @@ set(mpi_online_optimize_SRCS mpi_online_optimize.cc) add_executable(mpi_online_optimize ${mpi_online_optimize_SRCS}) -target_link_libraries(mpi_online_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(mpi_online_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) ########### next target ############### @@ -73,6 +68,5 @@ set(mpi_baum_welch_SRCS mpi_baum_welch.cc) add_executable(mpi_baum_welch ${mpi_baum_welch_SRCS}) -target_link_libraries(mpi_baum_welch ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(mpi_baum_welch ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) diff --git a/training/dtrain/CMakeLists.txt b/training/dtrain/CMakeLists.txt index 653cfe2b..72e80b6b 100644 --- a/training/dtrain/CMakeLists.txt +++ b/training/dtrain/CMakeLists.txt @@ -12,4 +12,4 @@ set(dtrain_SRCS pairsampling.h score.h) add_executable(dtrain ${dtrain_SRCS}) -target_link_libraries(dtrain libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(dtrain libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) diff --git a/training/latent_svm/CMakeLists.txt b/training/latent_svm/CMakeLists.txt index ec48e02f..332b8d42 100644 --- a/training/latent_svm/CMakeLists.txt +++ b/training/latent_svm/CMakeLists.txt @@ -5,4 +5,4 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) set(latent_svm_SRCS latent_svm.cc) add_executable(latent_svm ${latent_svm_SRCS}) -target_link_libraries(latent_svm libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(latent_svm libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) diff --git a/training/mira/CMakeLists.txt b/training/mira/CMakeLists.txt index 3a8fa516..bba9ef5f 100644 --- a/training/mira/CMakeLists.txt +++ b/training/mira/CMakeLists.txt @@ -5,13 +5,12 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) set(kbest_mira_SRCS kbest_mira.cc) add_executable(kbest_mira ${kbest_mira_SRCS}) -target_link_libraries(kbest_mira ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(kbest_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) set(kbest_cut_mira_SRCS kbest_cut_mira.cc) add_executable(kbest_cut_mira ${kbest_cut_mira_SRCS}) -target_link_libraries(kbest_cut_mira ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) +target_link_libraries(kbest_cut_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) set(ada_opt_sm_SRCS ada_opt_sm.cc) add_executable(ada_opt_sm ${ada_opt_sm_SRCS}) -target_link_libraries(ada_opt_sm ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z) - +target_link_libraries(ada_opt_sm training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES}) diff --git a/training/utils/CMakeLists.txt b/training/utils/CMakeLists.txt index 1734ee73..2753b9f0 100644 --- a/training/utils/CMakeLists.txt +++ b/training/utils/CMakeLists.txt @@ -2,6 +2,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../utils) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../mteval) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder) +find_package(Threads REQUIRED) + set(training_utils_STAT_SRCS candidate_set.h entropy.h @@ -21,11 +23,11 @@ add_library(training_utils STATIC ${training_utils_STAT_SRCS}) set(sentserver_SRCS sentserver.cc) add_executable(sentserver ${sentserver_SRCS}) -target_link_libraries(sentserver) +target_link_libraries(sentserver ${CMAKE_THREAD_LIBS_INIT}) set(sentclient_SRCS sentclient.cc) add_executable(sentclient ${sentclient_SRCS}) -target_link_libraries(sentclient) +target_link_libraries(sentclient ${CMAKE_THREAD_LIBS_INIT}) set(grammar_convert_SRCS grammar_convert.cc) add_executable(grammar_convert ${grammar_convert_SRCS}) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 59fb644d..58668254 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,4 +1,4 @@ -include_directories() +set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(TEST_SRCS dict_test.cc logval_test.cc -- cgit v1.2.3 From 62b8b8e72c66da104e51d302ac41d1466b5d35ef Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Thu, 5 Mar 2015 15:15:28 -0500 Subject: fix extra logging --- training/mira/mira.py | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'training') diff --git a/training/mira/mira.py b/training/mira/mira.py index ccecb10e..32478c4f 100755 --- a/training/mira/mira.py +++ b/training/mira/mira.py @@ -22,23 +22,9 @@ dlog = None #scoring function using pycdec scoring def fast_score(hyps, refs, metric): - #scorer = cdec.score.Scorer(metric) - #logging.info('loaded {0} references for scoring with {1}'.format( - # len(refs), metric)) - #if metric=='BLEU': - # logging.warning('BLEU is ambiguous, assuming IBM_BLEU\n') - # metric = 'IBM_BLEU' - #elif metric=='COMBI': - # logging.warning('COMBI metric is no longer supported, switching to ' - # 'COMB:TER=-0.5;BLEU=0.5') - # metric = 'COMB:TER=-0.5;BLEU=0.5' - #stats = sum(scorer(r).evaluate(h) for h,r in itertools.izip(hyps,refs)) - #logging.info('Score={} ({})'.format(stats.score, stats.detail)) - #return stats.score cmd = ('{0} -r{1} -i {2} -m {3}').format(fast_score_binary, refs, hyps, metric) proc = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) o = proc.stdout.readline().strip() - print 'res: ', o return float(o) #create new parallel input file in output directory in sgml format -- cgit v1.2.3