fix kenlm's use of compression libraries

10 files changed, 30 insertions, 26 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 12e65119..89275948 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 2.8)
 
-add_definitions(-DKENLM_MAX_ORDER=6)
+add_definitions(-DKENLM_MAX_ORDER=6 -DHAVE_CONFIG_H)
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 set(CMAKE_CXX_FLAGS "-Wall -std=c++11 -O3")
 set(METEOR_JAR "" CACHE FILEPATH "Path to meteor.jar")
@@ -17,6 +17,11 @@ find_package(BZip2)
 if(BZIP2_FOUND)
   set(HAVE_BZLIB 1)
 endif(BZIP2_FOUND)
+find_package(LibLZMA)
+if(LIBLZMA_FOUND)
+  set(HAVE_XZLIB 1)
+endif(LIBLZMA_FOUND)
+
 find_package(LibDL REQUIRED)
 
 # for pycdec
diff --git a/config.h.cmake b/config.h.cmake
index b01a1e6d..6f270a60 100644
--- a/config.h.cmake
+++ b/config.h.cmake
@@ -1,8 +1,9 @@
 #ifndef CONFIG_H
 #define CONFIG_H
- 
+
 #cmakedefine METEOR_JAR "@METEOR_JAR@"
 #cmakedefine HAVE_ZLIB @HAVE_ZLIB@
 #cmakedefine HAVE_BZLIB @HAVE_BZLIB@
+#cmakedefine HAVE_XZLIB @HAVE_XZLIB@
 
 #endif // CONFIG_H
diff --git a/decoder/CMakeLists.txt b/decoder/CMakeLists.txt
index b49d6477..ad800588 100644
--- a/decoder/CMakeLists.txt
+++ b/decoder/CMakeLists.txt
@@ -151,7 +151,7 @@ add_library(libcdec STATIC ${libcdec_SRCS})
 
 set(cdec_SRCS cdec.cc)
 add_executable(cdec ${cdec_SRCS})
-target_link_libraries(cdec libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${LIBDL_LIBRARIES})
+target_link_libraries(cdec libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 set(TEST_SRCS
   grammar_test.cc
@@ -169,7 +169,7 @@ foreach(testSrc ${TEST_SRCS})
   add_executable(${testName} ${testSrc})
 
   #link to Boost libraries AND your targets and dependencies
-  target_link_libraries(${testName} libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES})
+  target_link_libraries(${testName} libcdec mteval utils ksearch klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
 
   #I like to move testing binaries into a testBin directory
   set_target_properties(${testName} PROPERTIES 
@@ -181,5 +181,3 @@ foreach(testSrc ${TEST_SRCS})
      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 endforeach(testSrc)
 
-
-
diff --git a/klm/lm/CMakeLists.txt b/klm/lm/CMakeLists.txt
index fcd288c1..ceb814c6 100644
--- a/klm/lm/CMakeLists.txt
+++ b/klm/lm/CMakeLists.txt
@@ -7,11 +7,11 @@ add_executable(build_binary ${build_binary_SRCS})
 set(ngram_query_SRCS query_main.cc)
 add_executable(ngram_query ${ngram_query_SRCS})
 if (HAVE_RT)
-  target_link_libraries(build_binary klm klm_util klm_util_double z ${RT_LIBRARIES})
-  target_link_libraries(ngram_query klm klm_util klm_util_double z ${RT_LIBRARIES})
+  target_link_libraries(build_binary klm klm_util klm_util_double ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${RT_LIBRARIES})
+  target_link_libraries(ngram_query klm klm_util klm_util_double ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${RT_LIBRARIES})
 else (HAVE_RT)
-  target_link_libraries(build_binary klm klm_util klm_util_double z)
-  target_link_libraries(ngram_query klm klm_util klm_util_double z)
+  target_link_libraries(build_binary klm klm_util klm_util_double ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
+  target_link_libraries(ngram_query klm klm_util klm_util_double ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
 endif (HAVE_RT)
 
 
diff --git a/klm/lm/builder/CMakeLists.txt b/klm/lm/builder/CMakeLists.txt
index 5fb1b2d2..e52875cb 100644
--- a/klm/lm/builder/CMakeLists.txt
+++ b/klm/lm/builder/CMakeLists.txt
@@ -24,8 +24,8 @@ set(lmplz_SRCS
     sort.hh)
 
 add_executable(lmplz ${lmplz_SRCS})
-target_link_libraries(lmplz klm klm_util_double klm_util_stream klm_util ${Boost_LIBRARIES})
+target_link_libraries(lmplz klm klm_util_double klm_util_stream klm_util ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
 
 set(dump_counts_SRCS print.cc dump_counts_main.cc)
 add_executable(dump_counts ${dump_counts_SRCS})
-target_link_libraries(dump_counts klm klm_util_double klm_util_stream klm_util ${Boost_LIBRARIES})
+target_link_libraries(dump_counts klm klm_util_double klm_util_stream klm_util ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
diff --git a/klm/util/CMakeLists.txt b/klm/util/CMakeLists.txt
index bef1db78..23eaffe9 100644
--- a/klm/util/CMakeLists.txt
+++ b/klm/util/CMakeLists.txt
@@ -3,7 +3,7 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/..)
 set(cat_compressed_SRCS cat_compressed_main.cc)
 
 add_executable(cat_compressed ${cat_compressed_SRCS})
-target_link_libraries(cat_compressed klm_util)
+target_link_libraries(cat_compressed klm_util ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES})
 
 set(klm_util_STAT_SRCS
     bit_packing.cc
diff --git a/training/crf/CMakeLists.txt b/training/crf/CMakeLists.txt
index 85fab7a0..59a5c1d9 100644
--- a/training/crf/CMakeLists.txt
+++ b/training/crf/CMakeLists.txt
@@ -9,7 +9,7 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder)
 
 set(mpi_batch_optimize_SRCS mpi_batch_optimize.cc cllh_observer.cc cllh_observer.h)
 add_executable(mpi_batch_optimize ${mpi_batch_optimize_SRCS})
-target_link_libraries(mpi_batch_optimize training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_batch_optimize training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 ########### next target ###############
 
@@ -17,7 +17,7 @@ set(mpi_adagrad_optimize_SRCS mpi_adagrad_optimize.cc cllh_observer.cc cllh_obse
 
 add_executable(mpi_adagrad_optimize ${mpi_adagrad_optimize_SRCS})
 
-target_link_libraries(mpi_adagrad_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_adagrad_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 
 ########### next target ###############
@@ -26,7 +26,7 @@ set(mpi_compute_cllh_SRCS mpi_compute_cllh.cc cllh_observer.cc cllh_observer.h)
 
 add_executable(mpi_compute_cllh ${mpi_compute_cllh_SRCS})
 
-target_link_libraries(mpi_compute_cllh ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_compute_cllh ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 
 ########### next target ###############
@@ -35,7 +35,7 @@ set(mpi_extract_features_SRCS mpi_extract_features.cc)
 
 add_executable(mpi_extract_features ${mpi_extract_features_SRCS})
 
-target_link_libraries(mpi_extract_features ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_extract_features ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 
 ########### next target ###############
@@ -44,7 +44,7 @@ set(mpi_extract_reachable_SRCS mpi_extract_reachable.cc)
 
 add_executable(mpi_extract_reachable ${mpi_extract_reachable_SRCS})
 
-target_link_libraries(mpi_extract_reachable ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_extract_reachable ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 ########### next target ###############
 
@@ -52,7 +52,7 @@ set(mpi_flex_optimize_SRCS mpi_flex_optimize.cc)
 
 add_executable(mpi_flex_optimize ${mpi_flex_optimize_SRCS})
 
-target_link_libraries(mpi_flex_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_flex_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 ########### next target ###############
 
@@ -60,7 +60,7 @@ set(mpi_online_optimize_SRCS mpi_online_optimize.cc)
 
 add_executable(mpi_online_optimize ${mpi_online_optimize_SRCS})
 
-target_link_libraries(mpi_online_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_online_optimize ${KDE4_KDECORE_LIBS} training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 ########### next target ###############
 
@@ -68,5 +68,5 @@ set(mpi_baum_welch_SRCS mpi_baum_welch.cc)
 
 add_executable(mpi_baum_welch ${mpi_baum_welch_SRCS})
 
-target_link_libraries(mpi_baum_welch ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(mpi_baum_welch ${KDE4_KDECORE_LIBS} libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
diff --git a/training/dtrain/CMakeLists.txt b/training/dtrain/CMakeLists.txt
index 72e80b6b..8423675f 100644
--- a/training/dtrain/CMakeLists.txt
+++ b/training/dtrain/CMakeLists.txt
@@ -12,4 +12,4 @@ set(dtrain_SRCS
     pairsampling.h
     score.h)
 add_executable(dtrain ${dtrain_SRCS})
-target_link_libraries(dtrain libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(dtrain libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
diff --git a/training/latent_svm/CMakeLists.txt b/training/latent_svm/CMakeLists.txt
index 332b8d42..4c8c0b18 100644
--- a/training/latent_svm/CMakeLists.txt
+++ b/training/latent_svm/CMakeLists.txt
@@ -5,4 +5,4 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder)
 
 set(latent_svm_SRCS latent_svm.cc)
 add_executable(latent_svm ${latent_svm_SRCS})
-target_link_libraries(latent_svm libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(latent_svm libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
diff --git a/training/mira/CMakeLists.txt b/training/mira/CMakeLists.txt
index bba9ef5f..55ca03e2 100644
--- a/training/mira/CMakeLists.txt
+++ b/training/mira/CMakeLists.txt
@@ -5,12 +5,12 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../../decoder)
 
 set(kbest_mira_SRCS kbest_mira.cc)
 add_executable(kbest_mira ${kbest_mira_SRCS})
-target_link_libraries(kbest_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(kbest_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 set(kbest_cut_mira_SRCS kbest_cut_mira.cc)
 add_executable(kbest_cut_mira ${kbest_cut_mira_SRCS})
-target_link_libraries(kbest_cut_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(kbest_cut_mira libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})
 
 set(ada_opt_sm_SRCS ada_opt_sm.cc)
 add_executable(ada_opt_sm ${ada_opt_sm_SRCS})
-target_link_libraries(ada_opt_sm training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} z ${LIBDL_LIBRARIES})
+target_link_libraries(ada_opt_sm training_utils libcdec ksearch mteval utils klm klm_util klm_util_double ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${LIBLZMA_LIBRARIES} ${LIBDL_LIBRARIES})