kenlm update including being nicer to NFS

3 files changed, 27 insertions, 4 deletions

diff --git a/klm/util/file_piece.cc b/klm/util/file_piece.cc
index 67681f7e..f447a70c 100644
--- a/klm/util/file_piece.cc
+++ b/klm/util/file_piece.cc
@@ -237,7 +237,12 @@ void FilePiece::MMapShift(off_t desired_begin) throw() {
 
   // Forcibly clear the existing mmap first.  
   data_.reset();
-  data_.reset(mmap(NULL, mapped_size, PROT_READ, MAP_PRIVATE, *file_, mapped_offset), mapped_size, scoped_memory::MMAP_ALLOCATED);
+  data_.reset(mmap(NULL, mapped_size, PROT_READ, MAP_SHARED
+  // Populate where available on linux
+#ifdef MAP_POPULATE
+        | MAP_POPULATE
+#endif
+        , *file_, mapped_offset), mapped_size, scoped_memory::MMAP_ALLOCATED);
   if (data_.get() == MAP_FAILED) {
     if (desired_begin) {
       if (((off_t)-1) == lseek(*file_, desired_begin, SEEK_SET)) UTIL_THROW(ErrnoException, "mmap failed even though it worked before.  lseek failed too, so using read isn't an option either.");
diff --git a/klm/util/mmap.cc b/klm/util/mmap.cc
index 456ce953..e7c0643b 100644
--- a/klm/util/mmap.cc
+++ b/klm/util/mmap.cc
@@ -15,8 +15,9 @@ namespace util {
 
 scoped_mmap::~scoped_mmap() {
   if (data_ != (void*)-1) {
-    if (munmap(data_, size_)) {
-      std::cerr << "munmap failed for " << size_ << " bytes." << std::endl;
+    // Thanks Denis Filimonov for pointing on NFS likes msync first.  
+    if (msync(data_, size_, MS_SYNC) || munmap(data_, size_)) {
+      std::cerr << "msync or mmap failed for " << size_ << " bytes." << std::endl;
       abort();
     }
   }
diff --git a/klm/util/string_piece.hh b/klm/util/string_piece.hh
index e5b16e38..5de053aa 100644
--- a/klm/util/string_piece.hh
+++ b/klm/util/string_piece.hh
@@ -60,6 +60,23 @@
 
 #ifdef HAVE_ICU
 #include <unicode/stringpiece.h>
+#include <unicode/uversion.h>
+
+// Old versions of ICU don't define operator== and operator!=.  
+#if (U_ICU_VERSION_MAJOR_NUM < 4) || ((U_ICU_VERSION_MAJOR_NUM == 4) && (U_ICU_VERSION_MINOR_NUM < 4))
+#warning You are using an old version of ICU.  Consider upgrading to ICU >= 4.6.  
+inline bool operator==(const StringPiece& x, const StringPiece& y) {
+  if (x.size() != y.size())
+    return false;
+
+  return std::memcmp(x.data(), y.data(), x.size()) == 0;
+}
+
+inline bool operator!=(const StringPiece& x, const StringPiece& y) {
+  return !(x == y);
+}
+#endif // old version of ICU
+
 U_NAMESPACE_BEGIN
 #else
 
@@ -209,7 +226,7 @@ inline bool operator!=(const StringPiece& x, const StringPiece& y) {
   return !(x == y);
 }
 
-#endif
+#endif // HAVE_ICU undefined
 
 inline bool operator<(const StringPiece& x, const StringPiece& y) {
   const int r = std::memcmp(x.data(), y.data(),