summaryrefslogtreecommitdiff
path: root/klm/util
diff options
context:
space:
mode:
authorKenneth Heafield <kheafiel@cluster12.lti.ece.cmu.local>2011-05-20 16:19:04 -0400
committerKenneth Heafield <kheafiel@cluster12.lti.ece.cmu.local>2011-05-20 16:19:04 -0400
commitbc95fedbaa083d557840db6ac2cbf14e2a3eccce (patch)
tree00717d119814232577f7345ddbf2f224d58911e8 /klm/util
parent066dcbd87b6a59d4ef032001763eebf620fe4702 (diff)
kenlm update including being nicer to NFS
Diffstat (limited to 'klm/util')
-rw-r--r--klm/util/file_piece.cc7
-rw-r--r--klm/util/mmap.cc5
-rw-r--r--klm/util/string_piece.hh19
3 files changed, 27 insertions, 4 deletions
diff --git a/klm/util/file_piece.cc b/klm/util/file_piece.cc
index 67681f7e..f447a70c 100644
--- a/klm/util/file_piece.cc
+++ b/klm/util/file_piece.cc
@@ -237,7 +237,12 @@ void FilePiece::MMapShift(off_t desired_begin) throw() {
// Forcibly clear the existing mmap first.
data_.reset();
- data_.reset(mmap(NULL, mapped_size, PROT_READ, MAP_PRIVATE, *file_, mapped_offset), mapped_size, scoped_memory::MMAP_ALLOCATED);
+ data_.reset(mmap(NULL, mapped_size, PROT_READ, MAP_SHARED
+ // Populate where available on linux
+#ifdef MAP_POPULATE
+ | MAP_POPULATE
+#endif
+ , *file_, mapped_offset), mapped_size, scoped_memory::MMAP_ALLOCATED);
if (data_.get() == MAP_FAILED) {
if (desired_begin) {
if (((off_t)-1) == lseek(*file_, desired_begin, SEEK_SET)) UTIL_THROW(ErrnoException, "mmap failed even though it worked before. lseek failed too, so using read isn't an option either.");
diff --git a/klm/util/mmap.cc b/klm/util/mmap.cc
index 456ce953..e7c0643b 100644
--- a/klm/util/mmap.cc
+++ b/klm/util/mmap.cc
@@ -15,8 +15,9 @@ namespace util {
scoped_mmap::~scoped_mmap() {
if (data_ != (void*)-1) {
- if (munmap(data_, size_)) {
- std::cerr << "munmap failed for " << size_ << " bytes." << std::endl;
+ // Thanks Denis Filimonov for pointing on NFS likes msync first.
+ if (msync(data_, size_, MS_SYNC) || munmap(data_, size_)) {
+ std::cerr << "msync or mmap failed for " << size_ << " bytes." << std::endl;
abort();
}
}
diff --git a/klm/util/string_piece.hh b/klm/util/string_piece.hh
index e5b16e38..5de053aa 100644
--- a/klm/util/string_piece.hh
+++ b/klm/util/string_piece.hh
@@ -60,6 +60,23 @@
#ifdef HAVE_ICU
#include <unicode/stringpiece.h>
+#include <unicode/uversion.h>
+
+// Old versions of ICU don't define operator== and operator!=.
+#if (U_ICU_VERSION_MAJOR_NUM < 4) || ((U_ICU_VERSION_MAJOR_NUM == 4) && (U_ICU_VERSION_MINOR_NUM < 4))
+#warning You are using an old version of ICU. Consider upgrading to ICU >= 4.6.
+inline bool operator==(const StringPiece& x, const StringPiece& y) {
+ if (x.size() != y.size())
+ return false;
+
+ return std::memcmp(x.data(), y.data(), x.size()) == 0;
+}
+
+inline bool operator!=(const StringPiece& x, const StringPiece& y) {
+ return !(x == y);
+}
+#endif // old version of ICU
+
U_NAMESPACE_BEGIN
#else
@@ -209,7 +226,7 @@ inline bool operator!=(const StringPiece& x, const StringPiece& y) {
return !(x == y);
}
-#endif
+#endif // HAVE_ICU undefined
inline bool operator<(const StringPiece& x, const StringPiece& y) {
const int r = std::memcmp(x.data(), y.data(),