summaryrefslogtreecommitdiff
path: root/klm/util/stream
diff options
context:
space:
mode:
authorKenneth Heafield <github@kheafield.com>2013-01-22 21:37:49 +0000
committerKenneth Heafield <github@kheafield.com>2013-01-22 21:37:49 +0000
commitb35a7f3a96ff8ae42e15922dd6949bf9f5d15501 (patch)
tree26edbe78931ffc50864a899c087d851005fe560b /klm/util/stream
parent51a412aa7f5f50035cf28a274a70508c839f3d40 (diff)
KenLM 58da338b
Diffstat (limited to 'klm/util/stream')
-rw-r--r--klm/util/stream/io.cc8
-rw-r--r--klm/util/stream/sort.hh12
2 files changed, 13 insertions, 7 deletions
diff --git a/klm/util/stream/io.cc b/klm/util/stream/io.cc
index c7ad2980..0459f706 100644
--- a/klm/util/stream/io.cc
+++ b/klm/util/stream/io.cc
@@ -29,15 +29,17 @@ void Read::Run(const ChainPosition &position) {
void PRead::Run(const ChainPosition &position) {
scoped_fd owner;
if (own_) owner.reset(file_);
- uint64_t size = SizeOrThrow(file_);
+ const uint64_t size = SizeOrThrow(file_);
UTIL_THROW_IF(size % static_cast<uint64_t>(position.GetChain().EntrySize()), ReadSizeException, "File size " << file_ << " size is " << size << " not a multiple of " << position.GetChain().EntrySize());
- std::size_t block_size = position.GetChain().BlockSize();
+ const std::size_t block_size = position.GetChain().BlockSize();
+ const uint64_t block_size64 = static_cast<uint64_t>(block_size);
Link link(position);
uint64_t offset = 0;
- for (; offset + block_size < size; offset += block_size, ++link) {
+ for (; offset + block_size64 < size; offset += block_size64, ++link) {
PReadOrThrow(file_, link->Get(), block_size, offset);
link->SetValidSize(block_size);
}
+ // size - offset is <= block_size, so it casts to 32-bit fine.
if (size - offset) {
PReadOrThrow(file_, link->Get(), size - offset, offset);
link->SetValidSize(size - offset);
diff --git a/klm/util/stream/sort.hh b/klm/util/stream/sort.hh
index a86f160f..16aa6a03 100644
--- a/klm/util/stream/sort.hh
+++ b/klm/util/stream/sort.hh
@@ -365,10 +365,14 @@ template <class Compare> class BlockSorter {
// Record the size of each block in a separate file.
offsets_->Append(link->ValidSize());
void *end = static_cast<uint8_t*>(link->Get()) + link->ValidSize();
- std::sort(
- SizedIt(link->Get(), entry_size),
- SizedIt(end, entry_size),
- compare_);
+#if defined(_WIN32) || defined(_WIN64)
+ std::stable_sort
+#else
+ std::sort
+#endif
+ (SizedIt(link->Get(), entry_size),
+ SizedIt(end, entry_size),
+ compare_);
}
offsets_->FinishedAppending();
}