diff options
| author | Kenneth Heafield <github@kheafield.com> | 2013-04-24 10:12:41 +0100 |
|---|---|---|
| committer | Kenneth Heafield <github@kheafield.com> | 2013-04-24 10:12:41 +0100 |
| commit | 5aee54869aa19cfe9be965e67a472e94449d16da (patch) | |
| tree | 8f2111e58c4cc4108b8daabb8c38f16adcaff2f0 /klm/lm/builder/corpus_count.hh | |
| parent | 9957c8a43354fe0a81b83659de965d6d0934adf8 (diff) | |
KenLM 0831569c3137536165b107c6841603c725dfa2b1
Diffstat (limited to 'klm/lm/builder/corpus_count.hh')
| -rw-r--r-- | klm/lm/builder/corpus_count.hh | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/klm/lm/builder/corpus_count.hh b/klm/lm/builder/corpus_count.hh index e255bad1..aa0ed8ed 100644 --- a/klm/lm/builder/corpus_count.hh +++ b/klm/lm/builder/corpus_count.hh @@ -23,6 +23,11 @@ class CorpusCount { // Memory usage will be DedupeMultipler(order) * block_size + total_chain_size + unknown vocab_hash_size static float DedupeMultiplier(std::size_t order); + // How much memory vocabulary will use based on estimated size of the vocab. + static std::size_t VocabUsage(std::size_t vocab_estimate); + + // token_count: out. + // type_count aka vocabulary size. Initialize to an estimate. It is set to the exact value. CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block); void Run(const util::stream::ChainPosition &position); |