diff options
| author | Patrick Simianer <p@simianer.de> | 2013-05-02 09:09:59 +0200 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2013-05-02 09:09:59 +0200 |
| commit | 9e50f0237413180fba11b500c9dce5c600e3c157 (patch) | |
| tree | 556fc31d231353c853a864afffddd43dc525549a /klm/lm/builder/corpus_count.hh | |
| parent | d18024a41cbc1b54db88d499571349a6234b6db8 (diff) | |
| parent | 14ed53426726202813a8e82d706b44266f015fe1 (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'klm/lm/builder/corpus_count.hh')
| -rw-r--r-- | klm/lm/builder/corpus_count.hh | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/klm/lm/builder/corpus_count.hh b/klm/lm/builder/corpus_count.hh index e255bad1..aa0ed8ed 100644 --- a/klm/lm/builder/corpus_count.hh +++ b/klm/lm/builder/corpus_count.hh @@ -23,6 +23,11 @@ class CorpusCount { // Memory usage will be DedupeMultipler(order) * block_size + total_chain_size + unknown vocab_hash_size static float DedupeMultiplier(std::size_t order); + // How much memory vocabulary will use based on estimated size of the vocab. + static std::size_t VocabUsage(std::size_t vocab_estimate); + + // token_count: out. + // type_count aka vocabulary size. Initialize to an estimate. It is set to the exact value. CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block); void Run(const util::stream::ChainPosition &position); |