diff options
| author | Patrick Simianer <p@simianer.de> | 2013-05-02 09:09:59 +0200 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2013-05-02 09:09:59 +0200 |
| commit | 0ce66778da6079506896739e9d97dc7dff83cd72 (patch) | |
| tree | f435457bb23dab0c566c9896f9d38cece9d15885 /klm/lm/builder/corpus_count.hh | |
| parent | b6754386f1109b960b05cdf2eabbc97bdd38e8df (diff) | |
| parent | b7ea2615bc9bb69031ff714ddce1539c9f1bda2d (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'klm/lm/builder/corpus_count.hh')
| -rw-r--r-- | klm/lm/builder/corpus_count.hh | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/klm/lm/builder/corpus_count.hh b/klm/lm/builder/corpus_count.hh index e255bad1..aa0ed8ed 100644 --- a/klm/lm/builder/corpus_count.hh +++ b/klm/lm/builder/corpus_count.hh @@ -23,6 +23,11 @@ class CorpusCount { // Memory usage will be DedupeMultipler(order) * block_size + total_chain_size + unknown vocab_hash_size static float DedupeMultiplier(std::size_t order); + // How much memory vocabulary will use based on estimated size of the vocab. + static std::size_t VocabUsage(std::size_t vocab_estimate); + + // token_count: out. + // type_count aka vocabulary size. Initialize to an estimate. It is set to the exact value. CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block); void Run(const util::stream::ChainPosition &position); |