diff options
| author | armatthews <armatthe@cmu.edu> | 2014-10-13 14:59:23 -0400 |
|---|---|---|
| committer | armatthews <armatthe@cmu.edu> | 2014-10-13 14:59:23 -0400 |
| commit | 9a06ff1465eb3477ac3d1e92ab52e7eae40316a8 (patch) | |
| tree | 808c266a3f510d00f37cd19c3f1da91d8fc683f7 /klm/lm/builder/corpus_count.hh | |
| parent | e51da099233df0a384b04fe5908b30e44040d13e (diff) | |
| parent | d3e2ec203a5cf550320caa8023ac3dd103b0be7d (diff) | |
Merge branch 'master' of github.com:redpony/cdec
Diffstat (limited to 'klm/lm/builder/corpus_count.hh')
| -rw-r--r-- | klm/lm/builder/corpus_count.hh | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/klm/lm/builder/corpus_count.hh b/klm/lm/builder/corpus_count.hh index aa0ed8ed..da4ff9fc 100644 --- a/klm/lm/builder/corpus_count.hh +++ b/klm/lm/builder/corpus_count.hh @@ -1,6 +1,7 @@ -#ifndef LM_BUILDER_CORPUS_COUNT__ -#define LM_BUILDER_CORPUS_COUNT__ +#ifndef LM_BUILDER_CORPUS_COUNT_H +#define LM_BUILDER_CORPUS_COUNT_H +#include "lm/lm_exception.hh" #include "lm/word_index.hh" #include "util/scoped.hh" @@ -28,7 +29,7 @@ class CorpusCount { // token_count: out. // type_count aka vocabulary size. Initialize to an estimate. It is set to the exact value. - CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block); + CorpusCount(util::FilePiece &from, int vocab_write, uint64_t &token_count, WordIndex &type_count, std::size_t entries_per_block, WarningAction disallowed_symbol); void Run(const util::stream::ChainPosition &position); @@ -40,8 +41,10 @@ class CorpusCount { std::size_t dedupe_mem_size_; util::scoped_malloc dedupe_mem_; + + WarningAction disallowed_symbol_action_; }; } // namespace builder } // namespace lm -#endif // LM_BUILDER_CORPUS_COUNT__ +#endif // LM_BUILDER_CORPUS_COUNT_H |