#ifndef LM_CONFIG__ #define LM_CONFIG__ #include "lm/lm_exception.hh" #include "util/mmap.hh" #include <iosfwd> #include <string> #include <vector> /* Configuration for ngram model. Separate header to reduce pollution. */ namespace lm { class EnumerateVocab; namespace ngram { struct Config { // EFFECTIVE FOR BOTH ARPA AND BINARY READS // (default true) print progress bar to messages bool show_progress; // Where to log messages including the progress bar. Set to NULL for // silence. std::ostream *messages; std::ostream *ProgressMessages() const { return show_progress ? messages : 0; } // This will be called with every string in the vocabulary. See // enumerate_vocab.hh for more detail. Config does not take ownership; you // are still responsible for deleting it (or stack allocating). EnumerateVocab *enumerate_vocab; // ONLY EFFECTIVE WHEN READING ARPA // What to do when <unk> isn't in the provided model. WarningAction unknown_missing; // What to do when <s> or </s> is missing from the model. // If THROW_UP, the exception will be of type util::SpecialWordMissingException. WarningAction sentence_marker_missing; // What to do with a positive log probability. For COMPLAIN and SILENT, map // to 0. WarningAction positive_log_probability; // The probability to substitute for <unk> if it's missing from the model. // No effect if the model has <unk> or unknown_missing == THROW_UP. float unknown_missing_logprob; // Size multiplier for probing hash table. Must be > 1. Space is linear in // this. Time is probing_multiplier / (probing_multiplier - 1). No effect // for sorted variant. // If you find yourself setting this to a low number, consider using the // TrieModel which has lower memory consumption. float probing_multiplier; // Amount of memory to use for building. The actual memory usage will be // higher since this just sets sort buffer size. Only applies to trie // models. std::size_t building_memory; // Template for temporary directory appropriate for passing to mkdtemp. // The characters XXXXXX are appended before passing to mkdtemp. Only // applies to trie. If NULL, defaults to write_mmap. If that's NULL, // defaults to input file name. const char *temporary_directory_prefix; // Level of complaining to do when loading from ARPA instead of binary format. enum ARPALoadComplain {ALL, EXPENSIVE, NONE}; ARPALoadComplain arpa_complain; // While loading an ARPA file, also write out this binary format file. Set // to NULL to disable. const char *write_mmap; enum WriteMethod { WRITE_MMAP, // Map the file directly. WRITE_AFTER // Write after we're done. }; WriteMethod write_method; // Include the vocab in the binary file? Only effective if write_mmap != NULL. bool include_vocab; // Left rest options. Only used when the model includes rest costs. enum RestFunction { REST_MAX, // Maximum of any score to the left REST_LOWER, // Use lower-order files given below. }; RestFunction rest_function; // Only used for REST_LOWER. std::vector<std::string> rest_lower_files; // Quantization options. Only effective for QuantTrieModel. One value is // reserved for each of prob and backoff, so 2^bits - 1 buckets will be used // to quantize (and one of the remaining backoffs will be 0). uint8_t prob_bits, backoff_bits; // Bhiksha compression (simple form). Only works with trie. uint8_t pointer_bhiksha_bits; // ONLY EFFECTIVE WHEN READING BINARY // How to get the giant array into memory: lazy mmap, populate, read etc. // See util/mmap.hh for details of MapMethod. util::LoadMethod load_method; // Set defaults. Config(); }; } /* namespace ngram */ } /* namespace lm */ #endif // LM_CONFIG__