summaryrefslogtreecommitdiff
path: root/klm/lm/read_arpa.cc
diff options
context:
space:
mode:
authorKenneth Heafield <github@kheafield.com>2012-08-16 17:02:56 -0400
committerKenneth Heafield <github@kheafield.com>2012-08-16 17:07:51 -0400
commitdc078281c2fd83db9fc4860e8b24979ab2a1e407 (patch)
treec78afb93ccfac30614f7fa44374a44aa49288b0f /klm/lm/read_arpa.cc
parent9c9213239263e8e8de2f154068cc3ad44e0c2100 (diff)
KenLM update. Remove a couple of segfaults for weird input. Other oddball stuff.
Diffstat (limited to 'klm/lm/read_arpa.cc')
-rw-r--r--klm/lm/read_arpa.cc16
1 files changed, 15 insertions, 1 deletions
diff --git a/klm/lm/read_arpa.cc b/klm/lm/read_arpa.cc
index 2d9a337d..70727e4c 100644
--- a/klm/lm/read_arpa.cc
+++ b/klm/lm/read_arpa.cc
@@ -7,9 +7,14 @@
#include <vector>
#include <ctype.h>
+#include <math.h>
#include <string.h>
#include <stdint.h>
+#ifdef WIN32
+#include <float.h>
+#endif
+
namespace lm {
// 1 for '\t', '\n', and ' '. This is stricter than isspace.
@@ -93,7 +98,16 @@ void ReadBackoff(util::FilePiece &in, float &backoff) {
case '\t':
backoff = in.ReadFloat();
if (backoff == ngram::kExtensionBackoff) backoff = ngram::kNoExtensionBackoff;
- if ((in.get() != '\n')) UTIL_THROW(FormatLoadException, "Expected newline after backoff");
+ {
+#ifdef WIN32
+ int float_class = _fpclass(backoff);
+ UTIL_THROW_IF(float_class == _FPCLASS_SNAN || float_class == _FPCLASS_QNAN || float_class == _FPCLASS_NINF || float_class == _FPCLASS_PINF, FormatLoadException, "Bad backoff " << backoff);
+#else
+ int float_class = fpclassify(backoff);
+ UTIL_THROW_IF(float_class == FP_NAN || float_class == FP_INFINITE, FormatLoadException, "Bad backoff " << backoff);
+#endif
+ }
+ UTIL_THROW_IF(in.get() != '\n', FormatLoadException, "Expected newline after backoff");
break;
case '\n':
backoff = ngram::kNoExtensionBackoff;