summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xcorpus/support/tokenizer.pl4
-rw-r--r--utils/exp_semiring.h2
-rw-r--r--utils/logval.h2
-rw-r--r--utils/star.h4
4 files changed, 8 insertions, 4 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index 7771201f..f57bc87a 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -240,6 +240,10 @@ sub proc_token {
return $token;
}
+ if($token =~ /^\d+(.\d+)+(亿|百万|万|千)?$/){
+ return $token;
+ }
+
## 1,234,345.34
if($token =~ /^\d+(\.\d{3})*,\d+$/){
## number
diff --git a/utils/exp_semiring.h b/utils/exp_semiring.h
index 7572ccf5..26a22071 100644
--- a/utils/exp_semiring.h
+++ b/utils/exp_semiring.h
@@ -56,7 +56,7 @@ const PRPair<P,R> operator*(const PRPair<P,R>& a, const PRPair<P,R>& b) {
}
template <class P, class R>
-const PRPair<P,R> star(const PRPair<P,R>& x) {
+inline const PRPair<P,R> star(const PRPair<P,R>& x) {
const P pstar = star(x.p);
return PRPair<P,R>(pstar, pstar * x.r * pstar);
}
diff --git a/utils/logval.h b/utils/logval.h
index 7f1e1024..0c9ee982 100644
--- a/utils/logval.h
+++ b/utils/logval.h
@@ -244,7 +244,7 @@ template <class T>
std::size_t hash_value(const LogVal<T>& x) { return x.hash_impl(); }
template <class T>
-LogVal<T> star(LogVal<T> x) {
+inline LogVal<T> star(LogVal<T> x) {
if (x.is_0()) return x;
if (x.v_ >= 0) {
x.v_ = std::numeric_limits<T>::infinity();
diff --git a/utils/star.h b/utils/star.h
index e7358ffa..21977dc9 100644
--- a/utils/star.h
+++ b/utils/star.h
@@ -4,14 +4,14 @@
// star(x) computes the infinite sum x^0 + x^1 + x^2 + ...
template <typename T>
-T star(const T& x) {
+inline T star(const T& x) {
if (!x) return T();
if (x > T(1)) return std::numeric_limits<T>::infinity();
if (x < -T(1)) return -std::numeric_limits<T>::infinity();
return T(1) / (T(1) - x);
}
-bool star(bool x) {
+inline bool star(bool x) {
return x;
}