From 606e3e38b8a830dbbe65963ebf6c5ce7866b7800 Mon Sep 17 00:00:00 2001
From: Chris Dyer <redpony@gmail.com>
Date: Tue, 18 Mar 2014 01:41:17 -0400
Subject: star function

---
 utils/logval.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'utils/logval.h')

diff --git a/utils/logval.h b/utils/logval.h
index ec1f6acd..7f1e1024 100644
--- a/utils/logval.h
+++ b/utils/logval.h
@@ -11,6 +11,7 @@
 #include <cassert>
 #include "semiring.h"
 #include "show.h"
+#include "star.h"
 
 //TODO: template for supporting negation or not - most uses are for nonnegative "probs" only; probably some 10-20% speedup available
 template <class T>
@@ -242,4 +243,15 @@ bool operator>=(const LogVal<T>& lhs, const LogVal<T>& rhs) {
 template <class T>
 std::size_t hash_value(const LogVal<T>& x) { return x.hash_impl(); }
 
+template <class T>
+LogVal<T> star(LogVal<T> x) {
+  if (x.is_0()) return x;
+  if (x.v_ >= 0) {
+    x.v_ = std::numeric_limits<T>::infinity();
+  } else {
+    x.v_ = -log1p(-x.as_float());
+  }
+  return x;
+}
+
 #endif
-- 
cgit v1.2.3


From 2a9ee1febae6a63173f74ae24e2bfe439e409525 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Tue, 18 Mar 2014 02:05:25 -0400
Subject: chris edits

---
 corpus/support/tokenizer.pl | 4 ++++
 utils/exp_semiring.h        | 2 +-
 utils/logval.h              | 2 +-
 utils/star.h                | 4 ++--
 4 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'utils/logval.h')

diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index 7771201f..f57bc87a 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -240,6 +240,10 @@ sub proc_token {
 	return $token;
     }
 
+    if($token =~ /^\d+(.\d+)+(亿|百万|万|千)?$/){
+	return $token;
+    }
+
     ## 1,234,345.34
     if($token =~ /^\d+(\.\d{3})*,\d+$/){
 	## number
diff --git a/utils/exp_semiring.h b/utils/exp_semiring.h
index 7572ccf5..26a22071 100644
--- a/utils/exp_semiring.h
+++ b/utils/exp_semiring.h
@@ -56,7 +56,7 @@ const PRPair<P,R> operator*(const PRPair<P,R>& a, const PRPair<P,R>& b) {
 }
 
 template <class P, class R>
-const PRPair<P,R> star(const PRPair<P,R>& x) {
+inline const PRPair<P,R> star(const PRPair<P,R>& x) {
   const P pstar = star(x.p);
   return PRPair<P,R>(pstar, pstar * x.r * pstar);
 }
diff --git a/utils/logval.h b/utils/logval.h
index 7f1e1024..0c9ee982 100644
--- a/utils/logval.h
+++ b/utils/logval.h
@@ -244,7 +244,7 @@ template <class T>
 std::size_t hash_value(const LogVal<T>& x) { return x.hash_impl(); }
 
 template <class T>
-LogVal<T> star(LogVal<T> x) {
+inline LogVal<T> star(LogVal<T> x) {
   if (x.is_0()) return x;
   if (x.v_ >= 0) {
     x.v_ = std::numeric_limits<T>::infinity();
diff --git a/utils/star.h b/utils/star.h
index e7358ffa..21977dc9 100644
--- a/utils/star.h
+++ b/utils/star.h
@@ -4,14 +4,14 @@
 // star(x) computes the infinite sum x^0 + x^1 + x^2 + ...
 
 template <typename T>
-T star(const T& x) {
+inline T star(const T& x) {
   if (!x) return T();
   if (x > T(1)) return std::numeric_limits<T>::infinity();
   if (x < -T(1)) return -std::numeric_limits<T>::infinity();
   return T(1) / (T(1) - x);
 }
 
-bool star(bool x) {
+inline bool star(bool x) {
   return x;
 }
 
-- 
cgit v1.2.3