1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
from __future__ import division
import math
from cdec.sa import isvar
MAXSCORE = 99
def EgivenF(ctx): # p(e|f) = c(e, f)/c(f)
if not ctx.online:
prob = ctx.paircount/ctx.fcount
else:
prob = (ctx.paircount + ctx.online.paircount) / (ctx.fcount + ctx.online.fcount)
return -math.log10(prob)
def CountEF(ctx): # c(e, f)
if not ctx.online:
count = 1 + ctx.paircount
else:
count = 1 + ctx.paircount + ctx.online.paircount
return math.log10(count)
def SampleCountF(ctx): # sample c(f)
if not ctx.online:
count = 1 + ctx.fsample_count
else:
count = 1 + ctx.fsample_count + ctx.online.fsample_count
return math.log10(count)
def EgivenFCoherent(ctx): # c(e, f) / sample c(f)
if not ctx.online:
prob = ctx.paircount/ctx.fsample_count
else:
prob = (ctx.paircount + ctx.online.paircount) / (ctx.fsample_count + ctx.online.fsample_count)
return -math.log10(prob) if prob > 0 else MAXSCORE
def CoherenceProb(ctx): # c(f) / sample c(f)
if not ctx.online:
prob = ctx.fcount/ctx.fsample_count
else:
prob = (ctx.fcount + ctx.online.fcount) / (ctx.fsample_count + ctx.online.fsample_count)
return -math.log10(prob)
# Not a feature, used for MaxLex
# bilex get_score for multiple instances
def get_lex_online(f, e, dir, bilex_list):
num = 0
denom = 0
for bilex in bilex_list:
if dir == 0:
denom += bilex.f.get(f, 0)
else:
denom += bilex.e.get(e, 0)
num += bilex.fe.get((f, e), 0)
if (not num) or (not denom):
return None
return num / denom
def MaxLexEgivenF(ttable):
def MaxLexEgivenF(ctx):
fwords = ctx.fphrase.words
fwords.append('NULL')
maxOffScore = 0.0
for e in ctx.ephrase.words:
if ctx.online:
maxScore = max(get_lex_online(f, e, 0, (ttable, ctx.online.bilex)) for f in fwords)
else:
maxScore = max(ttable.get_score(f, e, 0) for f in fwords)
maxOffScore += -math.log10(maxScore) if maxScore > 0 else MAXSCORE
return maxOffScore
return MaxLexEgivenF
def MaxLexFgivenE(ttable):
def MaxLexFgivenE(ctx):
ewords = ctx.ephrase.words
ewords.append('NULL')
maxOffScore = 0.0
for f in ctx.fphrase.words:
if ctx.online:
maxScore = max(get_lex_online(f, e, 1, (ttable, ctx.online.bilex)) for e in ewords)
else:
maxScore = max(ttable.get_score(f, e, 1) for e in ewords)
maxOffScore += -math.log10(maxScore) if maxScore > 0 else MAXSCORE
return maxOffScore
return MaxLexFgivenE
def IsSingletonF(ctx):
if not ctx.online:
count = ctx.fcount
else:
count = ctx.fcount + ctx.online.fcount
return math.fabs(count - 1) < 1e-6
def IsSingletonFE(ctx):
if not ctx.online:
count = ctx.paircount
else:
count = ctx.paircount + ctx.online.paircount
return (count == 1)
def IsNotSingletonF(ctx):
if not ctx.online:
count = ctx.fcount
else:
count = ctx.fcount + ctx.online.fcount
return (count > 1)
def IsNotSingletonFE(ctx):
if not ctx.online:
count = ctx.paircount
else:
count = ctx.paircount + ctx.online.paircount
return (ctx.paircount > 1)
def IsFEGreaterThanZero(ctx):
if not ctx.online:
count = ctx.paircount
else:
count = ctx.paircount + ctx.online.paircount
return (ctx.paircount > 0.01)
def IsSupportedOnline(ctx): # Occurs in online data?
if ctx.online:
return (ctx.online.paircount > 0.01)
else:
return False
|
