diff options
| author | trevor.cohn <trevor.cohn@ec762483-ff6d-05da-a07a-a48fb63a330f> | 2010-07-12 18:23:01 +0000 |
|---|---|---|
| committer | trevor.cohn <trevor.cohn@ec762483-ff6d-05da-a07a-a48fb63a330f> | 2010-07-12 18:23:01 +0000 |
| commit | bd046a0cda7317f9dc2b68a33a0de5e21d7cbb16 (patch) | |
| tree | 1c0359fcb0cd4b3ee90a1c6a9a301331d8646595 /gi/evaluation/evaluate_entropy.py | |
| parent | aaa0ad1ad9580d5eeb4fb790a937607d0ceefc4b (diff) | |
Added slash limits
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@224 ec762483-ff6d-05da-a07a-a48fb63a330f
Diffstat (limited to 'gi/evaluation/evaluate_entropy.py')
| -rw-r--r-- | gi/evaluation/evaluate_entropy.py | 19 |
1 files changed, 12 insertions, 7 deletions
diff --git a/gi/evaluation/evaluate_entropy.py b/gi/evaluation/evaluate_entropy.py index 88533544..55449c59 100644 --- a/gi/evaluation/evaluate_entropy.py +++ b/gi/evaluation/evaluate_entropy.py @@ -4,6 +4,10 @@ import sys, math, itertools ginfile = open(sys.argv[1]) pinfile = open(sys.argv[2]) +if len(sys.argv) >= 4: + slash_threshold = int(sys.argv[3]) +else: + slash_threshold = 99999 # evaluating: H(G | P) = sum_{g,p} p(g,p) log { p(p) / p(g,p) } # = sum_{g,p} c(g,p)/N { log c(p) - log N - log c(g,p) + log N } @@ -24,16 +28,17 @@ for gline, pline in itertools.izip(ginfile, pinfile): gtag = gpart.split(':',1)[1] ptag = ppart.split(':',1)[1] - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 + if gtag.count('/') <= slash_threshold: + joint_frequencies.setdefault((gtag, ptag), 0) + joint_frequencies[gtag,ptag] += 1 - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 + predict_frequencies.setdefault(ptag, 0) + predict_frequencies[ptag] += 1 - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 + gold_frequencies.setdefault(gtag, 0) + gold_frequencies[gtag] += 1 - N += 1 + N += 1 hg2p = 0 hp2g = 0 |