From bd046a0cda7317f9dc2b68a33a0de5e21d7cbb16 Mon Sep 17 00:00:00 2001 From: "trevor.cohn" Date: Mon, 12 Jul 2010 18:23:01 +0000 Subject: Added slash limits git-svn-id: https://ws10smt.googlecode.com/svn/trunk@224 ec762483-ff6d-05da-a07a-a48fb63a330f --- gi/evaluation/evaluate_entropy.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'gi/evaluation/evaluate_entropy.py') diff --git a/gi/evaluation/evaluate_entropy.py b/gi/evaluation/evaluate_entropy.py index 88533544..55449c59 100644 --- a/gi/evaluation/evaluate_entropy.py +++ b/gi/evaluation/evaluate_entropy.py @@ -4,6 +4,10 @@ import sys, math, itertools ginfile = open(sys.argv[1]) pinfile = open(sys.argv[2]) +if len(sys.argv) >= 4: + slash_threshold = int(sys.argv[3]) +else: + slash_threshold = 99999 # evaluating: H(G | P) = sum_{g,p} p(g,p) log { p(p) / p(g,p) } # = sum_{g,p} c(g,p)/N { log c(p) - log N - log c(g,p) + log N } @@ -24,16 +28,17 @@ for gline, pline in itertools.izip(ginfile, pinfile): gtag = gpart.split(':',1)[1] ptag = ppart.split(':',1)[1] - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 + if gtag.count('/') <= slash_threshold: + joint_frequencies.setdefault((gtag, ptag), 0) + joint_frequencies[gtag,ptag] += 1 - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 + predict_frequencies.setdefault(ptag, 0) + predict_frequencies[ptag] += 1 - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 + gold_frequencies.setdefault(gtag, 0) + gold_frequencies[gtag] += 1 - N += 1 + N += 1 hg2p = 0 hp2g = 0 -- cgit v1.2.3