summaryrefslogtreecommitdiff
path: root/gi/evaluation/entropy.py
diff options
context:
space:
mode:
authorKenneth Heafield <github@kheafield.com>2012-10-22 12:07:20 +0100
committerKenneth Heafield <github@kheafield.com>2012-10-22 12:07:20 +0100
commit5f98fe5c4f2a2090eeb9d30c030305a70a8347d1 (patch)
tree9b6002f850e6dea1e3400c6b19bb31a9cdf3067f /gi/evaluation/entropy.py
parentcf9994131993b40be62e90e213b1e11e6b550143 (diff)
parent21825a09d97c2e0afd20512f306fb25fed55e529 (diff)
Merge remote branch 'upstream/master'
Conflicts: Jamroot bjam decoder/Jamfile decoder/cdec.cc dpmert/Jamfile jam-files/sanity.jam klm/lm/Jamfile klm/util/Jamfile mira/Jamfile
Diffstat (limited to 'gi/evaluation/entropy.py')
-rw-r--r--gi/evaluation/entropy.py38
1 files changed, 0 insertions, 38 deletions
diff --git a/gi/evaluation/entropy.py b/gi/evaluation/entropy.py
deleted file mode 100644
index ec1ef502..00000000
--- a/gi/evaluation/entropy.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-import sys, math, itertools, getopt
-
-def usage():
- print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] input file'
- sys.exit(0)
-
-optlist, args = getopt.getopt(sys.argv[1:], 'hs:')
-slash_threshold = None
-for opt, arg in optlist:
- if opt == '-s':
- slash_threshold = int(arg)
- else:
- usage()
-if len(args) != 1:
- usage()
-
-infile = open(args[0])
-N = 0
-frequencies = {}
-
-for line in infile:
-
- for part in line.split('||| ')[1].split():
- tag = part.split(':',1)[1]
-
- if slash_threshold == None or tag.count('/') + tag.count('\\') <= slash_threshold:
- frequencies.setdefault(tag, 0)
- frequencies[tag] += 1
- N += 1
-
-h = 0
-for tag, c in frequencies.items():
- h -= c * (math.log(c, 2) - math.log(N, 2))
-h /= N
-
-print 'entropy', h