From e26434979adc33bd949566ba7bf02dff64e80a3e Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 2 Oct 2012 00:19:43 -0400 Subject: cdec cleanup, remove bayesian stuff, parsing stuff --- gi/pyp-topics/scripts/extract_leaves.py | 49 --------------------------------- 1 file changed, 49 deletions(-) delete mode 100755 gi/pyp-topics/scripts/extract_leaves.py (limited to 'gi/pyp-topics/scripts/extract_leaves.py') diff --git a/gi/pyp-topics/scripts/extract_leaves.py b/gi/pyp-topics/scripts/extract_leaves.py deleted file mode 100755 index 14783b36..00000000 --- a/gi/pyp-topics/scripts/extract_leaves.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python - -import nltk -import nltk.probability -import sys -import getopt - -lexicalise=False -rm_traces=False -cutoff=100 -length_cutoff=10000 -try: - opts, args = getopt.getopt(sys.argv[1:], "hs:c:l", ["help", "lexicalise", "cutoff","sentence-length","remove-traces"]) -except getopt.GetoptError: - print "Usage: extract_leaves.py [-lsc]" - sys.exit(2) -for opt, arg in opts: - if opt in ("-h", "--help"): - print "Usage: extract_leaves.py [-lsc]" - sys.exit() - elif opt in ("-l", "--lexicalise"): - lexicalise = True - elif opt in ("-c", "--cutoff"): - cutoff = int(arg) - elif opt in ("-s", "--sentence-length"): - length_cutoff = int(arg) - elif opt in ("--remove-traces"): - rm_traces = True - -token_freq = nltk.probability.FreqDist() -lines = [] -for line in sys.stdin: - t = nltk.Tree.parse(line) - pos = t.pos() - if len(pos) <= length_cutoff: - lines.append(pos) - for token, tag in pos: - token_freq.inc(token) - -for line in lines: - for token,tag in line: - if not (rm_traces and tag == "-NONE-"): - if lexicalise: - if token_freq[token] < cutoff: - token = '-UNK-' - print '%s|%s' % (token,tag), - else: - print '%s' % tag, - print -- cgit v1.2.3