From da222df300e4f87ad185a7decbf119ad56aa34e0 Mon Sep 17 00:00:00 2001
From: Chris Dyer <redpony@gmail.com>
Date: Fri, 29 Jan 2010 15:56:59 +0000
Subject: word aligner checkin

---
 word-aligner/extract_vocab.pl | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100755 word-aligner/extract_vocab.pl

(limited to 'word-aligner/extract_vocab.pl')

diff --git a/word-aligner/extract_vocab.pl b/word-aligner/extract_vocab.pl
new file mode 100755
index 00000000..070d4202
--- /dev/null
+++ b/word-aligner/extract_vocab.pl
@@ -0,0 +1,20 @@
+#!/usr/bin/perl -w
+use strict;
+
+print STDERR "Extracting vocabulary...\n";
+my %dict = ();
+my $wc = 0;
+while(<>) {
+  chomp;
+  my @words = split /\s+/;
+  for my $word (@words) { $wc++; $dict{$word}++; }
+}
+
+my $tc = 0;
+for my $word (sort {$dict{$b} <=> $dict{$a}} keys %dict) {
+  print "$word\n";
+  $tc++;
+}
+
+print STDERR "$tc types / $wc tokens\n";
+
-- 
cgit v1.2.3