From 7928695272b000de7142b91e05959a8fab6b1d2a Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Wed, 14 Nov 2012 20:33:51 -0500
Subject: major mert clean up, stuff for simple system demo

---
 compound-split/make-dict.pl | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100755 compound-split/make-dict.pl

(limited to 'compound-split/make-dict.pl')

diff --git a/compound-split/make-dict.pl b/compound-split/make-dict.pl
new file mode 100755
index 00000000..71f2b928
--- /dev/null
+++ b/compound-split/make-dict.pl
@@ -0,0 +1,24 @@
+#!/usr/bin/perl -w
+use strict;
+use utf8;
+my %d;
+my $z = 0;
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+while(<STDIN>) {
+  chomp;
+  s/[\–":“„!=+*.@«#%&,»\?\/{}\$\(\)\[\];\-0-9]+/ /g;
+  $_ = lc $_;
+  my @words = split /\s+/;
+  for my $w (@words) {
+    next if length($w) == 0;
+    $d{$w}++;
+    $z++;
+  }
+}
+my $lz = log($z);
+for my $w (sort {$d{$b} <=> $d{$a}} keys %d) {
+  my $c = $lz-log($d{$w});
+  print "$w $c\n";
+}
+
-- 
cgit v1.2.3