summaryrefslogtreecommitdiff
path: root/compound-split
diff options
context:
space:
mode:
Diffstat (limited to 'compound-split')
-rw-r--r--compound-split/README.md (renamed from compound-split/README)0
-rwxr-xr-xcompound-split/make-dict.pl24
2 files changed, 24 insertions, 0 deletions
diff --git a/compound-split/README b/compound-split/README.md
index b7491007..b7491007 100644
--- a/compound-split/README
+++ b/compound-split/README.md
diff --git a/compound-split/make-dict.pl b/compound-split/make-dict.pl
new file mode 100755
index 00000000..71f2b928
--- /dev/null
+++ b/compound-split/make-dict.pl
@@ -0,0 +1,24 @@
+#!/usr/bin/perl -w
+use strict;
+use utf8;
+my %d;
+my $z = 0;
+binmode(STDIN, ":utf8");
+binmode(STDOUT, ":utf8");
+while(<STDIN>) {
+ chomp;
+ s/[\–":“„!=+*.@«#%&,»\?\/{}\$\(\)\[\];\-0-9]+/ /g;
+ $_ = lc $_;
+ my @words = split /\s+/;
+ for my $w (@words) {
+ next if length($w) == 0;
+ $d{$w}++;
+ $z++;
+ }
+}
+my $lz = log($z);
+for my $w (sort {$d{$b} <=> $d{$a}} keys %d) {
+ my $c = $lz-log($d{$w});
+ print "$w $c\n";
+}
+