blob: 0bbd49b45aeb4ac2246baea41a5d2eeb32c8dd79 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
#!/usr/bin/perl -w
use strict;
my $f = <>;
my $IS_PARALLEL = ($f =~ / \|\|\| /);
if ($IS_PARALLEL) {
die "This script is only valid for monolingual corpora, but file contains |||\n";
}
my %d;
my $tc = 0;
my $lc = 0;
while($f) {
$lc++;
chomp $f;
my @toks = split /\s+/, $f;
for my $t (@toks) {
$d{$t}++;
$tc++;
}
$f=<>;
}
my $types = scalar keys %d;
my $ttr = $tc / $types;
my @mfts;
for my $k (sort {$d{$b} <=> $d{$a}} keys %d) {
push @mfts, $k;
last if scalar @mfts > 24;
}
my $sing = 0;
for my $k (keys %d) {
if ($d{$k} == 1) { $sing++; }
}
my $stypes = sqrt($types);
print <<EOT;
CORPUS STATISTICS
Lines: $lc
Tokens: $tc
Types: $types
sqrt(types): $stypes
Type-tok ratio: $ttr
Singletons: $sing
Most freq types: @mfts
EOT
|
