From 198cc8c885a69b9bbe06c842d3a868583885c526 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Thu, 25 Oct 2012 16:05:56 -0400 Subject: add self translation --- corpus/add-self-translations.pl | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 corpus/add-self-translations.pl (limited to 'corpus') diff --git a/corpus/add-self-translations.pl b/corpus/add-self-translations.pl new file mode 100755 index 00000000..153bc454 --- /dev/null +++ b/corpus/add-self-translations.pl @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w +use strict; + +# ADDS SELF-TRANSLATIONS OF POORLY ATTESTED WORDS TO THE PARALLEL DATA + +my %df; +my %def; +while(<>) { + print; + chomp; + my ($sf, $se) = split / \|\|\| /; + die "Format error: $_\n" unless defined $sf && defined $se; + my @fs = split /\s+/, $sf; + my @es = split /\s+/, $se; + for my $f (@fs) { + $df{$f}++; + for my $e (@es) { + if ($f eq $e) { $def{$f}++; } + } + } +} + +for my $k (sort keys %def) { + next if $df{$k} > 4; + print "$k ||| $k\n"; + print "$k ||| $k\n"; + print "$k ||| $k\n"; +} + -- cgit v1.2.3