From df5445c3651fa1cc99ed4bdb682dcf57092dd4e2 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@cs.cmu.edu>
Date: Thu, 25 Oct 2012 16:05:56 -0400
Subject: add self translation

---
 corpus/add-self-translations.pl | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100755 corpus/add-self-translations.pl

diff --git a/corpus/add-self-translations.pl b/corpus/add-self-translations.pl
new file mode 100755
index 00000000..153bc454
--- /dev/null
+++ b/corpus/add-self-translations.pl
@@ -0,0 +1,29 @@
+#!/usr/bin/perl -w
+use strict;
+
+# ADDS SELF-TRANSLATIONS OF POORLY ATTESTED WORDS TO THE PARALLEL DATA
+
+my %df;
+my %def;
+while(<>) {
+  print;
+  chomp;
+  my ($sf, $se) = split / \|\|\| /;
+  die "Format error: $_\n" unless defined $sf && defined $se;
+  my @fs = split /\s+/, $sf;
+  my @es = split /\s+/, $se;
+  for my $f (@fs) {
+    $df{$f}++;
+    for my $e (@es) {
+      if ($f eq $e) { $def{$f}++; }
+    }
+  }
+}
+
+for my $k (sort keys %def) {
+  next if $df{$k} > 4;
+  print "$k ||| $k\n";
+  print "$k ||| $k\n";
+  print "$k ||| $k\n";
+}
+
-- 
cgit v1.2.3