summaryrefslogtreecommitdiff
path: root/bitext-filter-length
blob: d1dc973dfcefbb88344ca23f04b057615e950914 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#!/usr/bin/env ruby

require 'zipf'
require 'optimist'

def main
  conf = Optimist::options do
    opt :inputs, "inputs, comma separated", :type => :string, :short => "-i", :required => true
    opt :output_suffix, "output suffix", :type => :string, :default => ".out", :short => "-S"
    opt :output_index, "output index", :type => :bool, :default => false, :short => "-J"
    opt :min_len, "minimum length", :type => :int, :default => 1, :short => "-m"
    opt :max_len, "maximum length", :type => :int, :default => 1000, :short => "-M"
    opt :ignore_below, "minimum length to apply ratio test", :type => :int, :default => 7, :short => "-I"
    opt :ratio_mean, "length ratio average", :type => :float, :required => true, :short => "-A"
    opt :ratio_stddev, "length ratio standard deviation", :type => :float, :required => true, :short => "-T"
    opt :stddev_mult, "+/- n stddevs", :type => :float, :default => 2.0, :short => "-N"
    opt :reverse, "length ratios alway > 1", :type => :bool, :default => false, :short => "-r"
  end

  fna,fnb = conf[:inputs].split ','
  a = ReadFile.new fna 
  b = ReadFile.new fnb

  if not conf[:output_index]
    a_out = WriteFile.new fna+conf[:output_suffix]
    b_out = WriteFile.new fnb+conf[:output_suffix]
  end

  ratio_lower = conf[:ratio_mean] - (conf[:stddev_mult] * conf[:ratio_stddev])
  ratio_upper = conf[:ratio_mean] + (conf[:stddev_mult] * conf[:ratio_stddev])

  i = 0
  while linea = a.gets
    lineb = b.gets
    sza = linea.strip.split.size
    szb = lineb.strip.split.size
    ratio = sza.to_f/szb.to_f
    if conf[:reverse] and ratio < 1
      ratio = ratio**(-1)
    end
    if (sza > 0 and sza <= conf[:ignore_below] and szb > 0 and szb <= conf[:ignore_below]) or
        (sza >= conf[:min_len] and szb >= conf[:min_len] and
         sza <= conf[:max_len] and szb <= conf[:max_len] and
         ratio >= ratio_lower and
         ratio <= ratio_upper)
      if not conf[:output_index]
        a_out.write linea
        b_out.write lineb
      else
        puts i
      end
    end
    i += 1
  end

  a.close
  b.close
  if not conf[:output_index]
    a_out.close
    b_out.close
  end
end

main