From 6a637210e65194041206be4fafc02188bfe4e01c Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 5 Feb 2014 21:56:55 +0100 Subject: init --- README.md | 11 ++ bold_reranking.rb | 288 +++++++++++++++++++++++++++++++++++++++++++++++ example/example.ini | 19 ++++ example/example.lists.gz | Bin 0 -> 77183 bytes example/example.oracles | 10 ++ example/example.output | 10 ++ example/example.source | 10 ++ 7 files changed, 348 insertions(+) create mode 100644 README.md create mode 100755 bold_reranking.rb create mode 100644 example/example.ini create mode 100644 example/example.lists.gz create mode 100644 example/example.oracles create mode 100644 example/example.output create mode 100644 example/example.source diff --git a/README.md b/README.md new file mode 100644 index 0000000..33eca72 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +bold_reranker +============= + +Reranker that does bold (online) updates on oracles. +Makes use of my little nlp library for ruby, see here +https://github.com/pks/nlp_ruby + + +./bold_reranking.rb example/example.ini + + diff --git a/bold_reranking.rb b/bold_reranking.rb new file mode 100755 index 0000000..3041ced --- /dev/null +++ b/bold_reranking.rb @@ -0,0 +1,288 @@ +#!/usr/bin/env ruby + +require 'nlp_ruby' +require 'bloom-filter' + + +class FeatureFactory + + def initialize cfg + @use_target_ngrams = false + if cfg['ff_target_ngrams'] + @use_target_ngrams = true + args = cfg['ff_target_ngrams'].split + @target_ngrams_n = args[0].to_i + @target_ngrams_fix = true if args.size==2&&args[1]=='fix' + end + @use_phrase_pairs = false + if cfg['ff_phrase_pairs'] + @use_phrase_pairs = true + @phrase_table = nil + args = cfg['ff_phrase_pairs'].split + if args.size==2 + @phrase_table = BloomFilter.load args.last + end + end + @additional_phrase_pairs = {} + @binary = false + @binary = true if cfg['binary_feature_values'] + @filter_features = false + if cfg['filter_features'] + @filter_features = true + @stopwords_target = ReadFile.new(cfg['filter_features']).readlines.map{ |i| i.strip.downcase } + end + end + + def produce translation, source + f = SparseVector.new + phrase_pairs(f, translation, source) if @use_phrase_pairs + target_ngrams(f, translation.s) if @use_target_ngrams + return f + end + + def filter a + single_char = only_stop = only_num = 1 + a.each { |i| + single_char = 0 if i.size > 1 + only_stop = 0 if not @stopwords_target.include? i.downcase + only_num = 0 if i.gsub(/[0-9]+/, '').size > 0 + } + return [single_char,only_stop,only_num].max==1 + end + + def phrase_pairs f, translation, source + target_phrases = translation.get_phrases + return if !target_phrases + spans = translation.get_spans + src_tok = source.split.map{ |i| i.strip } + src_sz = 0.0 + name = nil + spans.each_with_index { |i,j| + next if @filter_features && filter(target_phrases[j]) + i.pop if i.size==2 && i[0]==i[1] + if i.size == 2 + next if !src_tok[i[0]..i[1]] + if @phrase_table + pp = "#{src_tok[i[0]..i[1]].join ' '} ||| #{target_phrases[j]}" + next if !(@phrase_table.include?(pp) || @additional_phrase_pairs.has_key?(pp)) + end + name = "PP:#{src_tok[i[0]..i[1]].join ','}~#{target_phrases[j].split.join ','}" + src_sz = src_tok[i[0]..i[1]].size.to_f + else + if @phrase_table + pp ="#{src_tok[i[0]]} ||| #{target_phrases[j]}" + next if !(@phrase_table.include?(pp) || @additional_phrase_pairs.has_key?(pp)) + end + if i[0] >= 0 + name = "PP:#{src_tok[i[0]]}~#{target_phrases[j]}" + src_sz = 1.0 + end + end + if @binary + f[name] = 1.0 + else + f[name] = src_sz + end + } + end + + def add_phrase_pairs pairs + pairs.each { |i| @additional_phrase_pairs[i] = true } + end + + def target_ngrams f, s + ngrams(s, @target_ngrams_n, @target_ngrams_fix) { |ng| + next if @filter_features && filter(ng) + name = "NG:"+ng.join("_") + if @binary + f[name] = 1.0 + else + f[name] += ng.size + end + } + end +end + +class MosesKbestEntryWithPhraseAlignment < Translation + + def initialize + super + @other_score = -1.0/0 + end + + def get_phrases + @raw.split(/\|-?\d+\||\|\d+-\d+\|/).map{ |i| i.strip }.reject{ |i| i=='' } + end + + def _span span + if span == '-1' + return [-1] + else + return span.split('-').map { |i| i.to_i } + end + end + + def get_spans + @raw.scan(/\|-?\d+\||\|\d+-\d+\|/).map{ |i| i[1..-2] }.map{ |i| _span i } + end + + def other_score model=nil + if model + @other_score = model.dot(@f) + end + return @other_score + end +end + +class ConstrainedSearchOracle < MosesKbestEntryWithPhraseAlignment + + def from_s s + @id = -1 + @raw = s.strip.split(' : ', 2)[1].gsub(/(\[|\])/, '|') + @s = @raw.gsub(/\s*\|\d+-\d+\||\|-?\d+\|\s*/, ' ').gsub(/\s+/, ' ') + @score = 1.0/0 + @other_score = -1.0/0 + end +end + +def structured_update model, hypothesis, oracle + if hypothesis.s != oracle.s + model += oracle.f - hypothesis.f + return [model, 1] + end + return [model, 0] +end + +def ranking_update w, hypothesis, oracle + if oracle.other_score <= hypothesis.other_score \ + && oracle.s != hypothesis.s + model += oracle.f - hypothesis.f + return [model, 1] + end + return [model, 0] +end + +def write_model fn, w + f = WriteFile.new fn + f.write w.to_s+"\n" + f.close +end + +def read_additional_phrase_pairs fn + f = ReadFile.new fn + add = {} + while line = f.gets + id, phrase_pair = line.split ' ', 2 + id = id.to_i-1 + s, t = splitpipe phrase_pair, 3 + phrase_pair = "#{s.strip} ||| #{t.strip}" + if add.has_key? id + add[id] << phrase_pair + else + add[id] = [phrase_pair] + end + end + return add +end + +def usage + STDERR.write "#{__FILE__} \n" + exit 1 +end + +def main + usage if ARGV.size != 1 + cfg = read_cfg ARGV[0] + + sources = ReadFile.new(cfg['sources']).readlines + oracles = ReadFile.new(cfg['oracles']).readlines + kbest_lists = read_kbest_lists cfg['kbest_lists'], MosesKbestEntryWithPhraseAlignment + iterations = cfg['iterate'].to_i + output = WriteFile.new cfg['output'] + output_model = cfg['output_model'] + silent = true if cfg['silent'] + verbose = true if cfg['verbose'] + cheat = true if cfg['cheat'] + + additional_phrase_pairs = nil + if cfg['additional_phrase_pairs'] + additional_phrase_pairs = read_additional_phrase_pairs cfg['additional_phrase_pairs'] + end + + ff = FeatureFactory.new cfg + if !silent + STDERR.write "Running online-reranker with config '#{File.expand_path ARGV[0]}'\n" + cfg.each_pair { |k,v| STDERR.write " #{k} = #{v}\n" } + STDERR.write "\n" + end + + model = SparseVector.new + if cfg['init_model'] + model.from_s ReadFile.new(cfg['init_model']).read + end + + sz = sources.size + start = Time.now + iterations.times { + |t| + overall_errors = 0 + STDERR.write "Iteration #{t+1} of #{iterations}\n" + sources.each_with_index { |i,j| + STDERR.write " #{j+1}\n" if (j+1)%10==0 && !silent&&!verbose + + ff.add_phrase_pairs(additional_phrase_pairs[j]) if additional_phrase_pairs + + kbest = kbest_lists[j] + kbest.each { |k| + k.f = ff.produce k, sources[j] + k.other_score model + } + + hypothesis = kbest[ kbest.map{ |k| k.other_score }.max_index ] + + if !cheat + output.write "#{hypothesis.s}\n" + end + + oracle = ConstrainedSearchOracle.new + oracle.from_s oracles[j] + oracle.f = ff.produce oracle, sources[j] + oracle.other_score model + + err = 0 + case cfg['update'] + when 'structured' + model, err = structured_update model, hypothesis, oracle + when 'ranking' + model, err = ranking_update model, hypothesis, oracle + else + STDERR.write "Don't know update method '#{cfg['update']}', exiting.\n" + exit 1 + end + overall_errors += err + + if cheat + kbest.each { |k| k.other_score model } + hypothesis = kbest[ kbest.map{ |k| k.other_score }.max_index ] + output.write "#{hypothesis.s}\n" + end + + if verbose + counts = { 'PP'=>0, 'NG'=>0 } + model.each_pair { |k,v| + counts[k.split(':').first] += 1 + } + STDERR.write "errors=#{overall_errors}; model size=#{model.size} (PP #{counts['PP']}, ng #{counts['NG']})\n" if verbose + end + } + } + + elapsed = Time.now - start + STDERR.write"#{elapsed.round 2} s, #{(elapsed/Float(sz)).round 2} s per kbest; model size: #{model.size}\n\n" if !silent + write_model(output_model, model) if output_model + output.close +end + + +main + diff --git a/example/example.ini b/example/example.ini new file mode 100644 index 0000000..46686bd --- /dev/null +++ b/example/example.ini @@ -0,0 +1,19 @@ +sources = example/example.source +oracles = example/example.oracles +kbest_lists = example/example.lists.gz + +update = structured # ranking +ff_target_ngrams = 4 # 4 fix +ff_phrase_pairs = true # true /path/to/phrase_table +#filter_features = /path/to/target/stopwords_file +binary_feature_values = true +iterate = 3 + +output = - +output_model = /dev/null +#additional_phrase_pairs = /path/to/list/of/additional_phrase_pairs +#init_model = /path/to/model + +#silent = true +#cheat = true +verbose = true diff --git a/example/example.lists.gz b/example/example.lists.gz new file mode 100644 index 0000000..45ab7ba Binary files /dev/null and b/example/example.lists.gz differ diff --git a/example/example.oracles b/example/example.oracles new file mode 100644 index 0000000..ddf62f5 --- /dev/null +++ b/example/example.oracles @@ -0,0 +1,10 @@ +11101 4 -86 : Locking device [0] with a large [1-2] number [-1] of [-1] locking [-1] combinations [-1] . [4] +1111101111111111111111111111011111111011111111111111111111111111111111111111111 78 -42 : The invention relates to a [0-4] locking device [9-10] which [11-12] consists [-1] of [26-27] a [31-33] lock [35-36] and [29-30] key [13-14] and [6-7] which [8] has [19-20] tumblers [18] transferable [-1] into [55-58] the [62-65] release position [59-60] by means [52] of [49-50] the [46-47] key [51] and [40] , [38-39] particularly [22] for a large [23-25] number [-1] of [34] lock [-1] combinations [-1] , [61] proposes [-1] that [-1] at least one [66-67] of the [69-70] tumblers [-1] ( [73] 20 [-1] ) [75] , [77] in [-1] addition [-1] to [-1] or [-1] alternatively to [41-42] its [43] direct control [44-45] by [-1] the [-1] basic code [48] of [-1] the [-1] corresponding [-1] key [-1] , [21] can be [17] brought [-1] into the release position [15-16] by [-1] means [-1] of a magnetic coil [53-54] ( [-1] 43 [-1] ) [-1] which [-1] can [-1] be energised [76] by [-1] a [-1] reading device [72] ( [-1] 44 ) [74] detecting [71] at [-1] least [-1] one [-1] supplementary code [68] of [-1] the [-1] key [-1] . [78] +111111111111111111111 20 72 : The present invention relates to a [0-3] locking [9] device which consists of a lock [4-5] and a key [6-7] and [8] which [10-11] has [19] tumblers [18] that can be [17] brought [-1] into a release position [15-16] by [12] the key [13-14] . [20] +11111 4 96 : Such [0] locking devices [1] are known . [2-4] +1111111111111011111 12 -74 : They [0] can be formed [16-18] , [10-11] for [-1] instance [-1] , [8-9] of [2-3] locking [7] cylinders [-1] which [6] can be [1] locked [-1] by [4] flat keys [5] or [-1] else [-1] also [-1] of [-1] locking devices [15] which [14] are [-1] operated [-1] with [12] magnetic [-1] keys [-1] . [-1] +111100111110101111111111111111 29 -25 : All [0] of [6-10] these [1] locking devices [2] share [-1] the [19-20] common [-1] feature [-1] that [-1] while [-1] the [23-24] number [-1] of [16-17] locking [-1] combinations [-1] is [3] frequently [12] considerable [14-15] , [18] this [-1] number [-1] however [-1] , could be [27-28] further increased [25-26] in order to increase [21-22] security [-1] . [29] +1111111101111111111111111111111 30 71 : Particularly [0] with respect to [1-3] locking systems [4] having [5] master [-1] and [6-7] specific [-1] keys [9] there is [10] a need [11-13] to make available [21-24] in [14] a locking system [15-16] a [17] large number of [18-19] keys [20] which have different [25-27] access authorizations [28] . [29-30] +10111111111111111111111111111 28 66 : Such [2-3] locking systems [4] frequently [0] have the disadvantage that [5-8] the [9-10] main [-1] master [-1] key [11] which [12-13] can be used [19-20] for all locks [14-16] of the locking system [17-18] has [27] only [21-22] a relatively [23-24] simple coding [25-26] . [28] +111111111111111111011111111110111110011111111101111 50 8 : In the case of [0-3] locking devices [4] which [5-6] are used [11-12] by [7] continuously changing [8-9] users [10] ( for instance [13-16] , [22-23] in [17] hotels [-1] ) [19-21] , [37-38] it [40] is [27] necessary [-1] to ensure that [24-26] the [28] previous [-1] user [-1] cannot [30-31] easily [32] obtain [-1] a [33-34] copy [-1] of the [39] key [-1] , [-1] which [-1] would [-1] raise [-1] the [44] possibility [-1] of [-1] his [-1] unauthorized [41] access into the [42-43] room [48] inhabited [47] by the [45] subsequent [-1] user [-1] . [49-50] +1111001111101111110111111111 27 44 : In many known [0-2] locking devices [3] an increase in the number of [6-10] locking [-1] combinations [-1] also [12] means [-1] an [13] increase in the [15-16] structural [14] size [-1] of the [24] key [17] , which has a [19-21] detrimental effect on [22-23] convenience [25] . [26-27] diff --git a/example/example.output b/example/example.output new file mode 100644 index 0000000..546e694 --- /dev/null +++ b/example/example.output @@ -0,0 +1,10 @@ +Locking device with large Schliesskombinationsanzahl . +The invention relates to a locking device consisting of a lock and key , which by means of the key in the tractor tumblers , and proposes , in particular for a high number of Schließkombinationen before , that at least one of the tumblers ( 20 ) , in addition to or as an alternative to the direct controlling of the basic code of the corresponding key by means of a magnet coil ( 43 ) in the release position , the at least one supplementary code of the key is a reader ( 44 ) is excited . +The invention relates to a device consisting of a lock and key , by means of the key in the tractor tumblers . +This type of locking devices are known . +They can be , for example , of flat keys which lock cylinder or , however , also be formed with Magnetschlüsseln operating mechanisms . +All of these closing devices have in common is that the number of Schließkomtinationen often a considerable size which , however , in order to increase the security can be increased even further . +In particular in view of locking systems with lye and minor keys , there is a need , within a locking system , a large number of keys available which have different access authorizations . +Frequently , such locking systems have the disadvantage that the key , for all of the locks of the locking system is used , only a relatively simple coding . +If it is decoupled from the constantly changing users are used ( e.g. in Hotelbetrieb ) , so it is also ensured that the Vorbenutzer not in a simple manner a Schlüsselkopie throughout , it can access into the space occupied by the Nachbenutzer relinquished . +In many known locking devices , with an increase in the number of the Schließkombinationen also a constructional enlargement of the key , which has a disadvantageous effect on the comfort . diff --git a/example/example.source b/example/example.source new file mode 100644 index 0000000..56b617c --- /dev/null +++ b/example/example.source @@ -0,0 +1,10 @@ +Schliesseinrichtung mit grosser Schliesskombinationsanzahl . +Die Erfindung betrifft eine aus Schloß und Schlüssel bestehende Schließeinrichtung , die mittels des Schlüssels in Freigabestellung überführbare Zuhaltungen aufweist und schlägt insbesondere für eine hohe Anzahl von Schließkombinationen vor , daß mindestens eine der Zuhaltungen ( 20 ) zusätzlich oder alternativ zu ihrer direkten Steuerung von dem Grundcode des entsprechenden Schlüssels mittels einer Magnetspule ( 43 ) in Freigabestellung bringbar ist , die von einer mindestens einen Ergänzungscode des Schlüssels erfassenden Leseeinrichtung ( 44 ) erregbar ist . +Die Erfindung betrifft eine aus Schloß und Schlüssel bestehende Schließeinrichtung , die mittels des Schlüssels in Freigabestellung überführbare Zuhaltungen aufweist . +Derartige Schließeinrichtungen sind bekannt . +Sie können beispielsweise von mit Flachschlüsseln schließbaren Schließzylinder oder aber auch von mit Magnetschlüsseln arbeitenden Schließeinrichtungen gebildet sein . +Allen diesen Schließeinrichtungen ist es gemeinsam , daß die Anzahl der Schließkomtinationen oftmals zwar eine beträchtliche Größe aufweist , die jedoch zur Erhöhung der Sicherheit noch vergrößert werden könnte . +Insbesondere im Hinblick auf Schließanlagen mit über- und untergeordneten Schlüsseln besteht der Bedarf , innerhalb einer Schließanlage eine Vielzahl von Schlüsseln zur Verfügung zu stellen , die unterschiedliche Zugangsberechtigungen besitzen . +Häufig besteht bei derartigen Schließanlagen der Nachteil , daß gerade der Generalschlüssel , der für sämtliche Schlösser der Schließanlage benutzbar ist , nur eine relativ einfache Codierung besitzt . +Handelt es sich um Schließeinrichtungen , die von ständig wechselnden Benutzern benutzt werden ( z . B. im Hotelbetrieb ) , so ist überdies sicherzustellen , daß sich der Vorbenutzer nicht auf einfachem Wege eine Schlüsselkopie beschaffen kann , die ihm unberechtigten Zugang in den vom Nachbenutzer bewohnten Raum verschafft . +Bei vielen bekannten Schließeinrichtungen geht mit einer Erhöhung der Anzahl der Schließkombinationen auch eine bauliche Vergrößerung des Schlüssels einher , was sich nachteilig auf den Komfort auswirkt . -- cgit v1.2.3