lfpe 2017 (neural edition)

4 files changed, 20 insertions, 9 deletions

diff --git a/util/kill_all b/util/kill_all
index e82f822..00c0a65 100755
--- a/util/kill_all
+++ b/util/kill_all
@@ -1,4 +1,4 @@
 #!/bin/bash
 
-for i in {1..6}; do ps ax | grep -P "(server.rb|wrapper.rb|atools|net_fa|sa.extract|dtrain|truecase.perl)" | grep -v vim | grep -v -P "^\s\+$" | cut -d " " -f $i | xargs kill -9 &>/dev/null; done
+for i in {1..6}; do ps ax | grep -P "(server.rb|server-neural.rb|wrapper.rb|atools|net_fa|sa.extract|dtrain|lamtram|truecase.perl)" | grep -v vim | grep -v -P "^\s\+$" | cut -d " " -f $i | xargs kill -9 &>/dev/null; done
 
diff --git a/util/nanomsg_wrapper.rb b/util/nanomsg_wrapper.rb
index d0e6ca7..fbdaafa 100755
--- a/util/nanomsg_wrapper.rb
+++ b/util/nanomsg_wrapper.rb
@@ -5,17 +5,15 @@ require 'open3'
 require 'trollop'
 
 conf = Trollop::options do
-  opt :action,         "tokenize,  detokenize, truecase, or lowercase", :short => "-a", :type => :string, :required => true
+  opt :action,         "tokenize,  detokenize, truecase, lowercase, [de-]bpe", :short => "-a", :type => :string, :required => true
   opt :addr,           "socket address",                    :short => "-S", :type => :string, :required => true
   opt :ext,            "path to externals",                 :short => "-e", :type => :string, :required => true
   opt :lang,           "language",                          :short => "-l", :type => :string
   opt :truecase_model, "model file for truecaser",          :short => "-t", :type => :string
+  opt :bpe,            "codes",                             :short => "-b", :type => :string
+  opt :bpe_vocab,      "BPE vocab",                         :short => "-B", :type => :string
 end
 
-sock = NanoMsg::PairSocket.new
-sock.bind conf[:addr]
-sock.send "hello"
-
 if conf[:action] == "detokenize"
   cmd = "#{conf[:ext]}/detokenizer.perl -q -b -u -l #{conf[:lang]}"
   if !conf[:lang]
@@ -33,10 +31,23 @@ elsif conf[:action] == "truecase"
   end
 elsif conf[:action] == "lowercase"
   cmd = "#{conf[:ext]}/lowercase.perl"
+elsif conf[:action] == "bpe"
+  cmd = "#{conf[:ext]}/apply_bpe.py -c #{conf[:bpe]}" # --vocabulary #{conf[:bpe_vocab]} --vocabulary-threshold 1"
+elsif conf[:action] == "de-bpe"
+  cmd = "#{conf[:ext]}/de-bpe"
 else
   STDERR.write "[wrapper] Unknown action #{conf[:action]}, exiting!\n"; exit
 end
+STDERR.write "[wrapper] will run cmd '#{cmd}'\n"
+
+sock = NanoMsg::PairSocket.new
+STDERR.write "[wrapper] addr: #{conf[:addr]}\n"
+sock.bind conf[:addr]
+sock.send "hello"
+STDERR.write "[wrapper] sent hello\n"
+
 pin, pout, perr = Open3.popen3(cmd)
+
 while true
   inp = sock.recv.strip
   break if !inp||inp=="shutdown"
diff --git a/util/run_all b/util/run_all
index e82ced9..e98c4a7 100755
--- a/util/run_all
+++ b/util/run_all
@@ -4,6 +4,6 @@
 for i in `cat ../sessions/sessions | cut -f 1`; do
   echo $i
   ./util/run_session $i &
-  sleep 10
+  sleep 1
 done
 
diff --git a/util/run_session b/util/run_session
index a4b7a6c..4eecc17 100755
--- a/util/run_session
+++ b/util/run_session
@@ -12,6 +12,6 @@ rm $SESSION_DIR/work/lockfile
 rm -r $SESSION_DIR/work/
 mkdir -p $SESSION_DIR/work
 cp $SESSION_DIR/data.json.original $SESSION_DIR/data.json
-cp $SESSION_DIR/g/original/* $SESSION_DIR/g/
-$UTIL/../server.rb $SESSION_DIR/conf.rb &>$SESSION_DIR/work/session.out
+#cp $SESSION_DIR/g/original/* $SESSION_DIR/g/
+$UTIL/../server-neural.rb $SESSION_DIR/conf.rb &>$SESSION_DIR/work/session.out