diff options
Diffstat (limited to 'util')
| -rwxr-xr-x | util/kill_all | 2 | ||||
| -rwxr-xr-x | util/nanomsg_wrapper.rb | 21 | ||||
| -rwxr-xr-x | util/run_all | 2 | ||||
| -rwxr-xr-x | util/run_session | 4 |
4 files changed, 20 insertions, 9 deletions
diff --git a/util/kill_all b/util/kill_all index e82f822..00c0a65 100755 --- a/util/kill_all +++ b/util/kill_all @@ -1,4 +1,4 @@ #!/bin/bash -for i in {1..6}; do ps ax | grep -P "(server.rb|wrapper.rb|atools|net_fa|sa.extract|dtrain|truecase.perl)" | grep -v vim | grep -v -P "^\s\+$" | cut -d " " -f $i | xargs kill -9 &>/dev/null; done +for i in {1..6}; do ps ax | grep -P "(server.rb|server-neural.rb|wrapper.rb|atools|net_fa|sa.extract|dtrain|lamtram|truecase.perl)" | grep -v vim | grep -v -P "^\s\+$" | cut -d " " -f $i | xargs kill -9 &>/dev/null; done diff --git a/util/nanomsg_wrapper.rb b/util/nanomsg_wrapper.rb index d0e6ca7..fbdaafa 100755 --- a/util/nanomsg_wrapper.rb +++ b/util/nanomsg_wrapper.rb @@ -5,17 +5,15 @@ require 'open3' require 'trollop' conf = Trollop::options do - opt :action, "tokenize, detokenize, truecase, or lowercase", :short => "-a", :type => :string, :required => true + opt :action, "tokenize, detokenize, truecase, lowercase, [de-]bpe", :short => "-a", :type => :string, :required => true opt :addr, "socket address", :short => "-S", :type => :string, :required => true opt :ext, "path to externals", :short => "-e", :type => :string, :required => true opt :lang, "language", :short => "-l", :type => :string opt :truecase_model, "model file for truecaser", :short => "-t", :type => :string + opt :bpe, "codes", :short => "-b", :type => :string + opt :bpe_vocab, "BPE vocab", :short => "-B", :type => :string end -sock = NanoMsg::PairSocket.new -sock.bind conf[:addr] -sock.send "hello" - if conf[:action] == "detokenize" cmd = "#{conf[:ext]}/detokenizer.perl -q -b -u -l #{conf[:lang]}" if !conf[:lang] @@ -33,10 +31,23 @@ elsif conf[:action] == "truecase" end elsif conf[:action] == "lowercase" cmd = "#{conf[:ext]}/lowercase.perl" +elsif conf[:action] == "bpe" + cmd = "#{conf[:ext]}/apply_bpe.py -c #{conf[:bpe]}" # --vocabulary #{conf[:bpe_vocab]} --vocabulary-threshold 1" +elsif conf[:action] == "de-bpe" + cmd = "#{conf[:ext]}/de-bpe" else STDERR.write "[wrapper] Unknown action #{conf[:action]}, exiting!\n"; exit end +STDERR.write "[wrapper] will run cmd '#{cmd}'\n" + +sock = NanoMsg::PairSocket.new +STDERR.write "[wrapper] addr: #{conf[:addr]}\n" +sock.bind conf[:addr] +sock.send "hello" +STDERR.write "[wrapper] sent hello\n" + pin, pout, perr = Open3.popen3(cmd) + while true inp = sock.recv.strip break if !inp||inp=="shutdown" diff --git a/util/run_all b/util/run_all index e82ced9..e98c4a7 100755 --- a/util/run_all +++ b/util/run_all @@ -4,6 +4,6 @@ for i in `cat ../sessions/sessions | cut -f 1`; do echo $i ./util/run_session $i & - sleep 10 + sleep 1 done diff --git a/util/run_session b/util/run_session index a4b7a6c..4eecc17 100755 --- a/util/run_session +++ b/util/run_session @@ -12,6 +12,6 @@ rm $SESSION_DIR/work/lockfile rm -r $SESSION_DIR/work/ mkdir -p $SESSION_DIR/work cp $SESSION_DIR/data.json.original $SESSION_DIR/data.json -cp $SESSION_DIR/g/original/* $SESSION_DIR/g/ -$UTIL/../server.rb $SESSION_DIR/conf.rb &>$SESSION_DIR/work/session.out +#cp $SESSION_DIR/g/original/* $SESSION_DIR/g/ +$UTIL/../server-neural.rb $SESSION_DIR/conf.rb &>$SESSION_DIR/work/session.out |