summaryrefslogtreecommitdiff
path: root/training/dtrain
diff options
context:
space:
mode:
authorPatrick Simianer <p@simianer.de>2015-02-17 11:16:05 +0100
committerPatrick Simianer <p@simianer.de>2015-02-17 11:16:05 +0100
commit2a37a7ad1b21ab54701de3b5b44dc4ea55a75307 (patch)
tree41645592c04bd78d44cf00df86b5699820b9dee6 /training/dtrain
parent160dbdfa96ae57df82bc33475578904e2cd23317 (diff)
dtrain: parallelize.rb qsub args
Diffstat (limited to 'training/dtrain')
-rwxr-xr-xtraining/dtrain/parallelize.rb9
1 files changed, 4 insertions, 5 deletions
diff --git a/training/dtrain/parallelize.rb b/training/dtrain/parallelize.rb
index fe3a6cf5..9315b7f7 100755
--- a/training/dtrain/parallelize.rb
+++ b/training/dtrain/parallelize.rb
@@ -18,11 +18,11 @@ opts = Trollop::options do
opt :shards, "number of shards", :type => :int
opt :processes_at_once, "have this number (max) running at the same time", :type => :int, :default => 9999
opt :input, "input (bitext f ||| e ||| ...)", :type => :string
- opt :qsub, "use qsub", :type => :bool, :default => false
opt :dtrain_binary, "path to dtrain binary", :type => :string
- opt :extra_qsub, "extra qsub args", :type => :string, :default => ""
- opt :per_shard_decoder_configs, "give special decoder config per shard", :type => :string, :short => '-o'
+ opt :qsub, "use qsub", :type => :bool, :default => false
+ opt :qsub_args, "extra args for qsub", :type => :string, :default => "-l h_vmem=5G"
opt :first_input_weights, "input weights for first iter", :type => :string, :default => '', :short => '-w'
+ opt :per_shard_decoder_configs, "give special decoder config per shard", :type => :string, :short => '-o'
end
usage if not opts[:config]&&opts[:shards]&&opts[:input]
@@ -54,7 +54,6 @@ input = opts[:input]
use_qsub = opts[:qsub]
shards_at_once = opts[:processes_at_once]
first_input_weights = opts[:first_input_weights]
-opts[:extra_qsub] = "-l #{opts[:extra_qsub]}" if opts[:extra_qsub]!=""
`mkdir work`
@@ -119,7 +118,7 @@ end
qsub_str_start = qsub_str_end = ''
local_end = ''
if use_qsub
- qsub_str_start = "qsub #{opts[:extra_qsub]} -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \""
+ qsub_str_start = "qsub #{opts[:qsub_args]} -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \""
qsub_str_end = "\""
local_end = ''
else