summaryrefslogtreecommitdiff
path: root/training/dtrain/parallelize.rb
diff options
context:
space:
mode:
authorChris Dyer <redpony@gmail.com>2013-08-08 13:32:44 -0700
committerChris Dyer <redpony@gmail.com>2013-08-08 13:32:44 -0700
commit951e7daa9539ffe640f9421897c374f786af53e7 (patch)
tree321898257090cc623fa7ea10d81b8e83126a5a0b /training/dtrain/parallelize.rb
parentf4a3a2547316ca5d31366e6808858fe94981415c (diff)
parentaf2b10dd036aa0088cfef108c1c9713b7e2d9f8f (diff)
Merge pull request #24 from pks/master
current dtrain version
Diffstat (limited to 'training/dtrain/parallelize.rb')
-rwxr-xr-xtraining/dtrain/parallelize.rb7
1 files changed, 4 insertions, 3 deletions
diff --git a/training/dtrain/parallelize.rb b/training/dtrain/parallelize.rb
index e661416e..285f3c9b 100755
--- a/training/dtrain/parallelize.rb
+++ b/training/dtrain/parallelize.rb
@@ -4,7 +4,7 @@ require 'trollop'
def usage
STDERR.write "Usage: "
- STDERR.write "ruby parallelize.rb -c <dtrain.ini> [-e <epochs=10>] [--randomize/-z] [--reshard/-y] -s <#shards|0> [-p <at once=9999>] -i <input> -r <refs> [--qsub/-q] [--dtrain_binary <path to dtrain binary>] [-l \"l2 select_k 100000\"]\n"
+ STDERR.write "ruby parallelize.rb -c <dtrain.ini> [-e <epochs=10>] [--randomize/-z] [--reshard/-y] -s <#shards|0> [-p <at once=9999>] -i <input> -r <refs> [--qsub/-q] [--dtrain_binary <path to dtrain binary>] [-l \"l2 select_k 100000\"] [--extra_qsub \"-l virtual_free=24G\"]\n"
exit 1
end
@@ -20,6 +20,7 @@ opts = Trollop::options do
opt :references, "references", :type => :string
opt :qsub, "use qsub", :type => :bool, :default => false
opt :dtrain_binary, "path to dtrain binary", :type => :string
+ opt :extra_qsub, "extra qsub args", :type => :string, :default => ""
end
usage if not opts[:config]&&opts[:shards]&&opts[:input]&&opts[:references]
@@ -119,11 +120,11 @@ end
qsub_str_start = qsub_str_end = ''
local_end = ''
if use_qsub
- qsub_str_start = "qsub -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \""
+ qsub_str_start = "qsub #{opts[:extra_qsub]} -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \""
qsub_str_end = "\""
local_end = ''
else
- local_end = "&>work/out.#{shard}.#{epoch}"
+ local_end = "2>work/out.#{shard}.#{epoch}"
end
pids << Kernel.fork {
`#{qsub_str_start}#{dtrain_bin} -c #{ini}\