summaryrefslogtreecommitdiff
path: root/training/dtrain/parallelize.rb
diff options
context:
space:
mode:
authorPatrick Simianer <simianer@cl.uni-heidelberg.de>2013-01-24 16:28:23 +0100
committerPatrick Simianer <simianer@cl.uni-heidelberg.de>2013-01-24 16:28:23 +0100
commit7c4a9e0825b15ce6c08c45c7654c614d542cf93a (patch)
treeebb16e789ccf4001ae9b84da80c2bf99b46f1840 /training/dtrain/parallelize.rb
parent3625d2efc7d3cf647c2bc0af9ba83a7cde4a31f3 (diff)
made examples work again
Diffstat (limited to 'training/dtrain/parallelize.rb')
-rwxr-xr-xtraining/dtrain/parallelize.rb12
1 files changed, 8 insertions, 4 deletions
diff --git a/training/dtrain/parallelize.rb b/training/dtrain/parallelize.rb
index 6e30cf9d..9b0923f6 100755
--- a/training/dtrain/parallelize.rb
+++ b/training/dtrain/parallelize.rb
@@ -7,10 +7,10 @@ if ARGV.size != 7
exit
end
-cdec_dir = '~/MAREC/cdec-dtrain/'
-dtrain_bin = "~/MAREC/cdec-dtrain/training/dtrain/dtrain"
+dtrain_dir = File.expand_path File.dirname(__FILE__)
+dtrain_bin = "#{dtrain_dir}/dtrain"
ruby = '/usr/bin/ruby'
-lplp_rb = "#{cdec_dir}/training/dtrain/hstreaming/lplp.rb"
+lplp_rb = "#{dtrain_dir}/hstreaming/lplp.rb"
lplp_args = 'l2 select_k 100000'
cat = '/bin/cat'
@@ -96,15 +96,19 @@ end
while remaining_shards > 0
shards_at_once.times {
qsub_str_start = qsub_str_end = ''
+ local_end = ''
if use_qsub
qsub_str_start = "qsub -cwd -sync y -b y -j y -o work/out.#{shard}.#{epoch} -N dtrain.#{shard}.#{epoch} \""
qsub_str_end = "\""
+ local_end = ''
+ else
+ local_end = "&>work/out.#{shard}.#{epoch}"
end
pids << Kernel.fork {
`#{qsub_str_start}#{dtrain_bin} -c #{ini}\
--input #{input_files[shard]}\
--refs #{refs_files[shard]} #{input_weights}\
- --output work/weights.#{shard}.#{epoch}#{qsub_str_end}`
+ --output work/weights.#{shard}.#{epoch}#{qsub_str_end} #{local_end}`
}
weights_files << "work/weights.#{shard}.#{epoch}"
shard += 1