From 32e8ad53760ad5743f39fd6a522ca7ba1e9516c3 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 20 Dec 2017 22:26:57 +0100 Subject: python/shards.py --- python/shards.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 python/shards.py diff --git a/python/shards.py b/python/shards.py new file mode 100644 index 0000000..f47921f --- /dev/null +++ b/python/shards.py @@ -0,0 +1,23 @@ +total = 100000 +data=range(total) +jobSize = 5000 +numJobs = (total // jobSize) + 1 +numSegmentsPerJob = total // numJobs +print numSegmentsPerJob +print(numJobs) +print('---') + +start = 0 +count = 0 +for i in range(numJobs): + if i == numJobs-1: + end = total + else: + end = start+numSegmentsPerJob + shard = data[start:end] + print(len(shard)) + count += len(shard) + start += numSegmentsPerJob + +print(count) + -- cgit v1.2.3