From 3131d8251e7729c9667e86b18797754e9690d6ae Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Wed, 24 Jun 2015 11:50:57 +0200 Subject: [PATCH] Modify batch_generator to allow create batch of increasing sizes (#7152) --- lib/metasync | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/metasync b/lib/metasync index 74c1a99..1c0cf04 100755 --- a/lib/metasync +++ b/lib/metasync @@ -228,13 +228,15 @@ if args.ldap_binddn: def lower_keys(d): return dict((key.lower(), value) for key, value in d.iteritems()) -def batch_generator(gen, batch_size): +def batch_generator(gen, *batch_size): batch = [] for result in gen: batch.append(result) - if len(batch) == batch_size: + if len(batch) == batch_size[0]: yield batch batch = [] + if len(batch_size) > 1: + batch_size = batch_size[1:] if len(batch): yield batch @@ -361,7 +363,7 @@ def apply_action(action): actions.sort(key=lambda x: (x[0] == DELETE, len(x[1]))) if args.verbose: print 'Actions:' -for batch in batch_generator(actions, 100): +for batch in batch_generator(actions, 1, 5, 10, 50, 100): for action in batch: if args.verbose: print_action(action)