entrouvert
/
wcs-olap
16
0
Fork 0
Code Issues Pull Requests Releases Activity

make batch size configurable (#17932)

This commit is contained in:
Frédéric Péters 2017-08-08 13:21:45 +02:00
parent d5debf5200
commit 212bdeddf7
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
wcs_olap/cmd.py
View File

@ -79,7 +79,7 @@ def main2():
parser.add_argument('--pg-dsn', help='Psycopg2 DB DSN')
parser.add_argument('--schema', help='schema name')
args = parser.parse_args()
for key in ('orig', 'key', 'pg_dsn', 'schema'):
for key in ('orig', 'key', 'pg_dsn', 'schema', 'batch_size'):
if getattr(args, key, None):
defaults[key] = getattr(args, key)
@ -96,6 +96,7 @@ def main2():
schema = defaults['schema']
pg_dsn = defaults['pg_dsn']
slugs = defaults.get('slugs', '').strip().split() or getattr(args, 'slug', [])
batch_size = int(defaults.get('batch_size', 1000))
except KeyError, e:
failure = True
logger.error('configuration incomplete for %s: %s', url, e)
@ -103,7 +104,8 @@ def main2():
def job(queue):
try:
api = wcs_api.WcsApi(url=url, orig=orig, key=key, slugs=slugs,
verify=defaults.get('verify', 'True') == 'True')
verify=defaults.get('verify', 'True') == 'True',
batch_size=batch_size)
logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url,
pg_dsn)
feeder = WcsOlapFeeder(api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,

5
wcs_olap/wcs_api.py
View File

@ -169,13 +169,14 @@ class Category(BaseObject):
class WcsApi(object):
def __init__(self, url, orig, key, verify=True, slugs=None):
def __init__(self, url, orig, key, verify=True, slugs=None, batch_size=1000):
self.url = url
self.orig = orig
self.key = key
self.verify = verify
self.cache = {}
self.slugs = slugs or []
self.batch_size = batch_size
@property
def formdefs_url(self):
@ -229,7 +230,7 @@ class WcsApi(object):
def get_formdata(self, slug):
offset = 0
limit = 1000
limit = self.batch_size
while True:
data = self.get_json(self.forms_url,
slug + '/list?anonymise&full=on&offset=%d&limit=%d' % (offset, limit))