entrouvert
/
wcs-olap
16
0
Fork 0
Code Issues Pull Requests Releases Activity

use multiprocessing to launch import in separate processes (fixes #13733)

This commit is contained in:
Benjamin Dauvergne 2016-10-25 11:18:20 +02:00
parent 18b023e91b
commit 561e91556f
1 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
wcs_olap/cmd.py
View File

@ -7,6 +7,7 @@ import logging.config
from . import wcs_api
from .feeder import WcsOlapFeeder
import locale
from multiprocessing import Process, Queue
from . import tb
@ -93,17 +94,25 @@ def main2():
failure = True
logger.error('configuration incomplete for %s: %s', url, e)
else:
try:
api = wcs_api.WcsApi(url=url, orig=orig, key=key, slugs=slugs,
verify=defaults.get('verify', 'True') == 'True')
logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url, pg_dsn)
feeder = WcsOlapFeeder(api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,
config=defaults, do_feed=feed)
feeder.feed()
logger.info('finished')
except:
failure = True
logger.exception('failed to synchronize with %s', url)
def job(queue):
try:
api = wcs_api.WcsApi(url=url, orig=orig, key=key, slugs=slugs,
verify=defaults.get('verify', 'True') == 'True')
logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url,
pg_dsn)
feeder = WcsOlapFeeder(api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,
config=defaults, do_feed=feed)
feeder.feed()
logger.info('finished')
queue.put(False)
except:
queue.put(True)
logger.exception('failed to synchronize with %s', url)
queue = Queue()
p = Process(target=job, args=(queue,))
p.start()
p.join()
failure = queue.get()
defaults = {}
if failure:
sys.exit(1)