From 0763276cca283fa0e905fc2eb692f13a7cf5048e Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Mon, 19 Aug 2019 18:38:18 +0200 Subject: [PATCH] log to syslog and eventually to stderr --- cd06/script.py | 158 +++++++++++++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 59 deletions(-) diff --git a/cd06/script.py b/cd06/script.py index 6322055..3bab6b0 100644 --- a/cd06/script.py +++ b/cd06/script.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- -import glob +from __future__ import unicode_literals + +import logging +import logging.handlers import sys import os import shutil @@ -8,13 +11,6 @@ from cd06.wcs_api import WcsApi from config import WCS_URL, WCS_ORIG, WCS_APIKEY, USER_UUID -api = WcsApi(WCS_URL, - WCS_ORIG, - WCS_APIKEY, - name_id=USER_UUID) - -target_dir = sys.argv[1] - FORMDEFS = { 'premiere-demande-d-apa-a-domicile': { 'DIRECTORY': 'premiere-demande-apa', @@ -59,62 +55,106 @@ FORMDEFS = { }, } -print('Moving files into', target_dir) +logger = logging.getLogger('cd06.cron.apa') +logger.setLevel(logging.INFO) +logger.propagate = False -tmp_dir = os.path.join(target_dir, 'tmp') -if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) +syslog_formatter = logging.Formatter('%(levelname)s - %(name)s - %(message)s') +syslog_handler = logging.handlers.SysLogHandler(address='/dev/log') +syslog_handler.setLevel(logging.DEBUG) +syslog_handler.setFormatter(syslog_formatter) -for formdef in api.formdefs: - if formdef.slug not in FORMDEFS: - continue - print(formdef.title) - FORMDEF = FORMDEFS[formdef.slug] - MAPPINGS = FORMDEF['MAPPINGS'] - DIRECTORY = FORMDEF['DIRECTORY'] - ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS'] - formdef_dir = os.path.join(target_dir, DIRECTORY) - if not os.path.exists(formdef_dir): - os.makedirs(formdef_dir) +console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +console_handler = logging.StreamHandler(sys.stderr) +console_handler.setLevel(logging.DEBUG) +console_handler.setFormatter(console_formatter) - known_ids = set() - statuses = {s.id: s.name for s in formdef.schema.workflow.statuses} +memory_handler = logging.handlers.MemoryHandler( + capacity=1024 * 100, + flushLevel=logging.ERROR, + target=console_handler, +) +logger.addHandler(memory_handler) +logger.addHandler(syslog_handler) - for formdata in formdef.datas: - form_id = str(formdata.id).rsplit('/')[-1] - known_ids.add(form_id) - formdata_dir = os.path.join(formdef_dir, form_id) - formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id) - if os.path.exists(formdata_dir): - status_name = statuses[formdata.evolution[-1].status] +def do(): + api = WcsApi(WCS_URL, + WCS_ORIG, + WCS_APIKEY, + name_id=USER_UUID) - if status_name == ANONYMISATION_STATUS: - print('Anonymisation de ', form_id) - for name in os.listdir(formdata_dir): - os.unlink(os.path.join(formdata_dir, name)) + target_dir = sys.argv[1] + + tmp_dir = os.path.join(target_dir, 'tmp') + if not os.path.exists(tmp_dir): + logger.info('creating %s', tmp_dir) + os.makedirs(tmp_dir) + + for formdef in api.formdefs: + if formdef.slug not in FORMDEFS: continue - try: - shutil.rmtree(formdata_tmp_dir, True) - os.makedirs(formdata_tmp_dir) - print(' Form', form_id) - for name, key, filename in MAPPINGS: - value = formdata[key] - if not value: - continue - print(' -', name, ':', value.filename) - prefix, suffix = os.path.splitext(filename) - new_filename = prefix + os.path.splitext(value.filename)[1] - attachment_path = os.path.join(formdata_tmp_dir, new_filename) - print('Putting in', attachment_path) - with open(attachment_path, 'wb') as f: - f.write(value.content) - os.rename(formdata_tmp_dir, formdata_dir) - finally: - shutil.rmtree(formdata_tmp_dir, True) + logger.debug('handling formdef %s', formdef.slug) + FORMDEF = FORMDEFS[formdef.slug] + MAPPINGS = FORMDEF['MAPPINGS'] + DIRECTORY = FORMDEF['DIRECTORY'] + ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS'] + formdef_dir = os.path.join(target_dir, DIRECTORY) + if not os.path.exists(formdef_dir): + logger.info('creating %s', formdef_dir) + os.makedirs(formdef_dir) - existing_ids = set(os.listdir(formdef_dir)) - for old_id in (existing_ids - known_ids): - path_to_delete = os.path.join(formdef_dir, old_id) - print('Would delete', path_to_delete) - print() + known_ids = set() + statuses = {s.id: s.name for s in formdef.schema.workflow.statuses} + + for formdata in formdef.datas: + form_id = str(formdata.id).rsplit('/')[-1] + logging.debug('handling formdata %s', form_id) + known_ids.add(form_id) + formdata_dir = os.path.join(formdef_dir, form_id) + formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id) + + if os.path.exists(formdata_dir): + status_name = statuses[formdata.evolution[-1].status] + + if status_name == ANONYMISATION_STATUS: + if os.listdir(formdata_dir): + logger.info('anonymizing %s', formdata_dir) + for name in os.listdir(formdata_dir): + os.unlink(os.path.join(formdata_dir, name)) + else: + logger.debug('already anonymized %s', formdata_dir) + else: + logger.info('%s: copying formdata %s', formdef.slug, form_id) + try: + shutil.rmtree(formdata_tmp_dir, True) + os.makedirs(formdata_tmp_dir) + for name, key, filename in MAPPINGS: + value = formdata[key] + if not value: + logger.info('no file for "%s"', name) + continue + prefix, suffix = os.path.splitext(filename) + new_filename = prefix + os.path.splitext(value.filename)[1] + attachment_path = os.path.join(formdata_tmp_dir, new_filename) + logger.info('putting %s into %s for "%s"', value.filname, attachment_path, name) + with open(attachment_path, 'wb') as f: + f.write(value.content) + os.rename(formdata_tmp_dir, formdata_dir) + finally: + shutil.rmtree(formdata_tmp_dir, True) + + existing_ids = set(os.listdir(formdef_dir)) + for old_id in (existing_ids - known_ids): + path_to_delete = os.path.join(formdef_dir, old_id) + logger.delete('deleting %s', path_to_delete) + shutil.rmtree(path_to_delete) + +logger.info('start sending APA requests') +try: + do() +except Exception: + logger.exception('unexpected failure') +logger.info('end sending APA requests') +# clear buffered log records, to prevent flush on close +super(logging.handlers.MemoryHandler, memory_handler).flush()