log to syslog and eventually to stderr

This commit is contained in:
Benjamin Dauvergne 2019-08-19 18:38:18 +02:00
parent 6af35222db
commit 0763276cca
1 changed files with 99 additions and 59 deletions

View File

@ -1,6 +1,9 @@
# -*- coding: utf-8 -*-
import glob
from __future__ import unicode_literals
import logging
import logging.handlers
import sys
import os
import shutil
@ -8,13 +11,6 @@ from cd06.wcs_api import WcsApi
from config import WCS_URL, WCS_ORIG, WCS_APIKEY, USER_UUID
api = WcsApi(WCS_URL,
WCS_ORIG,
WCS_APIKEY,
name_id=USER_UUID)
target_dir = sys.argv[1]
FORMDEFS = {
'premiere-demande-d-apa-a-domicile': {
'DIRECTORY': 'premiere-demande-apa',
@ -59,62 +55,106 @@ FORMDEFS = {
},
}
print('Moving files into', target_dir)
logger = logging.getLogger('cd06.cron.apa')
logger.setLevel(logging.INFO)
logger.propagate = False
tmp_dir = os.path.join(target_dir, 'tmp')
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
syslog_formatter = logging.Formatter('%(levelname)s - %(name)s - %(message)s')
syslog_handler = logging.handlers.SysLogHandler(address='/dev/log')
syslog_handler.setLevel(logging.DEBUG)
syslog_handler.setFormatter(syslog_formatter)
for formdef in api.formdefs:
if formdef.slug not in FORMDEFS:
continue
print(formdef.title)
FORMDEF = FORMDEFS[formdef.slug]
MAPPINGS = FORMDEF['MAPPINGS']
DIRECTORY = FORMDEF['DIRECTORY']
ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS']
formdef_dir = os.path.join(target_dir, DIRECTORY)
if not os.path.exists(formdef_dir):
os.makedirs(formdef_dir)
console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
console_handler = logging.StreamHandler(sys.stderr)
console_handler.setLevel(logging.DEBUG)
console_handler.setFormatter(console_formatter)
known_ids = set()
statuses = {s.id: s.name for s in formdef.schema.workflow.statuses}
memory_handler = logging.handlers.MemoryHandler(
capacity=1024 * 100,
flushLevel=logging.ERROR,
target=console_handler,
)
logger.addHandler(memory_handler)
logger.addHandler(syslog_handler)
for formdata in formdef.datas:
form_id = str(formdata.id).rsplit('/')[-1]
known_ids.add(form_id)
formdata_dir = os.path.join(formdef_dir, form_id)
formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id)
if os.path.exists(formdata_dir):
status_name = statuses[formdata.evolution[-1].status]
def do():
api = WcsApi(WCS_URL,
WCS_ORIG,
WCS_APIKEY,
name_id=USER_UUID)
if status_name == ANONYMISATION_STATUS:
print('Anonymisation de ', form_id)
for name in os.listdir(formdata_dir):
os.unlink(os.path.join(formdata_dir, name))
target_dir = sys.argv[1]
tmp_dir = os.path.join(target_dir, 'tmp')
if not os.path.exists(tmp_dir):
logger.info('creating %s', tmp_dir)
os.makedirs(tmp_dir)
for formdef in api.formdefs:
if formdef.slug not in FORMDEFS:
continue
try:
shutil.rmtree(formdata_tmp_dir, True)
os.makedirs(formdata_tmp_dir)
print(' Form', form_id)
for name, key, filename in MAPPINGS:
value = formdata[key]
if not value:
continue
print(' -', name, ':', value.filename)
prefix, suffix = os.path.splitext(filename)
new_filename = prefix + os.path.splitext(value.filename)[1]
attachment_path = os.path.join(formdata_tmp_dir, new_filename)
print('Putting in', attachment_path)
with open(attachment_path, 'wb') as f:
f.write(value.content)
os.rename(formdata_tmp_dir, formdata_dir)
finally:
shutil.rmtree(formdata_tmp_dir, True)
logger.debug('handling formdef %s', formdef.slug)
FORMDEF = FORMDEFS[formdef.slug]
MAPPINGS = FORMDEF['MAPPINGS']
DIRECTORY = FORMDEF['DIRECTORY']
ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS']
formdef_dir = os.path.join(target_dir, DIRECTORY)
if not os.path.exists(formdef_dir):
logger.info('creating %s', formdef_dir)
os.makedirs(formdef_dir)
existing_ids = set(os.listdir(formdef_dir))
for old_id in (existing_ids - known_ids):
path_to_delete = os.path.join(formdef_dir, old_id)
print('Would delete', path_to_delete)
print()
known_ids = set()
statuses = {s.id: s.name for s in formdef.schema.workflow.statuses}
for formdata in formdef.datas:
form_id = str(formdata.id).rsplit('/')[-1]
logging.debug('handling formdata %s', form_id)
known_ids.add(form_id)
formdata_dir = os.path.join(formdef_dir, form_id)
formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id)
if os.path.exists(formdata_dir):
status_name = statuses[formdata.evolution[-1].status]
if status_name == ANONYMISATION_STATUS:
if os.listdir(formdata_dir):
logger.info('anonymizing %s', formdata_dir)
for name in os.listdir(formdata_dir):
os.unlink(os.path.join(formdata_dir, name))
else:
logger.debug('already anonymized %s', formdata_dir)
else:
logger.info('%s: copying formdata %s', formdef.slug, form_id)
try:
shutil.rmtree(formdata_tmp_dir, True)
os.makedirs(formdata_tmp_dir)
for name, key, filename in MAPPINGS:
value = formdata[key]
if not value:
logger.info('no file for "%s"', name)
continue
prefix, suffix = os.path.splitext(filename)
new_filename = prefix + os.path.splitext(value.filename)[1]
attachment_path = os.path.join(formdata_tmp_dir, new_filename)
logger.info('putting %s into %s for "%s"', value.filname, attachment_path, name)
with open(attachment_path, 'wb') as f:
f.write(value.content)
os.rename(formdata_tmp_dir, formdata_dir)
finally:
shutil.rmtree(formdata_tmp_dir, True)
existing_ids = set(os.listdir(formdef_dir))
for old_id in (existing_ids - known_ids):
path_to_delete = os.path.join(formdef_dir, old_id)
logger.delete('deleting %s', path_to_delete)
shutil.rmtree(path_to_delete)
logger.info('start sending APA requests')
try:
do()
except Exception:
logger.exception('unexpected failure')
logger.info('end sending APA requests')
# clear buffered log records, to prevent flush on close
super(logging.handlers.MemoryHandler, memory_handler).flush()