log to syslog and eventually to stderr
This commit is contained in:
parent
6af35222db
commit
0763276cca
158
cd06/script.py
158
cd06/script.py
|
@ -1,6 +1,9 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
import glob
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
|
@ -8,13 +11,6 @@ from cd06.wcs_api import WcsApi
|
|||
|
||||
from config import WCS_URL, WCS_ORIG, WCS_APIKEY, USER_UUID
|
||||
|
||||
api = WcsApi(WCS_URL,
|
||||
WCS_ORIG,
|
||||
WCS_APIKEY,
|
||||
name_id=USER_UUID)
|
||||
|
||||
target_dir = sys.argv[1]
|
||||
|
||||
FORMDEFS = {
|
||||
'premiere-demande-d-apa-a-domicile': {
|
||||
'DIRECTORY': 'premiere-demande-apa',
|
||||
|
@ -59,62 +55,106 @@ FORMDEFS = {
|
|||
},
|
||||
}
|
||||
|
||||
print('Moving files into', target_dir)
|
||||
logger = logging.getLogger('cd06.cron.apa')
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
|
||||
tmp_dir = os.path.join(target_dir, 'tmp')
|
||||
if not os.path.exists(tmp_dir):
|
||||
os.makedirs(tmp_dir)
|
||||
syslog_formatter = logging.Formatter('%(levelname)s - %(name)s - %(message)s')
|
||||
syslog_handler = logging.handlers.SysLogHandler(address='/dev/log')
|
||||
syslog_handler.setLevel(logging.DEBUG)
|
||||
syslog_handler.setFormatter(syslog_formatter)
|
||||
|
||||
for formdef in api.formdefs:
|
||||
if formdef.slug not in FORMDEFS:
|
||||
continue
|
||||
print(formdef.title)
|
||||
FORMDEF = FORMDEFS[formdef.slug]
|
||||
MAPPINGS = FORMDEF['MAPPINGS']
|
||||
DIRECTORY = FORMDEF['DIRECTORY']
|
||||
ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS']
|
||||
formdef_dir = os.path.join(target_dir, DIRECTORY)
|
||||
if not os.path.exists(formdef_dir):
|
||||
os.makedirs(formdef_dir)
|
||||
console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||
console_handler = logging.StreamHandler(sys.stderr)
|
||||
console_handler.setLevel(logging.DEBUG)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
|
||||
known_ids = set()
|
||||
statuses = {s.id: s.name for s in formdef.schema.workflow.statuses}
|
||||
memory_handler = logging.handlers.MemoryHandler(
|
||||
capacity=1024 * 100,
|
||||
flushLevel=logging.ERROR,
|
||||
target=console_handler,
|
||||
)
|
||||
logger.addHandler(memory_handler)
|
||||
logger.addHandler(syslog_handler)
|
||||
|
||||
for formdata in formdef.datas:
|
||||
form_id = str(formdata.id).rsplit('/')[-1]
|
||||
known_ids.add(form_id)
|
||||
formdata_dir = os.path.join(formdef_dir, form_id)
|
||||
formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id)
|
||||
|
||||
if os.path.exists(formdata_dir):
|
||||
status_name = statuses[formdata.evolution[-1].status]
|
||||
def do():
|
||||
api = WcsApi(WCS_URL,
|
||||
WCS_ORIG,
|
||||
WCS_APIKEY,
|
||||
name_id=USER_UUID)
|
||||
|
||||
if status_name == ANONYMISATION_STATUS:
|
||||
print('Anonymisation de ', form_id)
|
||||
for name in os.listdir(formdata_dir):
|
||||
os.unlink(os.path.join(formdata_dir, name))
|
||||
target_dir = sys.argv[1]
|
||||
|
||||
tmp_dir = os.path.join(target_dir, 'tmp')
|
||||
if not os.path.exists(tmp_dir):
|
||||
logger.info('creating %s', tmp_dir)
|
||||
os.makedirs(tmp_dir)
|
||||
|
||||
for formdef in api.formdefs:
|
||||
if formdef.slug not in FORMDEFS:
|
||||
continue
|
||||
try:
|
||||
shutil.rmtree(formdata_tmp_dir, True)
|
||||
os.makedirs(formdata_tmp_dir)
|
||||
print(' Form', form_id)
|
||||
for name, key, filename in MAPPINGS:
|
||||
value = formdata[key]
|
||||
if not value:
|
||||
continue
|
||||
print(' -', name, ':', value.filename)
|
||||
prefix, suffix = os.path.splitext(filename)
|
||||
new_filename = prefix + os.path.splitext(value.filename)[1]
|
||||
attachment_path = os.path.join(formdata_tmp_dir, new_filename)
|
||||
print('Putting in', attachment_path)
|
||||
with open(attachment_path, 'wb') as f:
|
||||
f.write(value.content)
|
||||
os.rename(formdata_tmp_dir, formdata_dir)
|
||||
finally:
|
||||
shutil.rmtree(formdata_tmp_dir, True)
|
||||
logger.debug('handling formdef %s', formdef.slug)
|
||||
FORMDEF = FORMDEFS[formdef.slug]
|
||||
MAPPINGS = FORMDEF['MAPPINGS']
|
||||
DIRECTORY = FORMDEF['DIRECTORY']
|
||||
ANONYMISATION_STATUS = FORMDEF['ANONYMISATION_STATUS']
|
||||
formdef_dir = os.path.join(target_dir, DIRECTORY)
|
||||
if not os.path.exists(formdef_dir):
|
||||
logger.info('creating %s', formdef_dir)
|
||||
os.makedirs(formdef_dir)
|
||||
|
||||
existing_ids = set(os.listdir(formdef_dir))
|
||||
for old_id in (existing_ids - known_ids):
|
||||
path_to_delete = os.path.join(formdef_dir, old_id)
|
||||
print('Would delete', path_to_delete)
|
||||
print()
|
||||
known_ids = set()
|
||||
statuses = {s.id: s.name for s in formdef.schema.workflow.statuses}
|
||||
|
||||
for formdata in formdef.datas:
|
||||
form_id = str(formdata.id).rsplit('/')[-1]
|
||||
logging.debug('handling formdata %s', form_id)
|
||||
known_ids.add(form_id)
|
||||
formdata_dir = os.path.join(formdef_dir, form_id)
|
||||
formdata_tmp_dir = os.path.join(tmp_dir, DIRECTORY, form_id)
|
||||
|
||||
if os.path.exists(formdata_dir):
|
||||
status_name = statuses[formdata.evolution[-1].status]
|
||||
|
||||
if status_name == ANONYMISATION_STATUS:
|
||||
if os.listdir(formdata_dir):
|
||||
logger.info('anonymizing %s', formdata_dir)
|
||||
for name in os.listdir(formdata_dir):
|
||||
os.unlink(os.path.join(formdata_dir, name))
|
||||
else:
|
||||
logger.debug('already anonymized %s', formdata_dir)
|
||||
else:
|
||||
logger.info('%s: copying formdata %s', formdef.slug, form_id)
|
||||
try:
|
||||
shutil.rmtree(formdata_tmp_dir, True)
|
||||
os.makedirs(formdata_tmp_dir)
|
||||
for name, key, filename in MAPPINGS:
|
||||
value = formdata[key]
|
||||
if not value:
|
||||
logger.info('no file for "%s"', name)
|
||||
continue
|
||||
prefix, suffix = os.path.splitext(filename)
|
||||
new_filename = prefix + os.path.splitext(value.filename)[1]
|
||||
attachment_path = os.path.join(formdata_tmp_dir, new_filename)
|
||||
logger.info('putting %s into %s for "%s"', value.filname, attachment_path, name)
|
||||
with open(attachment_path, 'wb') as f:
|
||||
f.write(value.content)
|
||||
os.rename(formdata_tmp_dir, formdata_dir)
|
||||
finally:
|
||||
shutil.rmtree(formdata_tmp_dir, True)
|
||||
|
||||
existing_ids = set(os.listdir(formdef_dir))
|
||||
for old_id in (existing_ids - known_ids):
|
||||
path_to_delete = os.path.join(formdef_dir, old_id)
|
||||
logger.delete('deleting %s', path_to_delete)
|
||||
shutil.rmtree(path_to_delete)
|
||||
|
||||
logger.info('start sending APA requests')
|
||||
try:
|
||||
do()
|
||||
except Exception:
|
||||
logger.exception('unexpected failure')
|
||||
logger.info('end sending APA requests')
|
||||
# clear buffered log records, to prevent flush on close
|
||||
super(logging.handlers.MemoryHandler, memory_handler).flush()
|
||||
|
|
Loading…
Reference in New Issue