From 6ba761eca6dbe0498f81ba9efaeb0783cdc149a6 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 17 Jun 2022 11:19:25 +0200 Subject: [PATCH] =?UTF-8?q?chorus:=20ajout=20d'une=20t=C3=A2che=20cron=20p?= =?UTF-8?q?our=20mise=20=C3=A0=20jour=20des=20structures=20Chorus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eo_gestion/chorus/api.py | 2 +- eo_gestion/chorus/utils.py | 148 +++++++++++++++++++++++++++++++++++++ eo_gestion/mule.py | 11 ++- test-requirements.txt | 1 + 4 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 eo_gestion/chorus/utils.py diff --git a/eo_gestion/chorus/api.py b/eo_gestion/chorus/api.py index b3a0753..6f12766 100644 --- a/eo_gestion/chorus/api.py +++ b/eo_gestion/chorus/api.py @@ -176,7 +176,7 @@ class ChorusAPI: def _parse_structure(cls, structure): d = {} for node in structure: - if not len(node): + if len(node) == 0: value = node.text elif len({sub.tag for sub in node}) != 1: value = cls._parse_structure(node) diff --git a/eo_gestion/chorus/utils.py b/eo_gestion/chorus/utils.py new file mode 100644 index 0000000..26fd958 --- /dev/null +++ b/eo_gestion/chorus/utils.py @@ -0,0 +1,148 @@ +# barbacompta - accounting for dummies +# Copyright (C) 2010-2022 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import io +import itertools +import logging +import time + +from django.conf import settings +from django.db import transaction + +from eo_gestion.chorus.api import ChorusAPI +from eo_gestion.chorus.models import Structure +from eo_gestion.eo_facture.models import Client + +logger = logging.getLogger('barbacompta') + + +def get_api(): + return ChorusAPI( + platform=settings.CHORUS_PLATFORM, + piste_client_id=settings.CHORUS_PISTE_CLIENT_ID, + piste_client_secret=settings.CHORUS_PISTE_CLIENT_SECRET, + chorus_tech_username=settings.CHORUS_TECH_USER_LOGIN, + chorus_tech_password=settings.CHORUS_TECH_USER_PASSWORD, + ) + + +def push_to_chorus(pdf: bytes, name: str): + '''Wrapper around eo_gestion.chorus.ChorusAPI.deposer_flux_facturx''' + api = get_api() + data = {} + try: + data = api.deposer_flux_facturx(pdf=pdf, name=name) + except api.Error as e: + data['http.requests_error'] = str(e) + if e.response is not None: + data['http.response.status-code'] = e.response.status_code + data['http.response.headers'] = dict(e.response.headers) + data['http.response.body'] = repr(e.response.content[:1024]) + return data + + +def _grouper(it, size): + '''Split iterator in equal size chunk of `size` elements.''' + it = iter(it) + return iter(lambda: tuple(itertools.islice(it, size)), ()) + + +def _convert_annuaire_to_models(api, content): + with io.BytesIO(content) as fd: + for structure in api.Annuaire.parse(fd): + if not structure.structure_active: + continue + if structure.gestion_service: + for service in structure.services: + if not service.service_actif: + continue + yield Structure( + name=structure.raison_sociale[:80], + siret=structure.identifiant, + service_code=service.code, + service_name=service.nom[:80] or service.code, + email=service.adresse_postale.get('courriel') + or structure.adresse_postale.get('courriel'), + engagement_obligatoire=service.gestion_egmt or structure.gestion_engagement, + ) + else: + yield Structure( + name=structure.raison_sociale[:80], + siret=structure.identifiant, + email=structure.adresse_postale.get('courriel'), + engagement_obligatoire=structure.gestion_engagement, + ) + + +def _update_structures(): + api = get_api() + content = api.telecharger_annuaire_destinataire() + known_identifiers = set() + structure_fields = ['name', 'siret', 'service_code', 'service_name', 'email', 'engagement_obligatoire'] + count_updated = 0 + count_inserted = 0 + for structures in _grouper(_convert_annuaire_to_models(api, content), 1000): + inserts = [] + updates = [] + identifiers = {structure.full_identifier for structure in structures} + qs = Structure.objects.filter(full_identifier__in=identifiers) + identifier_to_structure = {structure.full_identifier: structure for structure in qs} + for structure in structures: + known_identifiers.add(structure.full_identifier) + if structure.full_identifier in identifier_to_structure: + old_structure = identifier_to_structure[structure.full_identifier] + equals = all( + getattr(structure, key) == getattr(old_structure, key) for key in structure_fields + ) + if equals: + continue + structure.id = old_structure.id + updates.append(structure) + else: + inserts.append(structure) + with transaction.atomic(): + count_updated += len(updates) + count_inserted += len(inserts) + if inserts: + Structure.objects.bulk_create(inserts) + if updates: + Structure.objects.bulk_update(updates, fields=structure_fields) + # delete obsolete structures + qs = Structure.objects.exclude(full_identifier__in=known_identifiers) + count_deleted, _ = qs.delete() + + # update Client objects + Client.update_siret_and_service_code() + return {'deleted': count_deleted, 'updated': count_updated, 'created': count_inserted} + + +def update_structures(): + start = time.time() + logger.debug('chorus: update structures started.') + try: + result = _update_structures() + except Exception: + logger.exception('chorus: update structures finished with error (%.1f seconds).', time.time() - start) + else: + report = [] + for key, value in result.items(): + if value: + report.append(f'{value} {key}') + if report: + msg = ', '.join(report) + else: + msg = 'nothing done' + logger.info('chorus: update structures finished (%s, %.1f seconds).', msg, time.time() - start) diff --git a/eo_gestion/mule.py b/eo_gestion/mule.py index ca78515..fad05b7 100644 --- a/eo_gestion/mule.py +++ b/eo_gestion/mule.py @@ -18,11 +18,11 @@ import logging import django -from uwsgidecorators import timer +from uwsgidecorators import cron, timer django.setup() -logger = logging.getLogger('django.server') +logger = logging.getLogger('barbacompta') @timer(300) @@ -39,3 +39,10 @@ def update_cache(num): logger.exception('failed to update cache for %s', func.__name__) else: logger.info('updated cache for %s.%s', func.__module__, func.__name__) + + +@cron(0, 2, -1, -1, -1) +def update_structures(num): + from eo_gestion.chorus.utils import update_structures + + update_structures() diff --git a/test-requirements.txt b/test-requirements.txt index 8ce315d..9b6e006 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,3 +7,4 @@ httmock django-webtest uwsgidecorators pyquery +click