chorus: ajout d'une tâche cron pour mise à jour des structures Chorus

eo_gestion/chorus/api.py

@@ -176,7 +176,7 @@ class ChorusAPI:
         def _parse_structure(cls, structure):
             d = {}
             for node in structure:
-                if not len(node):
+                if len(node) == 0:
                     value = node.text
                 elif len({sub.tag for sub in node}) != 1:
                     value = cls._parse_structure(node)

eo_gestion/chorus/utils.py

@@ -0,0 +1,148 @@
+# barbacompta - accounting for dummies
+# Copyright (C) 2010-2022 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import io
+import itertools
+import logging
+import time
+
+from django.conf import settings
+from django.db import transaction
+
+from eo_gestion.chorus.api import ChorusAPI
+from eo_gestion.chorus.models import Structure
+from eo_gestion.eo_facture.models import Client
+
+logger = logging.getLogger('barbacompta')
+
+
+def get_api():
+    return ChorusAPI(
+        platform=settings.CHORUS_PLATFORM,
+        piste_client_id=settings.CHORUS_PISTE_CLIENT_ID,
+        piste_client_secret=settings.CHORUS_PISTE_CLIENT_SECRET,
+        chorus_tech_username=settings.CHORUS_TECH_USER_LOGIN,
+        chorus_tech_password=settings.CHORUS_TECH_USER_PASSWORD,
+    )
+
+
+def push_to_chorus(pdf: bytes, name: str):
+    '''Wrapper around eo_gestion.chorus.ChorusAPI.deposer_flux_facturx'''
+    api = get_api()
+    data = {}
+    try:
+        data = api.deposer_flux_facturx(pdf=pdf, name=name)
+    except api.Error as e:
+        data['http.requests_error'] = str(e)
+        if e.response is not None:
+            data['http.response.status-code'] = e.response.status_code
+            data['http.response.headers'] = dict(e.response.headers)
+            data['http.response.body'] = repr(e.response.content[:1024])
+    return data
+
+
+def _grouper(it, size):
+    '''Split iterator in equal size chunk of `size` elements.'''
+    it = iter(it)
+    return iter(lambda: tuple(itertools.islice(it, size)), ())
+
+
+def _convert_annuaire_to_models(api, content):
+    with io.BytesIO(content) as fd:
+        for structure in api.Annuaire.parse(fd):
+            if not structure.structure_active:
+                continue
+            if structure.gestion_service:
+                for service in structure.services:
+                    if not service.service_actif:
+                        continue
+                    yield Structure(
+                        name=structure.raison_sociale[:80],
+                        siret=structure.identifiant,
+                        service_code=service.code,
+                        service_name=service.nom[:80] or service.code,
+                        email=service.adresse_postale.get('courriel')
+                        or structure.adresse_postale.get('courriel'),
+                        engagement_obligatoire=service.gestion_egmt or structure.gestion_engagement,
+                    )
+            else:
+                yield Structure(
+                    name=structure.raison_sociale[:80],
+                    siret=structure.identifiant,
+                    email=structure.adresse_postale.get('courriel'),
+                    engagement_obligatoire=structure.gestion_engagement,
+                )
+
+
+def _update_structures():
+    api = get_api()
+    content = api.telecharger_annuaire_destinataire()
+    known_identifiers = set()
+    structure_fields = ['name', 'siret', 'service_code', 'service_name', 'email', 'engagement_obligatoire']
+    count_updated = 0
+    count_inserted = 0
+    for structures in _grouper(_convert_annuaire_to_models(api, content), 1000):
+        inserts = []
+        updates = []
+        identifiers = {structure.full_identifier for structure in structures}
+        qs = Structure.objects.filter(full_identifier__in=identifiers)
+        identifier_to_structure = {structure.full_identifier: structure for structure in qs}
+        for structure in structures:
+            known_identifiers.add(structure.full_identifier)
+            if structure.full_identifier in identifier_to_structure:
+                old_structure = identifier_to_structure[structure.full_identifier]
+                equals = all(
+                    getattr(structure, key) == getattr(old_structure, key) for key in structure_fields
+                )
+                if equals:
+                    continue
+                structure.id = old_structure.id
+                updates.append(structure)
+            else:
+                inserts.append(structure)
+        with transaction.atomic():
+            count_updated += len(updates)
+            count_inserted += len(inserts)
+            if inserts:
+                Structure.objects.bulk_create(inserts)
+            if updates:
+                Structure.objects.bulk_update(updates, fields=structure_fields)
+    # delete obsolete structures
+    qs = Structure.objects.exclude(full_identifier__in=known_identifiers)
+    count_deleted, _ = qs.delete()
+
+    # update Client objects
+    Client.update_siret_and_service_code()
+    return {'deleted': count_deleted, 'updated': count_updated, 'created': count_inserted}
+
+
+def update_structures():
+    start = time.time()
+    logger.debug('chorus: update structures started.')
+    try:
+        result = _update_structures()
+    except Exception:
+        logger.exception('chorus: update structures finished with error (%.1f seconds).', time.time() - start)
+    else:
+        report = []
+        for key, value in result.items():
+            if value:
+                report.append(f'{value} {key}')
+        if report:
+            msg = ', '.join(report)
+        else:
+            msg = 'nothing done'
+        logger.info('chorus: update structures finished (%s, %.1f seconds).', msg, time.time() - start)

eo_gestion/mule.py

@@ -18,11 +18,11 @@
 import logging
 
 import django
-from uwsgidecorators import timer
+from uwsgidecorators import cron, timer
 
 django.setup()
 
-logger = logging.getLogger('django.server')
+logger = logging.getLogger('barbacompta')
 
 
 @timer(300)
@@ -39,3 +39,10 @@ def update_cache(num):
             logger.exception('failed to update cache for %s', func.__name__)
         else:
             logger.info('updated cache for %s.%s', func.__module__, func.__name__)
+
+
+@cron(0, 2, -1, -1, -1)
+def update_structures(num):
+    from eo_gestion.chorus.utils import update_structures
+
+    update_structures()

test-requirements.txt

@@ -7,3 +7,4 @@ httmock
 django-webtest
 uwsgidecorators
 pyquery
+click