family: add support for arpege opus export (#50926)

passerelle/apps/family/loaders/opus_fondettes.py

@@ -0,0 +1,90 @@
+# Passerelle - uniform access to data and services
+# Copyright (C) 2016-2021  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import csv
+import datetime
+from decimal import Decimal
+import json
+import os
+import sys
+import zipfile
+
+from django.core.exceptions import ValidationError
+from django.utils.encoding import force_text
+from django.utils import six
+from django.utils.translation import ugettext_lazy as _
+
+from ..models import Invoice
+
+
+def u(s):
+    return force_text(s, 'iso-8859-15')
+
+
+class Loader(object):
+    def __init__(self, connector):
+        self.connector = connector
+
+    def clean(self, archive):
+        if 'publipostage.csv' not in archive.namelist():
+            raise ValidationError(_('Missing publipostage.csv file in zip.'))
+
+    def load(self, archive):
+        archive_files = archive.namelist()
+        fd = archive.open('publipostage.csv')
+        if six.PY3:
+            import io
+
+            fd = io.TextIOWrapper(fd, 'iso-8859-15')
+        csvfile = six.StringIO(fd.read())
+        csvreader = csv.reader(csvfile, delimiter=';')
+        first_row = next(csvreader)
+        csvfile.seek(0)
+        csvreader = csv.DictReader(csvfile, delimiter=';', fieldnames=first_row)
+        next(csvreader)
+        for row in csvreader:
+            invoice = {}
+            invoice['total_amount'] = row['MNT_FACTURE_FAC'].replace(',', '.')
+            invoice['amount'] = str(
+                Decimal(invoice['total_amount']) - Decimal(row['MNT_REGLE_FAC'].replace(',', '.'))
+            )
+            invoice['paid'] = bool(Decimal(invoice['amount']) == 0)
+            invoice['issue_date'] = datetime.datetime.strptime(
+                row['DAT_GENERATION_FAC'], '%d/%m/%Y'
+            ).strftime('%Y-%m-%d')
+            invoice['pay_limit_date'] = datetime.datetime.strptime(
+                row['DAT_LIMITEPAIE_FAC'], '%d/%m/%Y'
+            ).strftime('%Y-%m-%d')
+            invoice['online_payment'] = True
+            invoice['no_online_payment_reason'] = None
+            if not invoice['paid']:
+                if row['MNT_REGLE_FAC'] != '0,00':
+                    # it has been partially paid, we don't allow the rest to be
+                    # paid.
+                    invoice['online_payment'] = False
+                elif row['DAT_PRELEVEMENT_FAC'] or row['LIB_PRELEVAUTO_LST'] == 'Oui':
+                    invoice['online_payment'] = False
+                    invoice['no_online_payment_reason'] = 'autobilling'
+
+            obj, created = Invoice.objects.update_or_create(
+                resource=self.connector, external_id=row['ID_FAC'], defaults=invoice
+            )
+            invoice_filename = '%s_%s.pdf' % (
+                datetime.datetime.strptime(row['DAT_DEBUT_PGE'], '%d/%m/%Y').strftime('%Y-%m'),
+                row['ID_FAC'],
+            )
+            if invoice_filename in archive_files:
+                obj.write_pdf(archive.read(invoice_filename))

passerelle/apps/family/migrations/0011_auto_20200803_2326.py

@@ -38,6 +38,7 @@ class Migration(migrations.Migration):
                 choices=[
                     ('native', 'Native'),
                     ('concerto_fondettes', 'Concerto extract from Fondettes'),
+                    ('opus_fondettes', 'Opus extract from Fondettes'),
                     ('concerto_orleans', 'Concerto extract from Orléans'),
                 ],
                 default='native',

passerelle/apps/family/models.py

@@ -156,7 +156,8 @@ class GenericFamily(BaseResource):
         max_length=40,
         choices=(
             ('native', _('Native')),
-            ('concerto_fondettes', _('Concerto extract from Fondettes')),
+            ('concerto_fondettes', _('Concerto extract from Fondettes (legacy)')),
+            ('opus_fondettes', _('Opus extract from Fondettes')),
             ('concerto_orleans', _(u'Concerto extract from Orléans')),
             ('egee_thonon', _(u'Egee Invoices from Thonon Agglomération')),
         ),

tests/test_family.py

@@ -335,6 +335,22 @@ def test_fondettes_concerto_loader():
     assert Invoice.objects.filter(online_payment=False).count() == 2
 
 
+def test_fondettes_opus_loader():
+    Invoice.objects.all().delete()
+    filepath = os.path.join(os.path.dirname(__file__), 'data', 'family_data_opus_fondettes.zip')
+    with open(filepath, 'rb') as fd:
+        resource = GenericFamily.objects.create(
+            title='test fondettes',
+            slug='test-fondettes',
+            archive=File(fd, 'family_data_opus_fondettes.zip'),
+            file_format='opus_fondettes',
+        )
+    assert Invoice.objects.filter(resource=resource).count() == 630
+    assert len([x for x in Invoice.objects.filter(resource=resource) if x.has_pdf]) == 4
+    assert Invoice.objects.filter(paid=True).count() == 314
+    assert Invoice.objects.filter(paid=False).count() == 316
+
+
 def test_archive_validation():
     filepath = os.path.join(os.path.dirname(__file__), 'data', 'iparapheur.wsdl')
     with open(filepath) as fd: