toulouse_maelis: restart paid notification job every 5 minutes on failure (#87030)

passerelle/contrib/toulouse_maelis/models.py

@@ -40,12 +40,12 @@ from zeep.helpers import serialize_object
 from zeep.wsse.username import UsernameToken
 
 from passerelle.apps.base_adresse.models import CityModel
-from passerelle.base.models import BaseResource, HTTPResource
+from passerelle.base.models import BaseResource, HTTPResource, SkipJob
 from passerelle.base.signature import sign_url
 from passerelle.utils.api import endpoint
 from passerelle.utils.conversion import normalize, simplify
 from passerelle.utils.jsonresponse import APIError
-from passerelle.utils.soap import SOAPFault, SOAPServiceUnreachable
+from passerelle.utils.soap import SOAPError, SOAPFault, SOAPServiceUnreachable
 from passerelle.utils.templates import render_to_string
 from passerelle.utils.validation import is_number
 from passerelle.utils.wcs import WcsApi, WcsApiError
@@ -4447,7 +4447,11 @@ class ToulouseMaelis(BaseResource, HTTPResource):
             invoice = self.invoice_set.get(regie_id=regie_id, invoice_id=invoice_id)
         except Invoice.DoesNotExist:
             return
-        invoice.notify()
+        if not invoice.notify() and (now() - invoice.lingo_notification_date) < datetime.timedelta(
+            minutes=45
+        ):
+            # retry in 5 minutes...
+            raise SkipJob(5 * 60)
 
     def trigger_subscription_job(self, pk):
         try:
@@ -4639,11 +4643,14 @@ class Invoice(models.Model):
                 numInvoices=[obj.invoice_id],
                 numPerson=obj.maelis_data['payer']['num'],
             )
-        except SOAPServiceUnreachable:
-            return False
-        except SOAPFault as e:
-            self.resource.logger.error('fails to notify paid invoice %s: %s', repr(obj), e)
+        except SOAPServiceUnreachable as e:
+            self.resource.logger.warning(
+                'fails to notify paid invoice %s (%s), will retry later.', repr(obj), e
+            )
             return False
+        except SOAPError as e:
+            self.resource.logger.error('fails to notify paid invoice %s (%s), stopping.', repr(obj), e)
+            result = {'error': str(e)}
 
         obj.maelis_notification_date = now()
         obj.maelis_notification_data = result

tests/test_toulouse_maelis.py

@@ -11706,6 +11706,44 @@ def test_pay_invoice_job(invoice_service, con, app, freezer):
     assert job.update_timestamp > job.creation_timestamp
 
 
+def test_pay_invoice_job_error(invoice_service, con, app, freezer, caplog):
+    caplog.set_level('WARNING')
+
+    invoice_service.add_soap_response('readInvoices', get_xml_file('R_read_invoices.xml'))
+    invoice_service.add_soap_response('payInvoices', ConnectionError('boom!'))
+    invoice_service.add_soap_response('payInvoices', get_xml_file('R_pay_invoices.xml'))
+
+    url = get_endpoint('regie/102/invoice/1312-30/pay/')
+    freezer.move_to('2023-03-03 18:39:00')
+    app.post_json(
+        url,
+        params={
+            'transaction_date': '2023-03-03T18:38:00',
+            'transaction_id': 'xxx',
+        },
+    )
+    job = Job.objects.get()
+    assert job.status == 'registered'
+    assert caplog.messages == []
+
+    freezer.move_to('2023-03-03 18:40:00')
+    con.jobs()
+    job.refresh_from_db()
+    assert job.status == 'registered'
+    assert len(caplog.messages) == 1
+    assert 'fails to notify' in caplog.text
+
+    freezer.tick(6 * 60)
+    con.jobs()
+    job.refresh_from_db()
+    assert job.status == 'completed'
+
+    assert len(caplog.messages) == 1
+
+    invoice = con.invoice_set.get(regie_id=102, invoice_id=30)
+    assert invoice.status() == 'notified'
+
+
 def test_pay_invoice_cron(invoice_service, con, app, freezer):
     def request_check(request):
         assert dict(serialize_object(request)) == {
@@ -11756,6 +11794,8 @@ def test_pay_invoice_cron(invoice_service, con, app, freezer):
 
 
 def test_pay_invoice_cron_maelis_error(invoice_service, con, app, freezer, caplog):
+    caplog.set_level('WARNING')
+
     invoice_service.add_soap_response('readInvoices', get_xml_file('R_read_2_invoices_to_pay.xml'))
     invoice_service.add_soap_response('payInvoices', get_xml_file('R_pay_invoices_error.xml'))
     invoice_service.add_soap_response('payInvoices', get_xml_file('R_pay_invoices.xml'))
@@ -11775,15 +11815,13 @@ def test_pay_invoice_cron_maelis_error(invoice_service, con, app, freezer, caplo
     assert con.invoice_set.get(regie_id=102, invoice_id=30).status() == 'paid'
     assert con.invoice_set.get(regie_id=102, invoice_id=8).status() == 'paid'
 
+    caplog.clear()
     # run hourly cron that fails on first invoice, but continue notifying the second one
     freezer.move_to('2023-03-03 18:55:00')
     con.notify_invoices_paid()
-    assert textwrap.wrap(caplog.records[-2].message, 80) == [
-        'fails to notify paid invoice <Invoice "102/30">: SOAP service at',
-        'https://example.org/InvoiceService?wsdl returned an error "E500 : Le montant est',
-        'requis et ne peut être vide"',
-    ]
-    assert con.invoice_set.get(regie_id=102, invoice_id=30).status() == 'paid'
+    assert 'fails to notify' in caplog.text
+    assert 'stopping' in caplog.text
+    assert con.invoice_set.get(regie_id=102, invoice_id=30).status() == 'notified'
     assert con.invoice_set.get(regie_id=102, invoice_id=8).status() == 'notified'