toulouse-maelis: put global catalog in cache (#82379)

passerelle/contrib/toulouse_maelis/models.py

@@ -156,6 +156,39 @@ class ToulouseMaelis(BaseResource, HTTPResource):
             # delete extraneous items
             self.referential.filter(referential_name=referential_name, updated__lt=last_update).delete()
 
+    def update_catalog_referential(self):
+        last_update = now()
+        ref_date = last_update.date()
+        try:
+            data = self.call(
+                'Activity',
+                'readActivityList',
+                # pass schoolyear as '1970', it's not actually used and activities will be
+                # returned according to dateStartCalend/dateEndCalend.
+                schoolyear='1970',
+                dateStartCalend=(ref_date - datetime.timedelta(days=365)).isoformat(),
+                dateEndCalend=(ref_date + datetime.timedelta(days=365)).isoformat(),
+            )
+        except Exception as e:
+            raise UpdateError('Service indisponible : %s' % str(e))
+
+        for item in data or []:
+            id_key = item['activityPortail']['idAct']
+            text = item['activityPortail'].get('libelle2') or item['activityPortail']['libelle'] or ''
+            text = text.strip()
+            self.referential.update_or_create(
+                resource_id=self.id,
+                referential_name='Activity',
+                item_id=id_key,
+                defaults={
+                    'item_text': text,
+                    'item_data': dict({'id': id_key, 'text': text}, **item),
+                    'updated': last_update,
+                },
+            )
+        # delete extraneous items
+        self.referential.filter(referential_name='Activity', updated__lt=last_update).delete()
+
     def get_referential_data(self, service_name, referential_name):
         try:
             return self.call(service_name, 'read' + referential_name + 'List')
@@ -223,6 +256,7 @@ class ToulouseMaelis(BaseResource, HTTPResource):
             if referential_name in ['Direct', 'Service']:
                 id_key, text_key = 'id', 'lib1'
             self.update_referential(referential_name, data, id_key, text_key)
+        self.update_catalog_referential()
 
     def update_ape_referentials(self):
         indicators = self.call('Ape', 'readApeIndicatorList')
@@ -258,11 +292,13 @@ class ToulouseMaelis(BaseResource, HTTPResource):
         super().daily()
         self.update_referentials()
 
+    def every5min(self):
+        self.update_activity_referentials()
+
     def update_referentials(self):
         try:
             self.update_family_referentials()
             self.update_site_referentials()
-            self.update_activity_referentials()
             self.update_ape_referentials()
             self.update_invoice_referentials()
             # merge zip codes from base adresse into town referential
@@ -2699,7 +2735,6 @@ class ToulouseMaelis(BaseResource, HTTPResource):
         name='read-activity-list',
     )
     def read_activity_list(self, request):
-        ref_date = now().date()
         labels = {
             'service': 'Service',
             'nature': "Nature de l'activité",
@@ -2712,16 +2747,6 @@ class ToulouseMaelis(BaseResource, HTTPResource):
         all_criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()}
         criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()}
 
-        activities = self.call(
-            'Activity',
-            'readActivityList',
-            # pass schoolyear as '1970', it's not actually used and activities will be
-            # returned according to dateStartCalend/dateEndCalend.
-            schoolyear='1970',
-            dateStartCalend=(ref_date - datetime.timedelta(days=365)).isoformat(),
-            dateEndCalend=(ref_date + datetime.timedelta(days=365)).isoformat(),
-        )
-
         def add_criteria(label_key, criteria_key, criteria_value):
             if not criteria_value:
                 return
@@ -2742,15 +2767,11 @@ class ToulouseMaelis(BaseResource, HTTPResource):
                     ]
 
         data = []
-        for activity in activities:
+        for activity in self.get_referential('Activity'):
             activity_type = activity['activityPortail'].get('activityType')
             activity_nature = activity_type.get('natureSpec') if activity_type else None
             if not activity_nature or activity_nature['code'] not in self.get_loisir_nature_codes():
                 continue
-            activity['id'] = activity['activityPortail']['idAct']
-            activity['text'] = (
-                activity['activityPortail']['libelle2'] or activity['activityPortail']['libelle']
-            )
             service_id = activity['activityPortail']['idService']
             service_text = self.get_referential_value('Service', service_id, default=None)
             activity['activityPortail']['idService_text'] = service_text
@@ -2774,9 +2795,13 @@ class ToulouseMaelis(BaseResource, HTTPResource):
                 unit['text'] = unit['libelle']
 
                 criterias['public']['data'] = {}
-                for key, value in utils.get_public_criterias(
-                    datetime.date.today(), unit['birthDateStart'], unit['birthDateEnd']
-                ):
+                start_dob = unit['birthDateStart']
+                end_dob = unit['birthDateEnd']
+                if start_dob:
+                    start_dob = parse_date(start_dob)
+                if end_dob:
+                    end_dob = parse_date(end_dob)
+                for key, value in utils.get_public_criterias(datetime.date.today(), start_dob, end_dob):
                     add_criteria('public', key, value)
 
                 update_criterias_order_field(criterias, ['public'])

tests/test_toulouse_maelis.py

@@ -247,6 +247,9 @@ def django_db_setup(django_db_setup, django_db_blocker):
                     activity_mock.add_soap_response(
                         'readServiceList', get_xml_file('R_read_service_list.xml')
                     )
+                    activity_mock.add_soap_response(
+                        'readActivityList', get_xml_file('R_read_activity_list.xml')
+                    )
                     con.update_activity_referentials()
 
                 with ape_service() as ape_mock:
@@ -556,6 +559,7 @@ def test_update_referential_empty(mocked_get, con):
 def test_cron(db):
     assert Referential.objects.filter(referential_name='Category').count() == 3
     assert sorted(list({x.referential_name for x in Referential.objects.all()})) == [
+        'Activity',
         'ActivityNatureType',
         'ApeIndicator',
         'CSP',
@@ -6305,17 +6309,7 @@ def test_get_public_criterias(start_dob, end_dob, expected):
     assert expected == [x[1] for x in result]
 
 
-def test_read_activity_list(activity_service, con, app, freezer):
-    def request_check(request):
-        assert request.schoolyear == 1970
-        assert request.dateStartCalend == datetime.datetime(2023, 3, 1, 0, 0)
-        assert request.dateEndCalend == datetime.datetime(2025, 2, 28, 0, 0)
-
-    activity_service.add_soap_response(
-        'readActivityList',
-        get_xml_file('R_read_activity_list.xml'),
-        request_check=request_check,
-    )
+def test_read_activity_list(con, app, freezer):
     url = get_endpoint('read-activity-list')
 
     con.loisir_nature_codes = '1,4,L,, S '
@@ -6324,6 +6318,8 @@ def test_read_activity_list(activity_service, con, app, freezer):
     resp = app.get(url)
     assert resp.json['err'] == 0
     assert len(resp.json['data']) == 8
+    activity_text = [x['activity']['text'] for x in resp.json['data']]
+    assert activity_text == sorted(activity_text)
     assert [
         (
             x['id'],
@@ -6332,16 +6328,16 @@ def test_read_activity_list(activity_service, con, app, freezer):
         )
         for x in resp.json['data']
     ] == [
+        ('A10056517594-A10056517595-A10056517597', 'plop', None),
+        ('A10056514645-A10056514650-A10053179757', None, None),
+        ('A10056514645-A10056514648-A10053179876', None, None),
+        ('A10056514645-A10056514649-A10053179757', None, None),
         ('A10051141965-A10051141966-A10053179226', 'A10049329051', 'Sorties'),
         ('A10051141965-A10051141968-A10053179226', 'A10049329051', 'Sorties'),
         ('A10051141965-A10051141970-A10053179226', 'A10049329051', 'Sorties'),
         ('A10051141965-A10051141990-A10053179227', 'A10049329051', 'Sorties'),
-        ('A10056514645-A10056514650-A10053179757', None, None),
-        ('A10056514645-A10056514648-A10053179876', None, None),
-        ('A10056514645-A10056514649-A10053179757', None, None),
-        ('A10056517594-A10056517595-A10056517597', 'plop', None),
     ]
-    item = resp.json['data'][0]
+    item = resp.json['data'][4]
     item['activity'] = 'N/A'
     item['unit'] = 'N/A'
     item['place'] = 'N/A'
@@ -6465,9 +6461,17 @@ def test_read_activity_list(activity_service, con, app, freezer):
 
 
 def test_read_activity_list_no_nature(activity_service, con, app, freezer):
-    activity_service.add_soap_response('readActivityList', get_xml_file('R_read_activity_list_no_nature.xml'))
+    def request_check(request):
+        assert request.schoolyear == 1970
+        assert request.dateStartCalend == datetime.datetime(2023, 3, 1, 0, 0)
+        assert request.dateEndCalend == datetime.datetime(2025, 2, 28, 0, 0)
+
+    activity_service.add_soap_response(
+        'readActivityList', get_xml_file('R_read_activity_list_no_nature.xml'), request_check=request_check
+    )
     url = get_endpoint('read-activity-list')
     freezer.move_to('2024-02-29')
+    con.update_catalog_referential()
     resp = app.get(url)
     assert resp.json['err'] == 0
     assert len(resp.json['data']) == 0