From a7ff9bbc4af957cb0be9bd80d04cd423831c625b Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Wed, 1 Nov 2023 21:21:28 +0100 Subject: [PATCH] toulouse-maelis: put global catalog in cache (#82379) --- passerelle/contrib/toulouse_maelis/models.py | 65 ++++++++++++++------ tests/test_toulouse_maelis.py | 38 +++++++----- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/passerelle/contrib/toulouse_maelis/models.py b/passerelle/contrib/toulouse_maelis/models.py index 06cf9bd2..6750eb7c 100644 --- a/passerelle/contrib/toulouse_maelis/models.py +++ b/passerelle/contrib/toulouse_maelis/models.py @@ -156,6 +156,39 @@ class ToulouseMaelis(BaseResource, HTTPResource): # delete extraneous items self.referential.filter(referential_name=referential_name, updated__lt=last_update).delete() + def update_catalog_referential(self): + last_update = now() + ref_date = last_update.date() + try: + data = self.call( + 'Activity', + 'readActivityList', + # pass schoolyear as '1970', it's not actually used and activities will be + # returned according to dateStartCalend/dateEndCalend. + schoolyear='1970', + dateStartCalend=(ref_date - datetime.timedelta(days=365)).isoformat(), + dateEndCalend=(ref_date + datetime.timedelta(days=365)).isoformat(), + ) + except Exception as e: + raise UpdateError('Service indisponible : %s' % str(e)) + + for item in data or []: + id_key = item['activityPortail']['idAct'] + text = item['activityPortail'].get('libelle2') or item['activityPortail']['libelle'] or '' + text = text.strip() + self.referential.update_or_create( + resource_id=self.id, + referential_name='Activity', + item_id=id_key, + defaults={ + 'item_text': text, + 'item_data': dict({'id': id_key, 'text': text}, **item), + 'updated': last_update, + }, + ) + # delete extraneous items + self.referential.filter(referential_name='Activity', updated__lt=last_update).delete() + def get_referential_data(self, service_name, referential_name): try: return self.call(service_name, 'read' + referential_name + 'List') @@ -223,6 +256,7 @@ class ToulouseMaelis(BaseResource, HTTPResource): if referential_name in ['Direct', 'Service']: id_key, text_key = 'id', 'lib1' self.update_referential(referential_name, data, id_key, text_key) + self.update_catalog_referential() def update_ape_referentials(self): indicators = self.call('Ape', 'readApeIndicatorList') @@ -258,11 +292,13 @@ class ToulouseMaelis(BaseResource, HTTPResource): super().daily() self.update_referentials() + def every5min(self): + self.update_activity_referentials() + def update_referentials(self): try: self.update_family_referentials() self.update_site_referentials() - self.update_activity_referentials() self.update_ape_referentials() self.update_invoice_referentials() # merge zip codes from base adresse into town referential @@ -2699,7 +2735,6 @@ class ToulouseMaelis(BaseResource, HTTPResource): name='read-activity-list', ) def read_activity_list(self, request): - ref_date = now().date() labels = { 'service': 'Service', 'nature': "Nature de l'activité", @@ -2712,16 +2747,6 @@ class ToulouseMaelis(BaseResource, HTTPResource): all_criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()} criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()} - activities = self.call( - 'Activity', - 'readActivityList', - # pass schoolyear as '1970', it's not actually used and activities will be - # returned according to dateStartCalend/dateEndCalend. - schoolyear='1970', - dateStartCalend=(ref_date - datetime.timedelta(days=365)).isoformat(), - dateEndCalend=(ref_date + datetime.timedelta(days=365)).isoformat(), - ) - def add_criteria(label_key, criteria_key, criteria_value): if not criteria_value: return @@ -2742,15 +2767,11 @@ class ToulouseMaelis(BaseResource, HTTPResource): ] data = [] - for activity in activities: + for activity in self.get_referential('Activity'): activity_type = activity['activityPortail'].get('activityType') activity_nature = activity_type.get('natureSpec') if activity_type else None if not activity_nature or activity_nature['code'] not in self.get_loisir_nature_codes(): continue - activity['id'] = activity['activityPortail']['idAct'] - activity['text'] = ( - activity['activityPortail']['libelle2'] or activity['activityPortail']['libelle'] - ) service_id = activity['activityPortail']['idService'] service_text = self.get_referential_value('Service', service_id, default=None) activity['activityPortail']['idService_text'] = service_text @@ -2774,9 +2795,13 @@ class ToulouseMaelis(BaseResource, HTTPResource): unit['text'] = unit['libelle'] criterias['public']['data'] = {} - for key, value in utils.get_public_criterias( - datetime.date.today(), unit['birthDateStart'], unit['birthDateEnd'] - ): + start_dob = unit['birthDateStart'] + end_dob = unit['birthDateEnd'] + if start_dob: + start_dob = parse_date(start_dob) + if end_dob: + end_dob = parse_date(end_dob) + for key, value in utils.get_public_criterias(datetime.date.today(), start_dob, end_dob): add_criteria('public', key, value) update_criterias_order_field(criterias, ['public']) diff --git a/tests/test_toulouse_maelis.py b/tests/test_toulouse_maelis.py index 8a83e3fb..75dd2b12 100644 --- a/tests/test_toulouse_maelis.py +++ b/tests/test_toulouse_maelis.py @@ -247,6 +247,9 @@ def django_db_setup(django_db_setup, django_db_blocker): activity_mock.add_soap_response( 'readServiceList', get_xml_file('R_read_service_list.xml') ) + activity_mock.add_soap_response( + 'readActivityList', get_xml_file('R_read_activity_list.xml') + ) con.update_activity_referentials() with ape_service() as ape_mock: @@ -556,6 +559,7 @@ def test_update_referential_empty(mocked_get, con): def test_cron(db): assert Referential.objects.filter(referential_name='Category').count() == 3 assert sorted(list({x.referential_name for x in Referential.objects.all()})) == [ + 'Activity', 'ActivityNatureType', 'ApeIndicator', 'CSP', @@ -6305,17 +6309,7 @@ def test_get_public_criterias(start_dob, end_dob, expected): assert expected == [x[1] for x in result] -def test_read_activity_list(activity_service, con, app, freezer): - def request_check(request): - assert request.schoolyear == 1970 - assert request.dateStartCalend == datetime.datetime(2023, 3, 1, 0, 0) - assert request.dateEndCalend == datetime.datetime(2025, 2, 28, 0, 0) - - activity_service.add_soap_response( - 'readActivityList', - get_xml_file('R_read_activity_list.xml'), - request_check=request_check, - ) +def test_read_activity_list(con, app, freezer): url = get_endpoint('read-activity-list') con.loisir_nature_codes = '1,4,L,, S ' @@ -6324,6 +6318,8 @@ def test_read_activity_list(activity_service, con, app, freezer): resp = app.get(url) assert resp.json['err'] == 0 assert len(resp.json['data']) == 8 + activity_text = [x['activity']['text'] for x in resp.json['data']] + assert activity_text == sorted(activity_text) assert [ ( x['id'], @@ -6332,16 +6328,16 @@ def test_read_activity_list(activity_service, con, app, freezer): ) for x in resp.json['data'] ] == [ + ('A10056517594-A10056517595-A10056517597', 'plop', None), + ('A10056514645-A10056514650-A10053179757', None, None), + ('A10056514645-A10056514648-A10053179876', None, None), + ('A10056514645-A10056514649-A10053179757', None, None), ('A10051141965-A10051141966-A10053179226', 'A10049329051', 'Sorties'), ('A10051141965-A10051141968-A10053179226', 'A10049329051', 'Sorties'), ('A10051141965-A10051141970-A10053179226', 'A10049329051', 'Sorties'), ('A10051141965-A10051141990-A10053179227', 'A10049329051', 'Sorties'), - ('A10056514645-A10056514650-A10053179757', None, None), - ('A10056514645-A10056514648-A10053179876', None, None), - ('A10056514645-A10056514649-A10053179757', None, None), - ('A10056517594-A10056517595-A10056517597', 'plop', None), ] - item = resp.json['data'][0] + item = resp.json['data'][4] item['activity'] = 'N/A' item['unit'] = 'N/A' item['place'] = 'N/A' @@ -6465,9 +6461,17 @@ def test_read_activity_list(activity_service, con, app, freezer): def test_read_activity_list_no_nature(activity_service, con, app, freezer): - activity_service.add_soap_response('readActivityList', get_xml_file('R_read_activity_list_no_nature.xml')) + def request_check(request): + assert request.schoolyear == 1970 + assert request.dateStartCalend == datetime.datetime(2023, 3, 1, 0, 0) + assert request.dateEndCalend == datetime.datetime(2025, 2, 28, 0, 0) + + activity_service.add_soap_response( + 'readActivityList', get_xml_file('R_read_activity_list_no_nature.xml'), request_check=request_check + ) url = get_endpoint('read-activity-list') freezer.move_to('2024-02-29') + con.update_catalog_referential() resp = app.get(url) assert resp.json['err'] == 0 assert len(resp.json['data']) == 0