From ac4eef819e3f30ef96201b1888cd0d9443c7b12d Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Thu, 23 Mar 2023 14:05:57 +0100 Subject: [PATCH 1/5] toulouse-maelis: correct bug loading next year catalog (#75652) --- passerelle/contrib/toulouse_maelis/models.py | 2 +- tests/test_toulouse_maelis.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/passerelle/contrib/toulouse_maelis/models.py b/passerelle/contrib/toulouse_maelis/models.py index d39cb0aa..379288e0 100644 --- a/passerelle/contrib/toulouse_maelis/models.py +++ b/passerelle/contrib/toulouse_maelis/models.py @@ -195,7 +195,7 @@ class ToulouseMaelis(BaseResource, HTTPResource): # put activity catalog per year as referential data = [] reference_year = utils.get_reference_year_from_date(datetime.date.today()) - for year in range(reference_year, reference_year + 1): + for year in range(reference_year, reference_year + 2): response = self.call( 'Activity', 'readActivityList', diff --git a/tests/test_toulouse_maelis.py b/tests/test_toulouse_maelis.py index eb9d429f..75763b19 100644 --- a/tests/test_toulouse_maelis.py +++ b/tests/test_toulouse_maelis.py @@ -499,6 +499,10 @@ def test_cron(db): 'YearSchool', ] + assert [ + (x.item_id, x.item_text) for x in Referential.objects.filter(referential_name='ActivityCatalog').all() + ] == [('2022', '2022-2023'), ('2023', '2023-2024')] + def test_link(family_service, con, app): def request_check(request): -- 2.39.2 From 0c8f40826696ff4419ebe41856c8fa6a81f7ba23 Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Thu, 23 Mar 2023 15:10:37 +0100 Subject: [PATCH 2/5] toulouse-maelis: correct school year period on catalog (#75652) --- passerelle/contrib/toulouse_maelis/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/passerelle/contrib/toulouse_maelis/models.py b/passerelle/contrib/toulouse_maelis/models.py index 379288e0..f686dc6a 100644 --- a/passerelle/contrib/toulouse_maelis/models.py +++ b/passerelle/contrib/toulouse_maelis/models.py @@ -201,7 +201,7 @@ class ToulouseMaelis(BaseResource, HTTPResource): 'readActivityList', schoolyear=year, dateStartCalend='%s-09-01' % year, - dateEndCalend='%s-09-01' % (year + 1), + dateEndCalend='%s-08-31' % (year + 1), ) data.append( { -- 2.39.2 From b78f698616363bc77803478ed0708aca929c4025 Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Thu, 23 Mar 2023 15:00:40 +0100 Subject: [PATCH 3/5] toulouse_maelis: bypass cache on activity catalog (#75652) --- passerelle/contrib/toulouse_maelis/models.py | 29 +++++++- tests/test_toulouse_maelis.py | 69 +++++++++++++++----- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/passerelle/contrib/toulouse_maelis/models.py b/passerelle/contrib/toulouse_maelis/models.py index f686dc6a..43564a3f 100644 --- a/passerelle/contrib/toulouse_maelis/models.py +++ b/passerelle/contrib/toulouse_maelis/models.py @@ -28,6 +28,7 @@ from django.utils import dateformat from django.utils.dateparse import parse_date from django.utils.text import slugify from django.utils.timezone import now +from requests import RequestException from zeep.helpers import serialize_object from zeep.wsse.username import UsernameToken @@ -110,7 +111,7 @@ class ToulouseMaelis(BaseResource, HTTPResource): assert self.call('Site', 'isWSRunning') assert self.call('Ape', 'isWSRunning') - def update_referential(self, referential_name, data, id_key, text_key): + def update_referential(self, referential_name, data, id_key, text_key, delete=True): last_update = now() for item in data or []: text = item[text_key] or '' @@ -128,7 +129,9 @@ class ToulouseMaelis(BaseResource, HTTPResource): 'updated': last_update, }, ) - self.referential.filter(referential_name=referential_name, updated__lt=last_update).delete() + if delete: + # delete extraneous items + self.referential.filter(referential_name=referential_name, updated__lt=last_update).delete() def get_referential_data(self, service_name, referential_name): try: @@ -2388,6 +2391,28 @@ class ToulouseMaelis(BaseResource, HTTPResource): day_names = ['Lundi', 'Mardi', 'Mercredi', 'Jeudi', 'Vendredi', 'Samedi', 'Dimanche'] all_criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()} criterias = {key: {'text': value, 'data': {}} for key, value in labels.items()} + + # do not use cache, except on timeout + try: + response = self.call( + 'Activity', + 'readActivityList', + schoolyear=reference_year, + dateStartCalend='%s-09-01' % reference_year, + dateEndCalend='%s-08-31' % (reference_year + 1), + ) + except RequestException: + pass + else: + data = [ + { + 'id': str(reference_year), + 'text': '%s-%s' % (reference_year, reference_year + 1), + 'data': response, + } + ] + self.update_referential('ActivityCatalog', data, 'id', 'text', delete=False) + catalogs = self.get_referential('ActivityCatalog', id=reference_year) activities = catalogs[0]['data'] if catalogs else [] diff --git a/tests/test_toulouse_maelis.py b/tests/test_toulouse_maelis.py index 75763b19..ee933bba 100644 --- a/tests/test_toulouse_maelis.py +++ b/tests/test_toulouse_maelis.py @@ -23,7 +23,7 @@ import pytest import responses from django.db import transaction from django.utils.dateparse import parse_date -from requests.exceptions import ConnectionError +from requests.exceptions import ConnectionError, ReadTimeout from zeep import Settings from zeep.helpers import serialize_object @@ -5251,8 +5251,9 @@ def test_get_public_criterias(start_dob, end_dob, expected): assert expected == [x[1] for x in result] -def test_read_activity_list(con, app, freezer): +def test_read_activity_list(activity_service, con, app, freezer): freezer.move_to('2023-01-01 12:00') + activity_service.add_soap_response('readActivityList', get_xml_file('R_read_activity_list.xml')) url = get_endpoint('read-activity-list') con.loisir_nature_codes = '4,L,, S ' @@ -5303,21 +5304,10 @@ def test_read_activity_list(con, app, freezer): resp = app.get(url, params=params) assert resp.json['err'] == 0 assert len(resp.json['data']) == 0 - - con.loisir_nature_codes = '4,L,, S ' - con.save() - resp = app.get(url, params=params) - assert resp.json['err'] == 0 - assert len(resp.json['data']) == 4 - - params['ref_date'] = '1970-01-01' - resp = app.get(url, params=params) - assert resp.json['err'] == 0 - assert len(resp.json['data']) == 0 assert resp.json == { 'data': [], 'meta': { - 'reference_year': 1969, + 'reference_year': 2022, 'all_criterias': { 'nature': {'text': "Nature de l'activité", 'data': {}, 'order': []}, 'type': {'text': "Type de l'activité", 'data': {}, 'order': []}, @@ -5330,8 +5320,57 @@ def test_read_activity_list(con, app, freezer): } -def test_read_activity_list_no_nature(con, app, freezer): +@mock.patch('passerelle.utils.Request.get') +def test_read_activity_list_cache(mocked_get, con, app): + mocked_get.return_value = ACTIVITY_SERVICE_WSDL + url = get_endpoint('read-activity-list') + params = {'ref_date': '1970-01-01'} + + import pdb + + assert [x.item_id for x in Referential.objects.filter(referential_name='ActivityCatalog')] == [ + '2022', + '2023', + ] + + with mock.patch('passerelle.utils.Request.post') as mocked_post: + mocked_post.side_effect = ReadTimeout('timeout') + resp = app.get(url, params=params) + assert resp.json['err'] == 0 + assert len(resp.json['data']) == 0 + + assert [x.item_id for x in Referential.objects.filter(referential_name='ActivityCatalog')] == [ + '2022', + '2023', + ] + + with mock.patch('passerelle.utils.Request.post') as mocked_post: + mocked_post.return_value = FakedResponse( + content=get_xml_file('R_read_activity_list.xml'), status_code=200 + ) + resp = app.get(url, params=params) + assert resp.json['err'] == 0 + assert len(resp.json['data']) == 4 + + assert [x.item_id for x in Referential.objects.filter(referential_name='ActivityCatalog')] == [ + '1969', + '2022', + '2023', + ] + + with mock.patch('passerelle.utils.Request.post') as mocked_post: + mocked_post.side_effect = ReadTimeout('timeout') + resp = app.get(url, params=params) + assert resp.json['err'] == 0 + assert len(resp.json['data']) == 4 + + +@mock.patch('passerelle.utils.Request.get') +@mock.patch('passerelle.utils.Request.post') +def test_read_activity_list_no_nature(mocked_post, mocked_get, con, app, freezer): freezer.move_to('2023-01-01 12:00') + mocked_get.return_value = ACTIVITY_SERVICE_WSDL + mocked_post.side_effect = ReadTimeout('timeout') url = get_endpoint('read-activity-list') params = {'ref_date': datetime.date.today().strftime(json_date_format)} -- 2.39.2 From 791b341b837b4ee4fd624593042fd80c900b8472 Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Thu, 23 Mar 2023 15:09:05 +0100 Subject: [PATCH 4/5] toulouse-maelis: factorize updating activity referential (#75652) --- passerelle/contrib/toulouse_maelis/models.py | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/passerelle/contrib/toulouse_maelis/models.py b/passerelle/contrib/toulouse_maelis/models.py index 43564a3f..38d7897b 100644 --- a/passerelle/contrib/toulouse_maelis/models.py +++ b/passerelle/contrib/toulouse_maelis/models.py @@ -189,6 +189,15 @@ class ToulouseMaelis(BaseResource, HTTPResource): id_key, text_key = 'schoolYear', 'schoolYear' self.update_referential(referential_name, data, id_key, text_key) + def get_activity_catalog_raw(self, year): + return self.call( + 'Activity', + 'readActivityList', + schoolyear=year, + dateStartCalend='%s-09-01' % year, + dateEndCalend='%s-08-31' % (year + 1), + ) + def update_activity_referentials(self): for referential_name in ('ActivityNatureType',): id_key, text_key = 'code', 'libelle' @@ -199,18 +208,11 @@ class ToulouseMaelis(BaseResource, HTTPResource): data = [] reference_year = utils.get_reference_year_from_date(datetime.date.today()) for year in range(reference_year, reference_year + 2): - response = self.call( - 'Activity', - 'readActivityList', - schoolyear=year, - dateStartCalend='%s-09-01' % year, - dateEndCalend='%s-08-31' % (year + 1), - ) data.append( { 'id': str(year), 'text': '%s-%s' % (year, year + 1), - 'data': response, + 'data': self.get_activity_catalog_raw(year), } ) self.update_referential('ActivityCatalog', data, 'id', 'text') @@ -2394,13 +2396,7 @@ class ToulouseMaelis(BaseResource, HTTPResource): # do not use cache, except on timeout try: - response = self.call( - 'Activity', - 'readActivityList', - schoolyear=reference_year, - dateStartCalend='%s-09-01' % reference_year, - dateEndCalend='%s-08-31' % (reference_year + 1), - ) + response = self.get_activity_catalog_raw(reference_year) except RequestException: pass else: -- 2.39.2 From 685014df6b593dc207a027897869e3c9dcc0fb21 Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Thu, 23 Mar 2023 15:36:23 +0100 Subject: [PATCH 5/5] toulouse-maelis: remove no more required freezer on catalog test (#75652) --- tests/test_toulouse_maelis.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/test_toulouse_maelis.py b/tests/test_toulouse_maelis.py index ee933bba..1641683f 100644 --- a/tests/test_toulouse_maelis.py +++ b/tests/test_toulouse_maelis.py @@ -28,7 +28,7 @@ from zeep import Settings from zeep.helpers import serialize_object from passerelle.contrib.toulouse_maelis.models import Link, Referential, ToulouseMaelis -from passerelle.contrib.toulouse_maelis.utils import get_public_criterias, json_date_format +from passerelle.contrib.toulouse_maelis.utils import get_public_criterias from passerelle.utils.jsonresponse import APIError from passerelle.utils.soap import SOAPError from passerelle.utils.templates import render_to_string @@ -5251,16 +5251,13 @@ def test_get_public_criterias(start_dob, end_dob, expected): assert expected == [x[1] for x in result] -def test_read_activity_list(activity_service, con, app, freezer): - freezer.move_to('2023-01-01 12:00') +def test_read_activity_list(activity_service, con, app): activity_service.add_soap_response('readActivityList', get_xml_file('R_read_activity_list.xml')) url = get_endpoint('read-activity-list') con.loisir_nature_codes = '4,L,, S ' con.save() - params = { - 'ref_date': datetime.date.today().strftime(json_date_format), - } + params = {'ref_date': '2023-01-01'} resp = app.get(url, params=params) assert resp.json['err'] == 0 assert len(resp.json['data']) == 4 @@ -5326,8 +5323,6 @@ def test_read_activity_list_cache(mocked_get, con, app): url = get_endpoint('read-activity-list') params = {'ref_date': '1970-01-01'} - import pdb - assert [x.item_id for x in Referential.objects.filter(referential_name='ActivityCatalog')] == [ '2022', '2023', @@ -5367,12 +5362,11 @@ def test_read_activity_list_cache(mocked_get, con, app): @mock.patch('passerelle.utils.Request.get') @mock.patch('passerelle.utils.Request.post') -def test_read_activity_list_no_nature(mocked_post, mocked_get, con, app, freezer): - freezer.move_to('2023-01-01 12:00') +def test_read_activity_list_no_nature(mocked_post, mocked_get, con, app): mocked_get.return_value = ACTIVITY_SERVICE_WSDL mocked_post.side_effect = ReadTimeout('timeout') url = get_endpoint('read-activity-list') - params = {'ref_date': datetime.date.today().strftime(json_date_format)} + params = {'ref_date': '2023-01-01'} # Empty activity type return by referential def mocked_reply(referential_name, id=None): -- 2.39.2