entrouvert
/
ants-hub
15
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Implement API v2 "Optimistion des rendez-vous en mairies" (#76412) 

Swagger: https://rdvmairie-optimisation-dev.osc-secnum-fr1.scalingo.io/docs

* RendezVous models are now soft-deleted through the canceled datetime
  field when specifically stated as 'annule' on the
  rendez-vous-disponible endpoint or when absent from a full update,
* last-upload timestamp field on the RendezVous model allow filtering of
  created/updated RendezVous instances to push,
* new configuration keys are added to store the authentication token and
  base URL of the new API endpoints.
* new command upload-rdvs is run every 3 minutes by uwsgi cron to push
  new/deleted rendez-vous into the ANTS database.
* soft-deleted rendez-vous are excluded from aggregated statistics on
  active rendez-vous
* Tests added for success and error path in ants_hub.api.ants routines.
This commit is contained in:
Benjamin Dauvergne 2023-08-28 18:18:14 +02:00
parent c6b5852a50
commit d5e6f79257
10 changed files with 417 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
debian/uwsgi.ini vendored
View File

@ -26,6 +26,8 @@ ignore-sigpipe = true
logto2=/var/log/ants-hub/uwsgi.log
log-maxsize=2000000
cron2 = minute=-3,unique=1 /usr/bin/ants-hub-manage upload-rdvs
if-file = /etc/ants-hub/uwsgi-local.ini
include = /etc/ants-hub/uwsgi-local.ini
endif =

89
src/ants_hub/api/ants.py Normal file
View File

@ -0,0 +1,89 @@
# ANTS-Hub - Copyright (C) Entr'ouvert
import logging
import requests
from django.db.models import F, Q
from django.db.transaction import atomic
from ants_hub.data.models import Config, RendezVous
from ants_hub.timezone import localtime, now
logger = logging.getLogger('ants_hub.api.ants')
class ANTSError(Exception):
pass
def get_api_optimisation_auth_token():
return Config.get(Config.REQUEST_TO_ANTS_V2_AUTH_TOKEN)
def get_api_optimisation_url():
return Config.get(
Config.REQUEST_TO_ANTS_V2_BASE_URL, 'https://api-coordination.rendezvouspasseport.ants.gouv.fr/api/'
)
def push_rdv(rdv):
# swagger : https://api-coordination.rendezvouspasseport.ants.gouv.fr/docs#/Appointments%20-%20%C3%A9diteurs/add_appointment_api_appointments_post
auth_token = get_api_optimisation_auth_token()
base_url = get_api_optimisation_url()
try:
if rdv.canceled is None:
params = [
('application_id', rdv.identifiant_predemande),
('management_url', rdv.gestion_url),
('meeting_point', rdv.lieu.nom),
('meeting_point_id', str(rdv.lieu.id)),
('appointment_date', localtime(rdv.date).strftime('%Y-%m-%d %H:%M:%S')),
]
response = requests.post(
base_url + 'appointments',
params=params,
headers={'x-rdv-opt-auth-token': auth_token},
timeout=10,
)
response.raise_for_status()
return response.json()['success'] == 'true'
else:
params = [
('application_id', rdv.identifiant_predemande),
('meeting_point', rdv.lieu.nom),
('meeting_point_id', str(rdv.lieu.id)),
('appointment_date', localtime(rdv.date).strftime('%Y-%m-%d %H:%M:%S')),
]
response = requests.delete(
base_url + 'appointments',
params=params,
headers={'x-rdv-opt-auth-token': auth_token},
timeout=10,
)
response.raise_for_status()
return response.json()['rowcount'] == 1
except (requests.RequestException, ValueError, KeyError, TypeError) as e:
raise ANTSError(str(e))
def get_rdvs_to_upload():
return RendezVous.objects.filter(
Q(last_upload__isnull=True) | Q(last_upload__lt=F('last_update'))
).select_related('lieu')
def upload_rdvs():
# get never uploaded rdvs and rdv uploaded which changed (cancelled)
with atomic():
rdvs = get_rdvs_to_upload().select_for_update(of=('self',)).distinct()
start = now()
pushed = set()
for rdv in rdvs:
try:
if push_rdv(rdv):
pushed.add(rdv)
except ANTSError as e:
logger.warning('unable to push rdv(%s) of lieu %s: %r', rdv, rdv.lieu, e)
rdvs.filter(pk__in=[rdv.pk for rdv in pushed]).update(last_upload=start)
for rdv in pushed:
logger.info('pushed rdv(%s) of lieu %s', rdv, rdv.lieu)

12
src/ants_hub/api/management/commands/upload-rdvs.py Normal file
View File

@ -0,0 +1,12 @@
# ANTS-Hub - Copyright (C) Entr'ouvert
from django.core.management.base import BaseCommand
from ants_hub.api.ants import upload_rdvs
class Command(BaseCommand):
help = 'Télécharger les rdvs associés à des numéros de pré-demande'
def handle(self, *args, **options):
upload_rdvs()

54
src/ants_hub/api/views/chrono.py
View File

@ -108,7 +108,9 @@ class RendezVousDisponibleView(View):
'lieux': lieux_data,
'nombre_de_lieux': collectivite.lieux.count(),
'nombre_de_pre_demandes_actives': RendezVous.objects.filter(
date__gte=now(), lieu__collectivite=collectivite
date__gte=now(),
lieu__collectivite=collectivite,
canceled__isnull=True,
).count(),
'nombre_de_jours_avec_rdv_disponibles': Plage.objects.filter(
lieu__collectivite=collectivite, date__gte=now().date()
@ -126,7 +128,9 @@ class RendezVousDisponibleView(View):
'ville': lieu.ville,
'longitude': lieu.longitude,
'latitude': lieu.latitude,
'nombre_de_pre_demandes_actives': lieu.rdvs.filter(date__gte=now()).count(),
'nombre_de_pre_demandes_actives': lieu.rdvs.filter(
date__gte=now(), canceled__isnull=True
).count(),
'nombre_de_jours_avec_rdv_disponibles': lieu.plages.filter(
date__gte=now().date()
).aggregate(Count('date'))['date__count'],
@ -167,33 +171,49 @@ class RendezVousDisponibleView(View):
identifiant_predemande = rdv.pop('id').strip()
date = datetime.datetime.fromisoformat(rdv.pop('date'))
annule = bool(rdv.pop('annule', False))
rdv.setdefault('gestion_url', '')
rdv.setdefault('annulation_url', '')
if annule:
_, count_by_model = lieu.rdvs.filter(
identifiant_predemande=identifiant_predemande, date=date
).delete()
self.rdv_deleted += count_by_model.get('data.RendezVous', 0)
return
try:
rdv = lieu.rdvs.get(identifiant_predemande=identifiant_predemande, date=date, **rdv)
except RendezVous.DoesNotExist:
rdv, created = lieu.rdvs.update_or_create(
timestamp = now()
rdvs = list(
lieu.rdvs.filter(
identifiant_predemande=identifiant_predemande, date=date, canceled__isnull=True
).select_for_update()
)
count = lieu.rdvs.filter(
identifiant_predemande=identifiant_predemande, date=date, canceled__isnull=True
).update(canceled=timestamp, last_update=timestamp)
self.rdv_deleted += count
return rdvs
rendez_vous = lieu.rdvs.filter(
identifiant_predemande=identifiant_predemande, date=date, canceled__isnull=True, **rdv
).first()
if rendez_vous is None:
rendez_vous, created = lieu.rdvs.update_or_create(
identifiant_predemande=identifiant_predemande, date=date, defaults=rdv
)
if created:
self.rdv_created += 1
else:
self.rdv_updated += 1
return rdv
return [rendez_vous]
def handle_rdvs_payload(self, lieu, rdvs, full=False):
rdv_pks = set()
for rdv in rdvs:
rdv = self.handle_rdv_payload(lieu, rdv)
if full and rdv and rdv.pk:
rdv_pks.add(rdv.pk)
updated_rdvs = self.handle_rdv_payload(lieu, rdv)
if full and updated_rdvs:
rdv_pks.update(updated_rdv.pk for updated_rdv in updated_rdvs)
if full:
_, count_by_model = lieu.rdvs.exclude(pk__in=rdv_pks).delete()
self.rdv_deleted += count_by_model.get('data.RendezVous', 0)
timestamp = now()
count = (
lieu.rdvs.exclude(pk__in=rdv_pks)
.filter(canceled__isnull=True)
.update(canceled=timestamp, last_update=timestamp)
)
self.rdv_deleted += count
return lieu
def handle_plages_payload(self, lieu, plages, full=False):

20
src/ants_hub/data/migrations/0005_rendezvous_last_upload.py Normal file
View File

@ -0,0 +1,20 @@
# Generated by Django 3.2.19 on 2023-08-28 16:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('data', '0004_alter_plage_unique_together'),
]
operations = [
migrations.AddField(
model_name='rendezvous',
name='last_upload',
field=models.DateTimeField(
null=True, verbose_name="Dernière synchronisation avec l'ANTS", blank=True
),
),
]

18
src/ants_hub/data/migrations/0006_rendezvous_canceled.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 3.2.19 on 2023-08-29 11:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('data', '0005_rendezvous_last_upload'),
]
operations = [
migrations.AddField(
model_name='rendezvous',
name='canceled',
field=models.DateTimeField(blank=True, null=True, verbose_name='Annulation'),
),
]

45
src/ants_hub/data/migrations/0007_alter_config_key.py Normal file
View File

@ -0,0 +1,45 @@
# Generated by Django 3.2.19 on 2023-08-29 11:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('data', '0006_rendezvous_canceled'),
]
operations = [
migrations.AlterField(
model_name='config',
name='key',
field=models.CharField(
choices=[
(
'REQUEST_FROM_ANTS_AUTH_TOKEN',
"Token d'authentification pour les appels en provenance de l'ANTS",
),
(
'REQUEST_TO_ANTS_AUTH_TOKEN',
"Token d'authentification pour les appels en direction de l'ANTS - API rendez-vous",
),
(
'REQUEST_TO_ANTS_V2_AUTH_TOKEN',
"Token d'authentification pour les appels en direction de l'ANTS - API optimisation des rendez-vous",
),
(
'REQUEST_TO_ANTS_BASE_URL',
"URL de base des web-services de l'API Rendez-vous de l'ANTS",
),
(
'REQUEST_TO_ANTS_V2_BASE_URL',
"URL de base des web-services de l'API optimisation des rendez-vous de l'ANTS",
),
],
max_length=64,
primary_key=True,
serialize=False,
verbose_name='Clé',
),
),
]

19
src/ants_hub/data/models.py
View File

@ -400,6 +400,10 @@ class RendezVous(models.Model):
annulation_url = URLField(verbose_name='URL d\'annulation des rendez-vous', blank=True)
created = models.DateTimeField(verbose_name='Création', auto_now_add=True)
last_update = models.DateTimeField(verbose_name='Dernière mise à jour', auto_now=True)
last_upload = models.DateTimeField(
verbose_name='Dernière synchronisation avec l\'ANTS', null=True, blank=True
)
canceled = models.DateTimeField(verbose_name='Annulation', null=True, blank=True)
objects = RendezVousManager()
@ -465,12 +469,25 @@ class RendezVous(models.Model):
class Config(models.Model):
REQUEST_FROM_ANTS_AUTH_TOKEN = 'REQUEST_FROM_ANTS_AUTH_TOKEN'
REQUEST_TO_ANTS_AUTH_TOKEN = 'REQUEST_TO_ANTS_AUTH_TOKEN'
REQUEST_TO_ANTS_V2_AUTH_TOKEN = 'REQUEST_TO_ANTS_V2_AUTH_TOKEN'
REQUEST_TO_ANTS_BASE_URL = 'REQUEST_TO_ANTS_BASE_URL'
REQUEST_TO_ANTS_V2_BASE_URL = 'REQUEST_TO_ANTS_V2_BASE_URL'
KEYS = [
(REQUEST_FROM_ANTS_AUTH_TOKEN, 'Token d\'authentification pour les appels en provenance de l\'ANTS'),
(REQUEST_TO_ANTS_AUTH_TOKEN, 'Token d\'authentification pour les appels en direction de l\'ANTS'),
(
REQUEST_TO_ANTS_AUTH_TOKEN,
'Token d\'authentification pour les appels en direction de l\'ANTS - API rendez-vous',
),
(
REQUEST_TO_ANTS_V2_AUTH_TOKEN,
'Token d\'authentification pour les appels en direction de l\'ANTS - API optimisation des rendez-vous',
),
(REQUEST_TO_ANTS_BASE_URL, 'URL de base des web-services de l\'API Rendez-vous de l\'ANTS'),
(
REQUEST_TO_ANTS_V2_BASE_URL,
'URL de base des web-services de l\'API optimisation des rendez-vous de l\'ANTS',
),
]
key = models.CharField(verbose_name='Clé', choices=KEYS, max_length=64, primary_key=True)

165
tests/api/test_ants.py
View File

@ -1,11 +1,15 @@
# ANTS-Hub - Copyright (C) Entr'ouvert
import datetime
import urllib.parse
import pytest
import responses
import responses.matchers
from django.core.management import call_command
from ants_hub.data.models import Config, Lieu, Plage, RendezVous, TypeDeRdv
from ants_hub.timezone import now
@pytest.mark.parametrize(
@ -348,3 +352,164 @@ class TestEndpoints:
def test_search_application_ids(self, db, django_app):
django_app.get('/api/ants/searchApplicationIds', params={}, status=422)
class TestAPIV2Push:
@pytest.fixture(autouse=True)
def setup(self, db, settings, django_app, freezer):
Config.set(Config.REQUEST_TO_ANTS_V2_AUTH_TOKEN, 'abcd')
freezer.move_to('2023-04-03T12:20:00+02:00')
call_command('loaddata', 'fixtures/example1.json')
@responses.activate
def test_push(self, db, freezer):
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(canceled__isnull=True).count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 1
post_response = responses.add(
responses.POST,
'https://api-coordination.rendezvouspasseport.ants.gouv.fr/api/appointments',
json={'success': 'true'},
status=200,
match=[responses.matchers.header_matcher({'x-rdv-opt-auth-token': 'abcd'})],
)
delete_response = responses.add(
responses.DELETE,
'https://api-coordination.rendezvouspasseport.ants.gouv.fr/api/appointments',
json={'rowcount': 1},
status=200,
match=[responses.matchers.header_matcher({'x-rdv-opt-auth-token': 'abcd'})],
)
call_command('upload-rdvs')
assert post_response.call_count == 1
assert delete_response.call_count == 0
called_url = urllib.parse.urlparse(responses._default_mock.calls[-1].request.url)
called_qs = urllib.parse.parse_qsl(called_url.query, keep_blank_values=True)
assert called_url._replace(query='')._asdict() == {
'scheme': 'https',
'netloc': 'api-coordination.rendezvouspasseport.ants.gouv.fr',
'path': '/api/appointments',
'params': '',
'query': '',
'fragment': '',
}
assert called_qs == [
('application_id', 'abcd'),
('management_url', ''),
('meeting_point', 'Mairie'),
('meeting_point_id', '1'),
('appointment_date', '2023-04-03 12:15:00'),
]
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 0
call_command('upload-rdvs')
assert post_response.call_count == 1
assert delete_response.call_count == 0
freezer.move_to('2023-04-03T13:00:00+02:00')
RendezVous.objects.update(canceled=now(), last_update=now())
call_command('upload-rdvs')
assert post_response.call_count == 1
assert delete_response.call_count == 1
called_url = urllib.parse.urlparse(responses._default_mock.calls[-1].request.url)
called_qs = urllib.parse.parse_qsl(called_url.query, keep_blank_values=True)
assert called_url._replace(query='')._asdict() == {
'scheme': 'https',
'netloc': 'api-coordination.rendezvouspasseport.ants.gouv.fr',
'path': '/api/appointments',
'params': '',
'query': '',
'fragment': '',
}
assert called_qs == [
('application_id', 'abcd'),
('meeting_point', 'Mairie'),
('meeting_point_id', '1'),
('appointment_date', '2023-04-03 12:15:00'),
]
@responses.activate
@pytest.mark.parametrize(
'response_kwargs',
[
{
'json': {},
'status': 200,
},
{
'json': None,
'status': 200,
},
{
'json': '',
'status': 200,
},
{
'status': 500,
},
],
)
def test_post_error(self, response_kwargs, db, freezer):
post_response = responses.add(
responses.POST,
'https://api-coordination.rendezvouspasseport.ants.gouv.fr/api/appointments',
match=[responses.matchers.header_matcher({'x-rdv-opt-auth-token': 'abcd'})],
**response_kwargs,
)
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 1
call_command('upload-rdvs')
assert post_response.call_count == 1
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 1
@responses.activate
@pytest.mark.parametrize(
'response_kwargs',
[
{
'json': {},
'status': 200,
},
{
'json': None,
'status': 200,
},
{
'json': '',
'status': 200,
},
{
'status': 500,
},
],
)
def test_delete_error(self, response_kwargs, db, freezer):
post_response = responses.add(
responses.DELETE,
'https://api-coordination.rendezvouspasseport.ants.gouv.fr/api/appointments',
match=[responses.matchers.header_matcher({'x-rdv-opt-auth-token': 'abcd'})],
**response_kwargs,
)
RendezVous.objects.update(canceled=now(), last_update=now())
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 1
call_command('upload-rdvs')
assert post_response.call_count == 1
assert RendezVous.objects.count() == 1
assert RendezVous.objects.filter(last_upload__isnull=True).count() == 1

14
tests/api/test_chrono.py
View File

@ -724,11 +724,19 @@ def test_rendez_vous_disponibles_full(django_app, db, freezer):
assert Collectivite.objects.count() == 1
assert Lieu.objects.count() == 1
assert Plage.objects.count() == 4
assert RendezVous.objects.count() == 2
assert set(RendezVous.objects.values_list('identifiant_predemande', flat=True)) == {'abcd1', 'abcd3'}
assert RendezVous.objects.filter(canceled__isnull=True).count() == 2
assert RendezVous.objects.filter(canceled__isnull=False).count() == 1
assert set(
RendezVous.objects.filter(canceled__isnull=True).values_list('identifiant_predemande', flat=True)
) == {'abcd1', 'abcd3'}
assert set(
RendezVous.objects.filter(canceled__isnull=False).values_list('identifiant_predemande', flat=True)
) == {'abcd2'}
# check objects are not updated/created uselessly
assert plage_last_update >= set(Plage.objects.values_list('last_update', flat=True))
assert rendez_vous_last_update >= set(RendezVous.objects.values_list('last_update', flat=True))
assert rendez_vous_last_update >= set(
RendezVous.objects.filter(canceled__isnull=True).values_list('last_update', flat=True)
)
assert response.json == {
'err': 0,
'data': {