entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 15 Releases Activity

rsa13: add fields to manage CSV columns (#70276)

This commit is contained in:
Benjamin Dauvergne 2022-10-14 12:43:42 +02:00
parent 4322b1a819
commit c0b972e7ac
4 changed files with 354 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
passerelle/contrib/rsa13/migrations/0002_add_csv_columns_fields.py Normal file
View File

@ -0,0 +1,72 @@
# Generated by Django 2.2.26 on 2022-10-14 10:35
from django.conf import settings
from django.db import migrations, models
def get_csv_columns_from_old_settings(setting_name):
def func():
columns = getattr(settings, setting_name, [])
lines = []
for column in columns:
try:
name, title = column
line = f'{name} {title}'
except ValueError:
line = column
lines.append(line)
return '\n'.join(lines)
return func
class Migration(migrations.Migration):
dependencies = [
('rsa13', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='rsa13resource',
name='beneficiaire_csv_columns',
field=models.TextField(
blank=True,
verbose_name='CSV columns for beneficiaires',
default=get_csv_columns_from_old_settings('RSA13_CSV_COLUMNS'),
),
),
migrations.AddField(
model_name='rsa13resource',
name='facturation_csv_columns',
field=models.TextField(
blank=True,
verbose_name='CSV columns for facturation',
default=get_csv_columns_from_old_settings('RSA13_FACTURATION_CSV_COLUMNS'),
),
),
migrations.AddField(
model_name='rsa13resource',
name='sorti_csv_columns',
field=models.TextField(
blank=True,
verbose_name='CSV columns for sorti',
default=get_csv_columns_from_old_settings('RSA13_BENEFICIAIRE_SORTI_CSV_COLUMNS'),
),
),
migrations.AlterField(
model_name='rsa13resource',
name='beneficiaire_csv_columns',
field=models.TextField(blank=True, verbose_name='CSV columns for beneficiaires'),
),
migrations.AlterField(
model_name='rsa13resource',
name='facturation_csv_columns',
field=models.TextField(blank=True, verbose_name='CSV columns for facturation'),
),
migrations.AlterField(
model_name='rsa13resource',
name='sorti_csv_columns',
field=models.TextField(blank=True, verbose_name='CSV columns for sorti'),
),
]

348
passerelle/contrib/rsa13/models.py
View File

@ -18,7 +18,6 @@ import csv
from urllib.parse import urljoin
import requests
from django.conf import settings
from django.db import models
from django.http import HttpResponse
from django.urls import reverse
@ -27,6 +26,7 @@ from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _
from passerelle.base.models import BaseResource, HTTPResource
from passerelle.forms import GenericConnectorForm
from passerelle.utils.api import endpoint
from passerelle.utils.json import response_schema
from passerelle.utils.jsonresponse import APIError
@ -51,11 +51,38 @@ def parameters(update=None):
return d
def dump_csv_columns(columns):
lines = []
for column in columns:
try:
name, title = columns
line = f'{name} {title}'
except ValueError:
line = column
lines.append(line)
return '\n'.join(lines)
class RSA13Form(GenericConnectorForm):
def __init__(self, *args, **kwargs):
kwargs['initial'] = {}
for name in RSA13Resource.CSV_EXPORTS:
field = f'{name}_csv_columns'
kwargs['initial'][field] = dump_csv_columns(DEFAULTS[field])
super().__init__(*args, **kwargs)
class RSA13Resource(BaseResource, HTTPResource):
category = _('Business Process Connectors')
webservice_base_url = models.URLField(_('Webservice Base URL'))
beneficiaire_csv_columns = models.TextField(_('CSV columns for beneficiaires'), blank=True)
facturation_csv_columns = models.TextField(_('CSV columns for facturation'), blank=True)
sorti_csv_columns = models.TextField(_('CSV columns for sorti'), blank=True)
hide_description_fields = ['beneficiaire_csv_columns', 'facturation_csv_columns', 'sorti_csv_columns']
manager_form_base_class = RSA13Form
log_requests_errors = False
class Meta:
@ -108,6 +135,39 @@ class RSA13Resource(BaseResource, HTTPResource):
if response.json().get('ping') != 'pong':
raise APIError('ping/pong expected received: "%s"' % repr(response)[:1024])
def get_columns(self, name):
def parse_csv_columns(content):
columns = []
for line in content.splitlines():
if not line.strip():
continue
row = line.strip().split(' ', 1)
if not row[0]:
continue
try:
json_name, column_name = row
columns.append((json_name, column_name.strip()))
except ValueError:
json_name = row[0]
columns.append(json_name)
return columns
columns = []
field = f'{name}_csv_columns'
for column in parse_csv_columns(getattr(self, field)) or DEFAULTS[field]:
if isinstance(column, str):
columns.append((column, column))
else:
columns.append(tuple(column))
return columns
CSV_EXPORTS = ['beneficiaire', 'facturation', 'sorti']
@property
def description_csv_tables(self):
for name in self.CSV_EXPORTS:
yield (name.title(), self.get_columns(name))
@endpoint(
description=_('Get nomenclature'),
long_description=_('Domain can be: MOTICLODAC, MOTIF_FIN_ACC, RESULTAT_RDV, RELANCE_RDV'),
@ -545,40 +605,19 @@ class RSA13Resource(BaseResource, HTTPResource):
params[key] = locals()[key]
return self.get('platform/%s/beneficiaire/' % platform_id, email=email, ip=ip, params=params)
CSV_DEFAULT_COLUMNS = [
"NUM_CAF",
"CODE_PER",
"NOM_PER",
"PRENOM_PER",
"DTNAI_PER",
"ACTIF_PER",
"CODE_PI",
"LIB_CODE_PI",
"TOPPERSDRODEVORSA",
"LIB_ETATDOSRSA",
"LIB_MOTIF_ETATDOSRSA",
"NB_JOUR_DEPUIS_ARR",
"DATE_DEB",
"DATE_1IERE_CONS",
"DATE_DERNIERE_CONSULT",
"DATE_REELLE_RDV",
"NUM_CINS",
"DATE_SIGN",
"DATE_DEB_CI",
"DATE_FIN_CI",
"REFERENT_CI",
"ACTION_EN_COURS",
"DELAI_REGUL",
"PROC_EN_COURS",
"REFERENT_AFFECTATION",
'COMPL1_ADR',
'COMPL2_ADR',
'VOIE_ADR',
'LIEU_DISTRIB_ADR',
'CP_ADR',
'VILLE_ADR',
'INSEE_ADR',
]
def csv_response(self, name, data, filename):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = f'attachment; filename="{filename}"'
# write Unicode BOM to the response stream
response.write(b'\xef\xbb\xbf')
csv_columns = self.get_columns(name)
names = [name for name, title in csv_columns]
titles = [title for name, title in csv_columns]
writer = csv.writer(response, delimiter=';')
writer.writerow(titles)
for row in data:
writer.writerow(str(row.get(name) or '') for name in names)
return response
@endpoint(
name='platform',
@ -626,24 +665,8 @@ class RSA13Resource(BaseResource, HTTPResource):
if locals().get(key):
params[key] = locals()[key]
content = self.get('platform/%s/beneficiaire/csv/' % platform_id, email=email, ip=ip, params=params)
data = content['data']
response = HttpResponse(content_type='text/csv')
date = now().strftime('%Y-%m-%d_%H:%M')
response['Content-Disposition'] = 'attachment; filename="beneficiaires-%s.csv"' % date
# write Unicode BOM to the response stream
response.write(b'\xef\xbb\xbf')
csv_columns = getattr(settings, 'RSA13_CSV_COLUMNS', self.CSV_DEFAULT_COLUMNS)
writer = csv.writer(response, delimiter=';')
# CSV_DEFAULT_COLUMNS can be a list of string giving column name and
# JSON property name or a 2-tuple (json-name, column title).
titles = [col if isinstance(col, str) else col[1] for col in csv_columns]
names = [col if isinstance(col, str) else col[0] for col in csv_columns]
writer.writerow(titles)
for row in data:
writer.writerow(str(row.get(name) or '') for name in names)
return response
return self.csv_response('beneficiaire', data=content['data'], filename=f'beneficiaire-{date}.csv')
@endpoint(
name='platform',
@ -1864,23 +1887,6 @@ class RSA13Resource(BaseResource, HTTPResource):
)
return response
FACTURATION_CSV_COLUMNS = [
"PLATEFORME",
"MATRICULE",
"NOM",
"PRENOM",
"DTNAI",
"GENRE",
"ROLE",
"CODE_POSTAL",
"COMMUNE",
"DATE_SIGN",
"DATE_DEB",
"DUREE",
"DATE_FIN",
"COEFFICIENT",
]
@endpoint(
name='platform',
pattern=r'^(?P<platform_id>[0-9]{1,10})/facturation/csv/$',
@ -1921,22 +1927,122 @@ class RSA13Resource(BaseResource, HTTPResource):
):
url = 'platform/%s/facturation/csv' % platform_id
content = self.get(url, email=email, ip=ip, params={'date_deb': date_deb, 'date_fin': date_fin})
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = (
'attachment; filename="facturation-%s-%s.csv"' % (date_deb, date_fin),
)
# write Unicode BOM
response.write(b'\xef\xbb\xbf')
csv_columns = getattr(settings, 'RSA13_FACTURATION_CSV_COLUMNS', self.FACTURATION_CSV_COLUMNS)
names = [col if isinstance(col, str) else col[0] for col in csv_columns]
titles = [col if isinstance(col, str) else col[1] for col in csv_columns]
writer = csv.writer(response, delimiter=';')
writer.writerow(titles)
for row in content.get('data') or []:
writer.writerow(str(row.get(name) or '') for name in names)
return response
filename = f'facturation-{date_deb}-{date_fin}.csv'
return self.csv_response('facturation', content.get('data') or [], filename=filename)
BENEFICIAIRE_SORTI_CSV_COLUMN = [
@endpoint(
name='platform',
pattern=r'^(?P<platform_id>[0-9]{1,10})/beneficiaire/sorti/csv/$',
example_pattern='{platform_id}/beneficiaire/sorti/csv/',
description_get=_('Get platform beneficiaries removed in the last 90 days'),
perm='can_access',
parameters=parameters(
{
'platform_id': {
'description': _('Platform numeric identifier'),
'example_value': '11',
},
}
),
display_category=_('Platform'),
display_order=4.9,
)
def platform_beneficiaire_sorti_csv(
self,
request,
platform_id,
email,
ip=None,
):
url = 'platform/%s/beneficiaire/sorti/csv' % platform_id
content = self.get(url, email=email, ip=ip)
date = now().strftime('%Y-%m-%d_%H:%M')
return self.csv_response(
'sorti', data=content.get('data') or [], filename=f'beneficiaire-sorti-{date}.csv'
)
@endpoint(
name='sous-action',
description=_('Get sub-actions'),
perm='can_access',
parameters=parameters(),
json_schema_response=response_schema(
{
'type': 'array',
'items': {
'type': 'object',
'required': ['id', 'text'],
'properties': {
'id': {
'type': 'string',
},
'text': {
'type': 'string',
},
'description': {
'type': 'string',
},
},
},
}
),
)
def sous_action(self, request, email, ip=None):
return self.get('sousaction/', email=email, ip=ip)
DEFAULTS = {
'beneficiaire_csv_columns': [
"NUM_CAF",
"CODE_PER",
"NOM_PER",
"PRENOM_PER",
"DTNAI_PER",
"ACTIF_PER",
"CODE_PI",
"LIB_CODE_PI",
"TOPPERSDRODEVORSA",
"LIB_ETATDOSRSA",
"LIB_MOTIF_ETATDOSRSA",
"NB_JOUR_DEPUIS_ARR",
"DATE_DEB",
"DATE_1IERE_CONS",
"DATE_DERNIERE_CONSULT",
"DATE_REELLE_RDV",
"NUM_CINS",
"DATE_SIGN",
"DATE_DEB_CI",
"DATE_FIN_CI",
"REFERENT_CI",
"ACTION_EN_COURS",
"DELAI_REGUL",
"PROC_EN_COURS",
"REFERENT_AFFECTATION",
'COMPL1_ADR',
'COMPL2_ADR',
'VOIE_ADR',
'LIEU_DISTRIB_ADR',
'CP_ADR',
'VILLE_ADR',
'INSEE_ADR',
],
'facturation_csv_columns': [
"PLATEFORME",
"MATRICULE",
"NOM",
"PRENOM",
"DTNAI",
"GENRE",
"ROLE",
"CODE_POSTAL",
"COMMUNE",
"DATE_SIGN",
"DATE_DEB",
"DUREE",
"DATE_FIN",
"COEFFICIENT",
],
'sorti_csv_columns': [
"NUM_CAF",
"CODE_PER",
"NOM_PER",
@ -1973,75 +2079,5 @@ class RSA13Resource(BaseResource, HTTPResource):
"NOUVEAU_DUREE_CI",
"NOUVEAU_DATE_DEB_CI",
"NOUVEAU_DATE_FIN_CI",
]
@endpoint(
name='platform',
pattern=r'^(?P<platform_id>[0-9]{1,10})/beneficiaire/sorti/csv/$',
example_pattern='{platform_id}/beneficiaire/sorti/csv/',
description_get=_('Get platform beneficiaries removed in the last 90 days'),
perm='can_access',
parameters=parameters(
{
'platform_id': {
'description': _('Platform numeric identifier'),
'example_value': '11',
},
}
),
display_category=_('Platform'),
display_order=4.9,
)
def platform_beneficiaire_sorti_csv(
self,
request,
platform_id,
email,
ip=None,
):
url = 'platform/%s/beneficiaire/sorti/csv' % platform_id
content = self.get(url, email=email, ip=ip)
response = HttpResponse(content_type='text/csv')
date = now().strftime('%Y-%m-%d_%H:%M')
response['Content-Disposition'] = f'attachment; filename="beneficiaire-sorti-{date}.csv"'
# write Unicode BOM
response.write(b'\xef\xbb\xbf')
csv_columns = getattr(
settings, 'RSA13_BENEFICIAIRE_SORTI_CSV_COLUMNS', self.BENEFICIAIRE_SORTI_CSV_COLUMN
)
names = [col if isinstance(col, str) else col[0] for col in csv_columns]
titles = [col if isinstance(col, str) else col[1] for col in csv_columns]
writer = csv.writer(response, delimiter=';')
writer.writerow(titles)
for row in content.get('data') or []:
writer.writerow(str(row.get(name) or '') for name in names)
return response
@endpoint(
name='sous-action',
description=_('Get sub-actions'),
perm='can_access',
parameters=parameters(),
json_schema_response=response_schema(
{
'type': 'array',
'items': {
'type': 'object',
'required': ['id', 'text'],
'properties': {
'id': {
'type': 'string',
},
'text': {
'type': 'string',
},
'description': {
'type': 'string',
},
},
},
}
),
)
def sous_action(self, request, email, ip=None):
return self.get('sousaction/', email=email, ip=ip)
],
}

32
passerelle/contrib/rsa13/templates/rsa13/rsa13resource_detail.html Normal file
View File

@ -0,0 +1,32 @@
{% extends "passerelle/manage/service_view.html" %}
{% load i18n passerelle %}
{% block extra-tab-buttons %}
{% for name, columns in object.description_csv_tables %}
<button role="tab" aria-selected="false" aria-controls="panel-csv-{{name }}" id="tab-csv-{{ name }}" tabindex="-1">{% blocktrans %}{{ name }} CSV columns{% endblocktrans %}</button>
{% endfor %}
{% endblock %}
{% block extra-sections %}
{% for name, columns in object.description_csv_tables %}
<div id="panel-csv-{{ name }}" role="tabpanel" tabindex="-1" aria-labelledby="tab-csv-{{ name }}" hidden>
<table class="main">
<thead>
<tr>
<th>{% trans "Field" %}</th>
<th>{% trans "Title" %}</th>
</tr>
</thead>
<tbody>
{% for field, title in columns %}
<tr>
<td><tt>{{ field }}</tt></td>
<td>{{ title }}</td>
</tr>
{% endfor %}
</tbody>
</table></p>
</div>
{% endfor %}
{% endblock %}

69
tests/test_rsa13.py
View File

@ -22,6 +22,8 @@ from urllib.parse import parse_qs
import httmock
import pytest
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
import tests.utils
from passerelle.contrib.rsa13.models import RSA13Resource
@ -340,7 +342,7 @@ BENEFICIAIRE_CSV = {
{'err': 0, 'data': [BENEFICIAIRE_CSV]},
],
)
def test_platform_beneficiaire_csv(app, rsa13, url, settings):
def test_platform_beneficiaire_csv(app, rsa13, url):
response = app.get(
url + 'platform/11/beneficiaire/csv/',
params={
@ -357,17 +359,18 @@ def test_platform_beneficiaire_csv(app, rsa13, url, settings):
b';1234;1234;;;;;;;;;;;;;;;;;;;;;;;;;;99999\r\n'
)
# configured through settings
settings.RSA13_CSV_COLUMNS = [
['NUM_CAF', 'Numéro CAF'],
]
# customized on model
rsa13.beneficiaire_csv_columns = '''\
NUM_CAF Numéro CAF
VILLE_ADR Ville'''
rsa13.save()
response = app.get(
url + 'platform/11/beneficiaire/csv/',
params={
'query': '1',
},
)
assert response.content == b'\xef\xbb\xbfNum\xc3\xa9ro CAF\r\n1234\r\n'
assert response.content == b'\xef\xbb\xbfNum\xc3\xa9ro CAF;Ville\r\n1234;\r\n'
response = app.get(
url + 'platform/12/beneficiaire/csv/',
@ -968,7 +971,7 @@ def test_platform_facturation_periodes(app, rsa13, url):
},
],
)
def test_platform_facturation_csv(app, rsa13, url, settings):
def test_platform_facturation_csv(app, rsa13, url):
response = app.get(url + 'platform/11/facturation/periodes/')
response = app.get(response.json['data'][1]['csv_url'])
stream = io.StringIO(response.content.decode('utf-8-sig'))
@ -1007,16 +1010,22 @@ def test_platform_facturation_csv(app, rsa13, url, settings):
],
]
settings.RSA13_FACTURATION_CSV_COLUMNS = ['PLATEFORME']
# customized on model
rsa13.facturation_csv_columns = '''\
PLATEFORME
DUREE'''
rsa13.save()
response = app.get(url + 'platform/11/facturation/periodes/')
response = app.get(response.json['data'][1]['csv_url'])
stream = io.StringIO(response.content.decode('utf-8-sig'))
assert list(csv.reader(stream, delimiter=';')) == [
[
'PLATEFORME',
'DUREE',
],
[
'APDL MARTIGUES',
'6',
],
]
@ -1110,7 +1119,7 @@ def test_platform_facturation_csv(app, rsa13, url, settings):
},
],
)
def test_platform_beneficiaire_sorti_csv(app, rsa13, url, settings):
def test_platform_beneficiaire_sorti_csv(app, rsa13, url):
response = app.get(url + 'platform/11/beneficiaire/sorti/csv/')
stream = io.StringIO(response.content.decode('utf-8-sig'))
assert list(csv.reader(stream, delimiter=';')) == [
@ -1232,10 +1241,19 @@ def test_platform_beneficiaire_sorti_csv(app, rsa13, url, settings):
],
]
settings.RSA13_BENEFICIAIRE_SORTI_CSV_COLUMNS = ['NUM_CAF']
# customized on model
rsa13.sorti_csv_columns = '''
NUM_CAF
CODE_PER
'''
rsa13.save()
response = app.get(url + 'platform/11/beneficiaire/sorti/csv/')
stream = io.StringIO(response.content.decode('utf-8-sig'))
assert list(csv.reader(stream, delimiter=';')) == [['NUM_CAF'], ['372927'], ['1677380']]
assert list(csv.reader(stream, delimiter=';')) == [
['NUM_CAF', 'CODE_PER'],
['372927', '415443'],
['1677380', '816754'],
]
@mock_response(
@ -1347,3 +1365,32 @@ def test_sous_action(app, rsa13, url):
],
'err': 0,
}
def test_csv_columns_migration(transactional_db, settings):
migrate_from = [('rsa13', '0001_initial')]
migrate_to = [('rsa13', '0002_add_csv_columns_fields')]
executor = MigrationExecutor(connection)
old_apps = executor.loader.project_state(migrate_from).apps
# state of the db is not important
executor.migrate(migrate_from)
RSA13Resource = old_apps.get_model('rsa13', 'RSA13Resource')
resource = RSA13Resource.objects.create()
settings.RSA13_CSV_COLUMNS = [('A', '2'), 'B']
settings.RSA13_FACTURATION_CSV_COLUMNS = ['C', 'D']
settings.RSA13_BENEFICIAIRE_SORTI_CSV_COLUMNS = ['E', 'F']
executor = MigrationExecutor(connection)
executor.migrate(migrate_to)
executor.loader.build_graph()
apps = executor.loader.project_state(migrate_to).apps
RSA13Resource = apps.get_model('rsa13', 'RSA13Resource')
resource = RSA13Resource.objects.get()
assert resource.beneficiaire_csv_columns == 'A 2\nB'
assert resource.facturation_csv_columns == 'C\nD'
assert resource.sorti_csv_columns == 'E\nF'