passerelle/passerelle/apps/cartads_cs/models.py

1278 lines
50 KiB
Python

# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2018 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import datetime
import json
import os
import random
import re
import zipfile
from ftplib import FTP
from urllib import parse as urlparse
from xml.etree import ElementTree as etree
import pdfrw
import pdfrw.findobjs
import zeep.helpers as zeep_helpers
from Cryptodome.Cipher import AES
from django.conf import settings
from django.core.files.storage import default_storage
from django.core.signing import Signer
from django.db import models
from django.db.models import JSONField
from django.http import HttpResponse
from django.urls import reverse
from django.utils.encoding import force_str
from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _
from passerelle.base.models import BaseResource
from passerelle.base.signature import sign_url
from passerelle.utils.api import endpoint
from passerelle.utils.jsonresponse import APIError, JSONEncoder
from passerelle.utils.soap import SOAPFault
def cartads_file_location(instance, filename):
return 'cartads_cs/%s/%s' % (instance.tracking_code, filename)
def key_value_of_stringstring(d):
return {'KeyValueOfstringstring': [{'Key': x, 'Value': y} for x, y in d.items()]}
class CartaDSDataCache(models.Model):
data_type = models.CharField(max_length=50)
data_parameters = JSONField(default=dict)
data_values = JSONField(default=dict)
last_update_datetime = models.DateTimeField(auto_now=True)
class CartaDSFile(models.Model):
tracking_code = models.CharField(max_length=20)
id_piece = models.CharField(max_length=20)
uploaded_file = models.FileField(upload_to=cartads_file_location)
sent_to_cartads = models.DateTimeField(null=True)
last_update_datetime = models.DateTimeField(auto_now=True)
class Meta:
ordering = ['id']
class CartaDSSubscriber(models.Model):
name_id = models.CharField(max_length=32, null=True)
class CartaDSDossier(models.Model):
email = models.CharField(max_length=256)
name_id = models.CharField(max_length=32, null=True)
tracking_code = models.CharField(max_length=20)
commune_id = models.CharField(max_length=20)
type_dossier_id = models.CharField(max_length=20)
objet_demande_id = models.CharField(max_length=20, null=True)
zip_ready = models.BooleanField(default=False)
zip_sent = models.BooleanField(default=False)
zip_ack_response = models.CharField(null=True, max_length=20)
notification_url = models.URLField(null=True)
notification_message = models.TextField(null=True)
cartads_id_dossier = models.CharField(max_length=50, null=True)
cartads_numero_dossier = models.CharField(max_length=50, null=True)
cartads_cache_code_acces = models.CharField(max_length=200, null=True)
cartads_cache_infos = JSONField(default=dict)
cartads_steps_cache = JSONField(default=dict)
last_update_datetime = models.DateTimeField(auto_now=True)
subscribers = models.ManyToManyField(CartaDSSubscriber, blank=True)
formdata_url = models.CharField(null=True, max_length=200)
deleted = models.BooleanField(default=False)
class AbstractCartaDSCS(BaseResource):
wsdl_base_url = models.URLField(
_('WSDL Base URL'), help_text=_('ex: https://example.net/adscs/webservices/')
)
username = models.CharField(_('Username'), max_length=64)
password = models.CharField(_('Password'), max_length=64)
client_name = models.CharField(
_('Client Name'),
max_length=64,
blank=True,
null=True,
help_text=_('Only useful in shared environments.'),
)
iv = models.CharField(_('Initialisation Vector'), max_length=16)
secret_key = models.CharField(_('Secret Key'), max_length=16)
ftp_server = models.CharField(_('FTP Server'), max_length=128)
ftp_username = models.CharField(_('FTP Username'), max_length=64)
ftp_password = models.CharField(_('FTP Password'), max_length=64)
ftp_client_name = models.CharField(_('FTP Client Name'), max_length=64)
class Meta:
# it is abstract to make it possible for an external connector (@GL) to
# reuse the full connector.
abstract = True
@property
def wsdl_url(self):
return self.get_wsdl_url()
def get_wsdl_url(self, service_type='ServicePortail'):
return self.wsdl_base_url + service_type + '.svc?singleWsdl'
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
if CartaDSDataCache.objects.count() == 0:
# don't wait for daily job to get initial data
self.add_job('update_data_cache')
def soap_client(self, **kwargs):
client = super().soap_client(**kwargs)
# fix URL that should have been changed by reverse proxy
parsed_wsdl_address = urlparse.urlparse(client.service._binding_options['address'])
parsed_real_address = urlparse.urlparse(self.wsdl_base_url)
client.service._binding_options['address'] = urlparse.urlunparse(
parsed_real_address[:2] + parsed_wsdl_address[2:]
)
return client
def get_token(self):
token_data = {
'date': datetime.datetime.now().strftime('%d/%m/%Y %H:%M:%S'),
'login': self.username,
'password': self.password,
}
if self.client_name:
token_data['client'] = self.client_name
token_data_str = json.dumps(token_data)
data_pad = AES.block_size - len(token_data_str) % AES.block_size
aes = AES.new(self.secret_key.encode(), AES.MODE_CBC, self.iv.encode())
token = aes.encrypt((token_data_str + (chr(data_pad) * data_pad)).encode())
return force_str(base64.encodebytes(token)).replace('\n', '').rstrip('=')
def check_status(self):
self.soap_client().service.GetCommunes(self.get_token(), {})
# description of common endpoint parameters
COMMUNE_ID_PARAM = {'description': _('Identifier of collectivity'), 'example_value': '2'}
TYPE_DOSSIER_ID_PARAM = {
'description': _('Identifier of file type'),
'example_value': 'CU',
}
OBJET_DEMANDE_ID_PARAM = {
'description': _('Identifier of demand subject'),
'example_value': '1',
}
TRACKING_CODE_PARAM = {
'description': _('Unique identifier (ex: tracking code)'),
'example_value': 'XCBTFRML',
}
PIECE_ID_PARAM = {
'description': _('Identifier of single file item'),
}
UPLOAD_TOKEN_PARAM = {
'description': _('Token for upload file'),
}
def get_cerfa_pdf(self, url):
# method subclasses can override if the URL returned for PDF documents
# by Cart@DS do not match reverse proxies, API managers, and stuff.
return self.requests.get(url)
def update_data_cache(self):
client = self.soap_client()
# communes
resp = client.service.GetCommunes(self.get_token(), {})
communes_cache, dummy = CartaDSDataCache.objects.get_or_create(data_type='communes')
communes_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
communes_cache.save()
# types dossier
types_dossier_ids = {}
for commune in communes_cache.data_values['data']:
resp = client.service.GetTypesDossier(self.get_token(), int(commune['id']), {})
if resp is None:
continue
data_cache, dummy = CartaDSDataCache.objects.get_or_create(
data_type='types_dossier', data_parameters={'commune_id': int(commune['id'])}
)
data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
types_dossier_ids.update({x['id']: True for x in data_cache.data_values['data']})
data_cache.save()
# objets_demande
types_dossiers_objets_demandes_tuples = []
objets_demande_ids = {}
for type_dossier_id in types_dossier_ids:
resp = client.service.GetObjetsDemande(self.get_token(), type_dossier_id)
if resp is None:
continue
data_cache, dummy = CartaDSDataCache.objects.get_or_create(
data_type='objets_demande', data_parameters={'type_dossier_id': type_dossier_id}
)
data_cache.data_values = {'data': [{'id': str(x['Key']), 'text': x['Value']} for x in resp]}
objets_demande_ids.update({x['id']: True for x in data_cache.data_values['data']})
types_dossiers_objets_demandes_tuples.extend(
[(type_dossier_id, x['id']) for x in data_cache.data_values['data']]
)
data_cache.save()
# liste_pdf
pdfs_path = default_storage.path('public/cartads_cs/%s/documents' % self.slug)
if not os.path.exists(pdfs_path):
os.makedirs(pdfs_path)
for type_compte in [1]:
for type_dossier_id in types_dossier_ids:
resp = client.service.GetListePdf(
self.get_token(), type_dossier_id, {'TypeCompteUtilisateur': type_compte}
)
if resp is None:
continue
def format_cerfa_label(x):
try:
if x['Description']:
return '%(Nom)s: %(Description)s' % x
except KeyError:
pass
return '%(Nom)s' % x
data_cache, dummy = CartaDSDataCache.objects.get_or_create(
data_type='liste_pdf',
data_parameters={
'type_dossier_id': type_dossier_id,
'type_compte': type_compte,
},
)
data_cache.data_values = {
'data': [
{
'id': x['Identifiant'],
'text': format_cerfa_label(x),
'url': x['UrlTelechargement'],
}
for x in resp or []
]
}
for value in data_cache.data_values['data']:
filepath = os.path.join(default_storage.path(self.pdf_path(value)))
resp = self.get_cerfa_pdf(value['url'])
if resp.ok and resp.content.startswith(b'%PDF'):
with open(filepath, 'wb') as fd:
fd.write(resp.content)
data_cache.save()
# pieces
for type_dossier_id, objet_demande_id in types_dossiers_objets_demandes_tuples:
resp = client.service.GetPieces(self.get_token(), type_dossier_id, objet_demande_id)
if resp is None:
continue
data_cache, dummy = CartaDSDataCache.objects.get_or_create(
data_type='pieces',
data_parameters={
'type_dossier_id': type_dossier_id,
'objet_demande_id': str(objet_demande_id),
},
)
if resp is not None:
data_cache.data_values = {
'data': [
{
'id': str(x['IdPiece']),
'text': x['Libelle'],
'description': x['Descriptif'],
'codePiece': x['CodePiece'],
'reglementaire': x['Reglementaire'],
'files': [],
'max_files': 6,
}
for x in resp
]
}
data_cache.save()
def get_dossier_steps(self, client, token, dossier):
resp = client.service.GetEtapesDossier(token, dossier.cartads_id_dossier, [])
steps = []
for step in resp:
step_dict = zeep_helpers.serialize_object(step)
for key, value in step_dict.items():
if isinstance(value, datetime.datetime):
step_dict[key] = value.strftime('%Y-%m-%dT%H:%M:%S')
steps.append(step_dict)
return steps
def update_dossier_cache(self):
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceEtapeDossier'))
client_suivi = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceSuiviNumerique'))
client_dossier = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDossier'))
token = self.get_token()
for dossier in CartaDSDossier.objects.filter(cartads_id_dossier__isnull=False, deleted=False):
try:
dossier.cartads_steps_cache = {'steps': self.get_dossier_steps(client, token, dossier)}
except SOAPFault as e:
if "n'existe pas" in str(e):
dossier.deleted = True
dossier.save()
continue
self.logger.exception('error getting etapes of dossier (%s) (%s)', dossier.id, e)
try:
dossier.cartads_cache_code_acces = client_suivi.service.GetMotPasse(
self.get_token(), dossier.cartads_id_dossier
)
except SOAPFault as e:
self.logger.exception('error getting access code (%s) (%s)', dossier.id, e)
try:
infos_dossier = client_dossier.service.GetInfosDossier(
self.get_token(), dossier.cartads_id_dossier
)
if infos_dossier:
# load(dump(...)) to serialize dates
dossier.cartads_cache_infos = json.loads(
json.dumps(zeep_helpers.serialize_object(infos_dossier), cls=JSONEncoder)
)
except SOAPFault as e:
self.logger.exception('error getting dossier infos (%s) (%s)', dossier.id, e)
dossier.save()
self.sync_subscribers_role(dossier)
def hourly(self):
super().hourly()
self.update_dossier_cache()
def daily(self):
super().daily()
self.update_data_cache()
@endpoint(description=_('Get list of collectivities'), perm='OPEN')
def communes(self, request):
cache = CartaDSDataCache.objects.get(data_type='communes')
return cache.data_values
def get_commune_id(self, commune_name):
for info in self.communes(request=None)['data']:
if info['text'] == commune_name:
return info['id']
return None
def get_commune_label(self, commune_id):
for info in self.communes(request=None)['data']:
if info['id'] == commune_id:
return info['text']
return None
def get_type_dossier_label(self, commune_id, type_dossier_id):
for info in self.types_dossier(request=None, commune_id=commune_id)['data']:
if info['id'] == type_dossier_id:
return info['text']
return None
@endpoint(
description=_('Get list of file types'),
perm='OPEN',
parameters={
'commune_id': COMMUNE_ID_PARAM,
'filter': {
'description': _('List of types to include (separated by commas)'),
'example_value': 'CU,OP',
},
},
)
def types_dossier(self, request, commune_id, filter=None):
cache = CartaDSDataCache.objects.get(
data_type='types_dossier', data_parameters={'commune_id': int(commune_id)}
)
response = cache.data_values
if filter:
filter_list = filter.split(',')
response['data'] = [x for x in response['data'] if x['id'] in filter_list]
return response
@endpoint(
description=_('Get list of demand subjects'),
perm='OPEN',
parameters={'type_dossier_id': TYPE_DOSSIER_ID_PARAM},
)
def objets_demande(self, request, type_dossier_id):
cache = CartaDSDataCache.objects.get(
data_type='objets_demande', data_parameters={'type_dossier_id': type_dossier_id}
)
return cache.data_values
@endpoint(
description=_('Get list of CERFA documents'),
perm='OPEN',
parameters={
'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
'type_compte': {'description': _('Type of account')},
},
)
def liste_pdf(self, request, type_dossier_id, type_compte=1):
cache = CartaDSDataCache.objects.get(
data_type='liste_pdf',
data_parameters={
'type_dossier_id': type_dossier_id,
'type_compte': type_compte,
},
)
if request: # point to local documents cache
for pdf in cache.data_values['data']:
pdf['url'] = request.build_absolute_uri(os.path.join(settings.MEDIA_URL, self.pdf_path(pdf)))
return cache.data_values
def pdf_path(self, pdf):
if '*' in pdf['id']: # cerfa
filename = 'cerfa_%s.pdf' % pdf['id'].replace('*', '-')
else:
filename = '%s.pdf' % pdf['id']
return os.path.join('public/cartads_cs', self.slug, 'documents', filename)
@endpoint(
perm='can_access',
description=_('Get list of file items'),
parameters={
'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
'tracking_code': TRACKING_CODE_PARAM,
'demolitions': {
'description': _('Include items for demolition work'),
'example_value': 'false',
'type': 'bool',
},
},
)
def pieces(self, request, type_dossier_id, objet_demande_id, tracking_code, demolitions=True):
cache, dummy = CartaDSDataCache.objects.get_or_create(
data_type='pieces',
data_parameters={
'type_dossier_id': type_dossier_id,
'objet_demande_id': objet_demande_id,
},
)
signer = Signer(salt='cart@ds_cs')
upload_token = signer.sign(tracking_code)
cerfa_pieces = [
{
'id': 'cerfa-%s-%s' % (type_dossier_id, objet_demande_id),
'text': 'Cerfa rempli',
'description': '',
'codePiece': '',
'reglementaire': True,
'files': [],
'max_files': 1,
'section_start': 'Cerfa',
},
{
'id': 'cerfa-autres-%s-%s' % (type_dossier_id, objet_demande_id),
'text': 'Cerfa demandeurs complémentaires',
'description': '',
'codePiece': '',
'reglementaire': False,
'files': [],
'max_files': 6,
},
]
pieces = cache.data_values['data'] if cache.data_values else []
if demolitions is False:
def is_demolition_piece(piece):
if piece['reglementaire']:
return False
for demolition_prefix in ('PCA', 'PCMIA'):
if re.match(r'^%s\d' % demolition_prefix, piece['codePiece']):
return True
return False
pieces = [x for x in pieces if not is_demolition_piece(x)]
required_pieces = [x for x in pieces if x['reglementaire']]
if required_pieces:
required_pieces[0]['section_start'] = 'Pièces réglementaires'
optional_pieces = [x for x in pieces if not x['reglementaire']]
if optional_pieces:
optional_pieces[0]['section_start'] = 'Pièces spécifiques'
pieces = cerfa_pieces + required_pieces + optional_pieces
known_files = CartaDSFile.objects.filter(tracking_code=tracking_code)
for piece in pieces:
if request:
upload_url = request.build_absolute_uri(
'%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
)
else:
upload_url = None
piece['files'] = [
{
'url': upload_url,
'name': os.path.basename(x.uploaded_file.name),
'token': signer.sign(str(x.id)),
'id': x.id,
}
for x in known_files
if x.id_piece == str(piece['id'])
]
if len(piece['files']) < piece['max_files']:
piece['files'].append({'url': upload_url})
return {'data': pieces}
@endpoint(
perm='can_access',
description=_('Check list of file items'),
parameters={
'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
'tracking_code': TRACKING_CODE_PARAM,
},
)
def check_pieces(self, request, type_dossier_id, objet_demande_id, tracking_code):
pieces = self.pieces(request, type_dossier_id, objet_demande_id, tracking_code)
result = True
for piece in pieces['data']:
if not piece['reglementaire']:
continue
if not [x for x in piece['files'] if x.get('name')]:
result = False
break
return {'result': result}
@endpoint(
perm='can_access',
description=_('Get list of additional file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def additional_pieces(self, request, tracking_code):
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
resp = client.service.GetPiecesDossierACompleter(self.get_token(), dossier.cartads_id_dossier)
if resp is None:
return {'data': []}
signer = Signer(salt='cart@ds_cs')
upload_token = signer.sign(tracking_code)
pieces = [
{
'id': 'comp-%s-%s' % (x['IdDosPiece'], x['IdPiece']),
'text': x['LibellePiece'],
'description': x['Descriptif'],
'codePiece': x['CodePiece'],
'files': [],
'max_files': 6,
}
for x in resp
]
for piece in pieces:
if request:
upload_url = request.build_absolute_uri(
'%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
)
else:
upload_url = None
piece['files'] = []
piece['files'].append({'url': upload_url})
return {'data': pieces}
@endpoint(
perm='can_access',
description=_('Get list of DOC file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def doc_pieces(self, request, tracking_code):
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
status = self.status(request, dossier.id)
if status['status_label'] != 'Attente DOC':
raise APIError('wrong status')
signer = Signer(salt='cart@ds_cs')
upload_token = signer.sign(tracking_code)
pieces = [
{
'id': 'cerfa-doc',
'text': 'CERFA',
'description': '',
'codePiece': '',
'reglementaire': True,
'files': [],
'max_files': 1,
},
]
for piece in pieces:
if request:
upload_url = request.build_absolute_uri(
'%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
)
else:
upload_url = None
piece['files'].append({'url': upload_url})
return {'data': pieces}
@endpoint(
perm='can_access',
description=_('Get list of DAACT file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def daact_pieces(self, request, tracking_code):
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
status = self.status(request, dossier.id)
if status['status_label'] != 'Attente DAACT':
raise APIError('wrong status')
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
resp = client.service.GetPiecesDaact(self.get_token(), dossier.cartads_id_dossier)
signer = Signer(salt='cart@ds_cs')
upload_token = signer.sign(tracking_code)
pieces = [
{
'id': 'cerfa-daact',
'text': 'CERFA',
'description': '',
'codePiece': '',
'reglementaire': True,
'files': [],
'max_files': 1,
},
]
pieces.extend(
[
{
'id': 'daact-%s' % x['IdPiece'],
'text': x['LibellePiece'],
'description': x['Descriptif'],
'codePiece': x['CodePiece'],
'files': [],
'max_files': 6,
}
for x in resp
]
)
for piece in pieces:
if request:
upload_url = request.build_absolute_uri(
'%supload/%s/%s/' % (self.get_absolute_url(), piece['id'], upload_token)
)
else:
upload_url = None
piece['files'] = []
piece['files'].append({'url': upload_url})
return {'data': pieces}
@endpoint(
methods=['post'],
pattern=r'^(?P<id_piece>[\w-]+)/(?P<token>[\w:_-]+)/$',
perm='OPEN',
description=_('Upload a single document file'),
parameters={
'id_piece': PIECE_ID_PARAM,
'token': UPLOAD_TOKEN_PARAM,
},
)
def upload(self, request, id_piece, token, **kwargs):
if not request.FILES.get('files[]'):
# silently ignore request without files
return []
signer = Signer(salt='cart@ds_cs')
tracking_code = signer.unsign(token)
if id_piece.startswith('cerfa-'):
try:
pdf = pdfrw.PdfReader(request.FILES['files[]'])
if not any(pdfrw.findobjs.find_objects(pdf, valid_subtypes=(pdfrw.PdfName.Form,))):
return [{'error': force_str(_('The CERFA should not be a scanned document.'))}]
except (pdfrw.PdfParseError, ValueError):
return [{'error': force_str(_('The CERFA should be a PDF file.'))}]
else:
if request.FILES['files[]'].content_type not in ('application/pdf', 'image/jpeg'):
return [{'error': force_str(_('The file should be a PDF document or a JPEG image.'))}]
if request.FILES['files[]'].size > 25 * 1024 * 1024:
return [{'error': force_str(_('The file should not exceed 25MB.'))}]
file_upload = CartaDSFile(
tracking_code=tracking_code, id_piece=id_piece, uploaded_file=request.FILES['files[]']
)
file_upload.save()
return [
{
'name': os.path.basename(file_upload.uploaded_file.name),
'token': signer.sign(str(file_upload.id)),
}
]
@endpoint(
methods=['post'],
name='upload',
perm='OPEN',
pattern=r'^(?P<id_piece>[\w-]+)/(?P<token>[\w:_-]+)/(?P<file_upload>[\w:_-]+)/delete/$',
description=_('Delete a single document file'),
parameters={
'id_piece': PIECE_ID_PARAM,
'token': UPLOAD_TOKEN_PARAM,
'file_upload': {
'description': _('Signed identifier of single document upload'),
},
},
)
def upload_delete(self, request, id_piece, token, file_upload, **kwargs):
# this cannot be verb DELETE as we have no way to set
# Access-Control-Allow-Methods
signer = Signer(salt='cart@ds_cs')
CartaDSFile.objects.filter(id=signer.unsign(file_upload)).delete()
return {'err': 0}
@endpoint(
perm='can_access',
description=_('Validate and send a file'),
parameters={
'commune_id': COMMUNE_ID_PARAM,
'type_dossier_id': TYPE_DOSSIER_ID_PARAM,
'objet_demande_id': OBJET_DEMANDE_ID_PARAM,
'tracking_code': TRACKING_CODE_PARAM,
'email': {
'description': _('Email of requester'),
},
'name_id': {
'description': _('UUID of requester'),
},
'formdata_url': {
'description': _('URL of user form'),
},
},
)
def send(
self,
request,
commune_id,
type_dossier_id,
objet_demande_id,
tracking_code,
email,
name_id=None,
formdata_url=None,
):
dossier = CartaDSDossier(
commune_id=commune_id,
type_dossier_id=type_dossier_id,
objet_demande_id=objet_demande_id,
tracking_code=tracking_code,
email=email,
formdata_url=formdata_url,
)
dossier.save()
signer = Signer(salt='cart@ds_cs/dossier')
notification_base_url = reverse(
'generic-endpoint',
kwargs={'connector': self.get_connector_slug(), 'slug': self.slug, 'endpoint': 'notification'},
)
dossier.notification_url = request.build_absolute_uri(
notification_base_url + '/%s/' % signer.sign(str(dossier.id))
)
dossier.save()
if name_id:
dossier.subscribers.add(CartaDSSubscriber.objects.get_or_create(name_id=name_id)[0])
self.add_job('pack', dossier_id=dossier.id)
return {'err': 0, 'dossier_id': dossier.id, 'tracking_code': dossier.tracking_code}
def pack(self, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
zip_filename = os.path.join(default_storage.path('cartads_cs'), '%s.zip' % dossier.tracking_code)
with zipfile.ZipFile(zip_filename, mode='w') as zip_file:
liste_pdf = self.liste_pdf(None, dossier.type_dossier_id)
cerfa_id = liste_pdf['data'][0]['id']
for cerfa in liste_pdf['data']:
if cerfa['id'] == 'AUTRES_DEMANDEURS':
continue
cerfa_id = cerfa['id']
break
cerfa_id = cerfa_id.replace('*', '-')
pieces = self.pieces(
None, dossier.type_dossier_id, dossier.objet_demande_id, dossier.tracking_code
)
for piece in pieces['data']:
cnt = 1
for file in piece['files']:
if not file.get('id'):
continue
cartads_file = CartaDSFile.objects.get(id=file['id'])
if piece['id'] == 'cerfa-%s-%s' % (dossier.type_dossier_id, dossier.objet_demande_id):
zip_file.write(cartads_file.uploaded_file.path, '%s.pdf' % cerfa_id)
elif piece['id'].startswith('cerfa-autres-'):
zip_file.write(
cartads_file.uploaded_file.path,
'Fiches_complementaires/Cerfa_autres_demandeurs_%d.pdf' % cnt,
)
else:
zip_file.write(
cartads_file.uploaded_file.path,
'Pieces/%s-%s%s%s'
% (
piece['id'],
piece['codePiece'],
cnt,
os.path.splitext(cartads_file.uploaded_file.path)[-1],
),
)
cnt += 1
dossier.zip_ready = True
dossier.save()
self.add_job('send_to_cartads', dossier_id=dossier.id)
def upload_zip(self, zip_filename):
ftp = FTP(self.ftp_server)
ftp.login(self.ftp_username, self.ftp_password)
ftp.cwd(self.ftp_client_name)
with open(zip_filename, 'rb') as fd:
ftp.storbinary('STOR %s' % os.path.basename(zip_filename), fd)
ftp.quit()
def send_to_cartads(self, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
zip_filename = os.path.join(default_storage.path('cartads_cs'), '%s.zip' % dossier.tracking_code)
self.upload_zip(zip_filename)
client = self.soap_client()
resp = client.service.NotifierDepotDossier(
self.get_token(),
dossier.commune_id,
dossier.type_dossier_id,
os.path.basename(zip_filename),
dossier.email,
key_value_of_stringstring(
{
'NotificationMailDemandeur': '0',
'IdDossierExterne': 'publik-%s-%s' % (dossier.id, dossier.tracking_code),
'NumeroDossierExterne': 'publik-%s-%s' % (dossier.id, dossier.tracking_code),
'TraitementImmediat': '0',
'UrlNotification': dossier.notification_url,
}
),
)
dossier.zip_sent = True
dossier.zip_ack_response = str(resp)
dossier.save()
CartaDSFile.objects.filter(tracking_code=dossier.tracking_code).update(sent_to_cartads=now())
self.sync_subscribers_role(dossier)
@endpoint(
pattern=r'^(?P<signed_dossier_id>[\w:_-]+)/$',
perm='OPEN',
methods=['post'],
description=_('Notification of file processing by Cart@DS CS'),
parameters={
'signed_dossier_id': {'description': _('Signed identifier of file')},
},
)
def notification(self, request, signed_dossier_id):
signer = Signer(salt='cart@ds_cs/dossier')
dossier_id = signer.unsign(signed_dossier_id)
dossier = CartaDSDossier.objects.get(id=dossier_id)
dossier.notification_message = request.POST['notification']
notification = etree.fromstring(dossier.notification_message.encode('utf-8'))
if notification.find('Succes').text == 'true':
dossier.cartads_id_dossier = notification.find(
'InformationsComplementaires/IdDossierCartads'
).text
dossier.cartads_numero_dossier = notification.find(
'InformationsComplementaires/NumeroDossier'
).text
self.sync_subscribers_role(dossier)
dossier.save()
return HttpResponse('ok', content_type='text/plain')
@endpoint(
perm='can_access',
description=_('Send requested additional file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def send_additional_pieces(self, request, tracking_code):
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
self.add_job('send_additional_pieces_to_cartads', dossier_id=dossier.id)
return {'err': 0, 'dossier_id': dossier.id}
def send_additional_pieces_to_cartads(self, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServicePiece'))
resp = client.service.GetPiecesDossierACompleter(self.get_token(), dossier.cartads_id_dossier)
pieces = [
{
'id': 'comp-%s-%s' % (x['IdDosPiece'], x['IdPiece']),
'idPiece': x['IdPiece'],
'codePiece': x['CodePiece'],
}
for x in resp
]
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))
for piece_type in pieces:
for i, piece in enumerate(
CartaDSFile.objects.filter(tracking_code=dossier.tracking_code, id_piece=piece_type['id'])
):
if piece.sent_to_cartads:
continue
id_dos_piece = piece.id_piece.split('-')[1]
filename = '%s-%s%s%s' % (
piece_type['idPiece'],
piece_type['codePiece'],
'%03d' % (i + 1),
os.path.splitext(piece.uploaded_file.name)[-1],
)
content = piece.uploaded_file.read()
try:
resp = client.service.UploadFile(
FileByteStream=content,
_soapheaders={
'IdDossier': dossier.cartads_id_dossier,
'NomFichier': filename,
'Length': piece.uploaded_file.size,
'Token': self.get_token(),
'InformationsComplementaires': key_value_of_stringstring(
{'idDosPiece': id_dos_piece}
),
},
)
except SOAPFault as e:
self.logger.exception('error pushing file item %d (%s)', piece.id, e)
continue
else:
assert resp is None
piece.sent_to_cartads = now()
piece.save()
@endpoint(
perm='can_access',
description=_('Send DOC file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def send_doc_pieces(self, request, tracking_code):
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
self.add_job('send_doc_pieces_to_cartads', dossier_id=dossier.id)
return {'err': 0, 'dossier_id': dossier.id}
def send_doc_pieces_to_cartads(self, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))
pieces = CartaDSFile.objects.filter(
tracking_code=dossier.tracking_code, id_piece='cerfa-doc', sent_to_cartads__isnull=True
)
assert pieces.count() == 1
piece = pieces[0]
content = piece.uploaded_file.read()
try:
resp = client.service.UploadFile(
FileByteStream=content,
_soapheaders={
'IdDossier': dossier.cartads_id_dossier,
'NomFichier': 'cerfa-doc.pdf',
'Length': piece.uploaded_file.size,
'Token': self.get_token(),
'InformationsComplementaires': key_value_of_stringstring(
{'docDaact': 'doc', 'renameFile': 'true'},
),
},
)
except SOAPFault as e:
self.logger.exception('error pushing file item %d (%s)', piece.id, e)
else:
assert resp is None
piece.sent_to_cartads = now()
piece.save()
@endpoint(
perm='can_access',
description=_('Send DAACT file items'),
parameters={
'tracking_code': TRACKING_CODE_PARAM,
},
)
def send_daact_pieces(self, request, tracking_code):
dossier = CartaDSDossier.objects.get(tracking_code=tracking_code)
self.add_job('send_daact_pieces_to_cartads', dossier_id=dossier.id)
return {'err': 0, 'dossier_id': dossier.id}
def send_daact_pieces_to_cartads(self, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
pieces = self.daact_pieces(None, dossier.tracking_code)['data']
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceDocumentation'))
for piece_type in pieces:
for piece in CartaDSFile.objects.filter(
tracking_code=dossier.tracking_code, sent_to_cartads__isnull=True, id_piece=piece_type['id']
):
content = piece.uploaded_file.read()
try:
infos = {
'renameFile': 'true',
}
if piece.id_piece == 'cerfa-daact':
infos['docDaact'] = 'daact'
filename = 'cerfa-daact.pdf'
else:
infos['docDaact'] = 'pieceDaact'
infos['idPieceDaact'] = piece.id_piece.split('-', 1)[-1]
filename = '%s%s' % (
piece_type['codePiece'],
os.path.splitext(piece.uploaded_file.name)[-1],
)
resp = client.service.UploadFile(
FileByteStream=content,
_soapheaders={
'IdDossier': dossier.cartads_id_dossier,
'NomFichier': filename,
'Length': piece.uploaded_file.size,
'Token': self.get_token(),
'InformationsComplementaires': key_value_of_stringstring(infos),
},
)
except SOAPFault as e:
self.logger.exception('error pushing daact file item %d (%s)', piece.id, e)
else:
assert resp is None
piece.sent_to_cartads = now()
piece.save()
def get_file_status(self, dossier):
response = {}
if dossier.deleted:
status_id = 'deleted'
status_label = _('Deleted')
elif dossier.cartads_id_dossier:
if dossier.cartads_steps_cache:
steps = dossier.cartads_steps_cache['steps']
else:
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceEtapeDossier'))
steps = self.get_dossier_steps(client, self.get_token(), dossier)
dossier.cartads_steps_cache['steps'] = steps
dossier.save()
steps.sort(key=lambda x: x['DateReference'])
status_id = 'cartads-%s' % steps[-1]['IdEtape']
status_label = steps[-1]['LibelleEtape']
response['extra'] = {}
for key in steps[-1]:
response['extra'][key] = steps[-1][key]
response['cartads_reference_dossier'] = dossier.cartads_numero_dossier
response['cartads_code_acces'] = dossier.cartads_cache_code_acces
elif dossier.notification_message: # but not dossier id -> error
status_id = 'refused'
notification = etree.fromstring(dossier.notification_message.encode('utf-8'))
error = notification.find('InformationsComplementaires/MessageErreur').text
status_label = _('File refused (%s)') % error
elif dossier.zip_sent:
status_id = 'zip-sent'
status_label = _('File sent')
if dossier.zip_ack_response == 'False':
status_id = 'zip-not-considered'
status_label = _('File not considered')
elif dossier.zip_ready:
status_id = 'zip-ready'
status_label = _('File ready to be sent')
else:
status_id = 'pending'
status_label = _('Pending')
response.update({'status_id': status_id, 'status_label': status_label})
return response
@endpoint(
perm='can_access',
description=_('Get status of file'),
parameters={
'dossier_id': {
'description': _('Identifier of file'),
}
},
)
def status(self, request, dossier_id):
dossier = CartaDSDossier.objects.get(id=dossier_id)
return self.get_file_status(dossier)
@endpoint(
perm='can_access',
description=_('Get list of files attached to user'),
parameters={
'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
'status': {
'description': _('File Status'),
'example_value': 'Attente DOC',
},
},
)
def files(self, request, name_id, status=None):
files = CartaDSDossier.objects.filter(subscribers__name_id__in=[name_id])
if status:
files = [
x
for x in files
if self.get_file_status(x).get('status_id') == status
or self.get_file_status(x).get('status_label') == status
]
def get_date(dossier):
if dossier.cartads_cache_infos and dossier.cartads_cache_infos['DateDepot']:
return dossier.cartads_cache_infos['DateDepot']
return ''
files = list(files)
files.sort(key=get_date)
return {
'data': [
{
'id': str(x.id),
'text': x.cartads_numero_dossier,
'tracking_code': x.tracking_code,
'status': self.get_file_status(x),
'commune_label': self.get_commune_label(x.commune_id),
'type_dossier_label': self.get_type_dossier_label(x.commune_id, x.type_dossier_id),
'formdata_url': x.formdata_url,
'cartads_infos': x.cartads_cache_infos,
}
for x in files
]
}
@endpoint(
perm='can_access',
description=_('Join dossier'),
parameters={
'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
'dossier_number': {
'description': _('Dossier Number'),
'example_value': 'PC 069 012 23 45678',
},
'dossier_password': {
'description': _('Dossier Password'),
'example_value': '5A3E36FE-80D3-45E5-9323-7415E04D3B14',
},
'formdata_url': {
'description': _('URL of user form'),
},
},
)
def join(self, request, name_id, dossier_number, dossier_password, formdata_url=None):
client = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceSuiviNumerique'))
try:
resp = client.service.ActiverServiceSuiviNumerique(
self.get_token(), dossier_number, dossier_password
)
except SOAPFault as e:
self.logger.error('error joining dossier %s (%s)', dossier_number, e)
raise APIError('error joining dossier (wrong password?)')
id_dossier = int(resp)
dossier, created = CartaDSDossier.objects.get_or_create(cartads_id_dossier=id_dossier)
if created:
dossier.cartads_numero_dossier = dossier_number
client_dossier = self.soap_client(wsdl_url=self.get_wsdl_url('ServiceRechercheDossier'))
infos = client_dossier.service.GetInfosDossier(self.client_name, id_dossier)
dossier.type_dossier_id = infos['CoTypeDossier']
dossier.commune_id = self.get_commune_id(infos['Commune'])
dossier.formdata_url = formdata_url
CHARS = 'BCDFGHJKLMNPQRSTVWXZ'
r = random.SystemRandom()
dossier.tracking_code = 'A-' + ''.join([r.choice(CHARS) for x in range(8)])
dossier.save()
dossier.subscribers.add(CartaDSSubscriber.objects.get_or_create(name_id=name_id)[0])
self.sync_subscribers_role(dossier)
return {
'err': 0,
'dossier_id': dossier.id,
'formdata_url': dossier.formdata_url,
'tracking_code': dossier.tracking_code,
}
@endpoint(
perm='can_access',
description=_('Unsubscribe from dossier'),
parameters={
'name_id': {'description': _('UUID of requester'), 'example_value': '3eb56fc'},
'dossier_number': {
'description': _('Dossier Number'),
'example_value': 'PC 069 012 23 45678',
},
},
)
def unsubscribe(self, request, name_id, dossier_number):
try:
dossier = CartaDSDossier.objects.get(cartads_numero_dossier=dossier_number)
except CartaDSDossier.DoesNotExist:
raise APIError('dossier does not exist')
try:
subscriber = CartaDSSubscriber.objects.get(name_id=name_id)
except CartaDSSubscriber.DoesNotExist:
raise APIError('subscriber does not exist')
if subscriber not in dossier.subscribers.all():
raise APIError('subscriber not subscribed to that dossier')
dossier.subscribers.remove(subscriber)
self.sync_subscribers_role(dossier)
return {'err': 0, 'dossier_id': dossier.id}
def sync_subscribers_role(self, dossier):
if not getattr(settings, 'KNOWN_SERVICES', {}).get('authentic'):
return
idp_service = list(settings.KNOWN_SERVICES['authentic'].values())[0]
# sync subscribers with an authentic role, this can fail and it will
# be retried again later.
role_api_url = sign_url(
urlparse.urljoin(
idp_service['url'], 'api/roles/?get_or_create=slug&orig=%s' % idp_service.get('orig')
),
key=idp_service.get('secret'),
)
response = self.requests.post(
role_api_url,
json={
'name': 'Suivi Cart@DS (%s)' % dossier.id,
'slug': '_cartads_%s' % dossier.id,
},
)
if response.status_code != 200:
return
try:
role_uuid = response.json()['uuid']
except (KeyError, TypeError, ValueError):
return
role_api_url = sign_url(
urlparse.urljoin(
idp_service['url'],
'api/roles/%s/relationships/members/?orig=%s' % (role_uuid, idp_service.get('orig')),
),
key=idp_service.get('secret'),
)
response = self.requests.patch(
role_api_url, json={'data': [{'uuid': x.name_id} for x in dossier.subscribers.all()]}
)
class CartaDSCS(AbstractCartaDSCS):
category = _('Misc')
class Meta:
verbose_name = 'Cart@DS CS'