362 lines
14 KiB
Python
362 lines
14 KiB
Python
# Passerelle - uniform access to data and services
|
|
# Copyright (C) 2019 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
import inspect
|
|
import os
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from collections import namedtuple
|
|
|
|
from django.db import IntegrityError, models, transaction
|
|
from django.db.models import JSONField
|
|
from django.http import HttpResponse
|
|
from django.urls import reverse
|
|
from django.utils import functional
|
|
from django.utils.translation import gettext_lazy as _
|
|
|
|
from passerelle.base.models import BaseResource, SkipJob
|
|
from passerelle.utils import sftp, xml
|
|
from passerelle.utils.api import endpoint
|
|
from passerelle.utils.conversion import exception_to_text
|
|
from passerelle.utils.jsonresponse import APIError
|
|
from passerelle.utils.zip import ZipTemplate
|
|
|
|
# Base abstract models for implementing MDEL compatible requests.
|
|
|
|
MDELStatus = namedtuple('MDELStatus', ['code', 'slug', 'label'])
|
|
|
|
MDEL_STATUSES = map(
|
|
lambda t: MDELStatus(*t),
|
|
[
|
|
('100', 'closed', _('closed')),
|
|
('20', 'rejected', _('rejected')),
|
|
('19', 'accepted', _('accepted')),
|
|
('17', 'information needed', _('information needed')),
|
|
('16', 'in progress', _('in progress')),
|
|
('15', 'invalid', _('invalid')),
|
|
('14', 'imported', _('imported')),
|
|
],
|
|
)
|
|
|
|
MDEL_STATUSES_BY_CODE = {mdel_status.code: mdel_status for mdel_status in MDEL_STATUSES}
|
|
|
|
|
|
class Resource(BaseResource):
|
|
outgoing_sftp = sftp.SFTPField(
|
|
verbose_name=_('Outgoing SFTP'),
|
|
blank=True,
|
|
help_text=_('MDEL request .zip will be pushed to.'),
|
|
)
|
|
incoming_sftp = sftp.SFTPField(
|
|
verbose_name=_('Incoming SFTP'),
|
|
blank=True,
|
|
help_text=_('MDEL response .zip will be pulled from.'),
|
|
)
|
|
recipient_siret = models.CharField(verbose_name=_('SIRET'), max_length=128)
|
|
recipient_service = models.CharField(verbose_name=_('Service'), max_length=128)
|
|
recipient_guichet = models.CharField(verbose_name=_('Guichet'), max_length=128)
|
|
code_insee = models.CharField(verbose_name=_('INSEE Code'), max_length=6)
|
|
|
|
xsd_path = 'schema.xsd'
|
|
xsd_root_element = None
|
|
flow_type = 'flow_type CHANGEME'
|
|
doc_type = 'doc_type CHANGEME'
|
|
zip_manifest = 'mdel/zip/manifest.json'
|
|
code_insee_id = 'CODE_INSEE'
|
|
xmlschema_class = xml.JSONSchemaFromXMLSchema
|
|
|
|
class Meta:
|
|
abstract = True
|
|
|
|
def check_status(self):
|
|
if self.outgoing_sftp:
|
|
with self.outgoing_sftp.client() as out_sftp:
|
|
out_sftp.listdir()
|
|
if self.incoming_sftp:
|
|
with self.incoming_sftp.client() as in_sftp:
|
|
in_sftp.listdir()
|
|
|
|
@classmethod
|
|
def get_doc_xml_schema(cls):
|
|
base_dir = os.path.dirname(inspect.getfile(cls))
|
|
path = os.path.join(base_dir, cls.xsd_path)
|
|
assert os.path.exists(path)
|
|
return cls.xmlschema_class(path, cls.xsd_root_element)
|
|
|
|
@classmethod
|
|
def get_doc_json_schema(cls):
|
|
return cls.get_doc_xml_schema().json_schema
|
|
|
|
@classmethod
|
|
def get_create_schema(cls):
|
|
base_schema = cls.get_doc_json_schema()
|
|
base_schema['unflatten'] = True
|
|
base_schema['merge_extra'] = True
|
|
base_schema['properties'].update(
|
|
{
|
|
'display_id': {'type': 'string'},
|
|
'email': {'type': 'string'},
|
|
'code_insee': {'type': 'string'},
|
|
}
|
|
)
|
|
base_schema.setdefault('required', []).append('display_id')
|
|
if hasattr(cls, 'pre_process_create'):
|
|
base_schema['pre_process'] = cls.pre_process_create
|
|
return base_schema
|
|
|
|
def _handle_create(self, request, payload):
|
|
reference = 'A-' + payload['display_id']
|
|
with transaction.atomic():
|
|
try:
|
|
demand = self.demand_set.create(reference=reference, step=1, data=payload)
|
|
except IntegrityError as e:
|
|
raise APIError(
|
|
'reference-non-unique', http_status=400, data={'original_exc': exception_to_text(e)}
|
|
)
|
|
self.add_job('push_demand', demand_id=demand.id)
|
|
return self.status(request, demand)
|
|
|
|
def push_demand(self, demand_id):
|
|
demand = self.demand_set.get(id=demand_id)
|
|
if not demand.push():
|
|
raise SkipJob(after_timestamp=3600 * 6)
|
|
|
|
@endpoint(
|
|
perm='can_access', methods=['get'], description=_('Demand status'), pattern=r'(?P<demand_id>\d+)/$'
|
|
)
|
|
def demand(self, request, demand_id):
|
|
try:
|
|
demand = self.demand_set.get(id=demand_id)
|
|
except self.demand_set.model.DoesNotExist:
|
|
raise APIError('demand not found', http_status=404)
|
|
return self.status(request, demand)
|
|
|
|
def status(self, request, demand):
|
|
return {
|
|
'id': demand.id,
|
|
'status': demand.status,
|
|
'url': request.build_absolute_uri(demand.status_url),
|
|
'zip_url': request.build_absolute_uri(demand.zip_url),
|
|
}
|
|
|
|
@endpoint(
|
|
perm='can_access',
|
|
methods=['get'],
|
|
description=_('Demand document'),
|
|
pattern=r'(?P<demand_id>\d+)/.*$',
|
|
)
|
|
def document(self, request, demand_id):
|
|
try:
|
|
demand = self.demand_set.get(id=demand_id)
|
|
except self.demand_set.model.DoesNotExist:
|
|
raise APIError('demand not found', http_status=404)
|
|
response = HttpResponse(demand.zip_content, content_type='application/octet-stream')
|
|
response['Content-Disposition'] = 'inline; filename=%s' % demand.zip_name
|
|
return response
|
|
|
|
@property
|
|
def response_re(self):
|
|
return re.compile(r'(?P<reference>[^-]+-[^-]+-[^-]+)-%s-' r'(?P<step>\d+).zip' % self.flow_type)
|
|
|
|
def hourly(self):
|
|
'''Get responses'''
|
|
if not self.incoming_sftp:
|
|
return
|
|
try:
|
|
with self.incoming_sftp.client() as client:
|
|
for name in client.listdir():
|
|
m = self.response_re.match(name)
|
|
if not m:
|
|
self.logger.warning(
|
|
'pull responses: unexpected file "%s" in sftp, file name does not match pattern %s',
|
|
name,
|
|
self.response_re,
|
|
)
|
|
continue
|
|
reference = m.groupdict()['reference']
|
|
step = int(m.groupdict()['step'])
|
|
demand = self.demand_set.filter(reference=reference).first()
|
|
if not demand:
|
|
self.logger.error(
|
|
'pull responses: unexpected file "%s" in sftp, no demand for reference "%s"',
|
|
name,
|
|
reference,
|
|
)
|
|
continue
|
|
if step < demand.step:
|
|
demand.logger.error(
|
|
'pull responses: unexpected file "%s" in sftp: step %s is inferior to demand step %s',
|
|
name,
|
|
step,
|
|
demand.step,
|
|
)
|
|
continue
|
|
demand.handle_response(sftp_client=client, filename=name, step=step)
|
|
except sftp.paramiko.SSHException as e:
|
|
self.logger.error('pull responses: sftp error %s', e)
|
|
return
|
|
|
|
|
|
class Demand(models.Model):
|
|
STATUS_PENDING = 'pending'
|
|
STATUS_PUSHED = 'pushed'
|
|
STATUS_ERROR = 'error'
|
|
|
|
STATUSES = [
|
|
(STATUS_PENDING, _('pending')),
|
|
(STATUS_PUSHED, _('pushed')),
|
|
(STATUS_ERROR, _('error')),
|
|
]
|
|
for mdel_status in MDEL_STATUSES:
|
|
STATUSES.append((mdel_status.slug, mdel_status.label))
|
|
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
updated_at = models.DateTimeField(auto_now=True)
|
|
reference = models.CharField(max_length=32, null=False, unique=True)
|
|
status = models.CharField(max_length=32, null=True, choices=STATUSES, default=STATUS_PENDING)
|
|
step = models.IntegerField(default=0)
|
|
data = JSONField()
|
|
|
|
@functional.cached_property
|
|
def logger(self):
|
|
return self.resource.logger.context(
|
|
demand_id=self.id, demand_status=self.status, demand_reference=self.reference
|
|
)
|
|
|
|
def push(self):
|
|
if not self.resource.outgoing_sftp:
|
|
return False
|
|
try:
|
|
with self.resource.outgoing_sftp.client() as client:
|
|
with client.open(self.zip_name, mode='w') as fd:
|
|
fd.write(self.zip_content)
|
|
except sftp.paramiko.SSHException as e:
|
|
self.logger.error('push demand: %s failed, "%s"', self, exception_to_text(e))
|
|
self.status = self.STATUS_ERROR
|
|
except Exception as e:
|
|
self.logger.exception('push demand: %s failed, "%s"', self, exception_to_text(e))
|
|
self.status = self.STATUS_ERROR
|
|
else:
|
|
self.resource.logger.info('push demand: %s success', self)
|
|
self.status = self.STATUS_PUSHED
|
|
self.save()
|
|
return True
|
|
|
|
@functional.cached_property
|
|
def zip_template(self):
|
|
return ZipTemplate(
|
|
self.resource.zip_manifest,
|
|
ctx={
|
|
'reference': self.reference,
|
|
'flow_type': self.resource.flow_type,
|
|
'doc_type': self.resource.doc_type,
|
|
'step': '1', # We never create more than one document for a reference
|
|
'siret': self.resource.recipient_siret,
|
|
'service': self.resource.recipient_service,
|
|
'guichet': self.resource.recipient_guichet,
|
|
'code_insee': self.data.get('code_insee', self.resource.code_insee),
|
|
'document': self.document,
|
|
'code_insee_id': self.resource.code_insee_id,
|
|
'date': self.created_at.isoformat(),
|
|
'email': self.data.get('email', ''),
|
|
},
|
|
)
|
|
|
|
@property
|
|
def zip_name(self):
|
|
return self.zip_template.name
|
|
|
|
@property
|
|
def zip_content(self):
|
|
return self.zip_template.render_to_bytes()
|
|
|
|
@property
|
|
def document(self):
|
|
xml_schema = self.resource.get_doc_xml_schema()
|
|
node = xml_schema.encode(self.data)
|
|
node.attrib['xmlns:xs'] = 'http://www.w3.org/2001/XMLSchema'
|
|
return b'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n' + ET.tostring(node)
|
|
|
|
@property
|
|
def status_url(self):
|
|
return reverse(
|
|
'generic-endpoint',
|
|
kwargs={
|
|
'connector': self.resource.get_connector_slug(),
|
|
'slug': self.resource.slug,
|
|
'endpoint': 'demand',
|
|
'rest': '%s/' % self.id,
|
|
},
|
|
)
|
|
|
|
@property
|
|
def zip_url(self):
|
|
return reverse(
|
|
'generic-endpoint',
|
|
kwargs={
|
|
'connector': self.resource.get_connector_slug(),
|
|
'slug': self.resource.slug,
|
|
'endpoint': 'document',
|
|
'rest': '%s/%s' % (self.id, self.zip_name),
|
|
},
|
|
)
|
|
|
|
def handle_response(self, sftp_client, filename, step):
|
|
try:
|
|
with sftp_client.open(filename) as fd:
|
|
with zipfile.ZipFile(fd) as zip_file:
|
|
with zip_file.open('message.xml') as fd:
|
|
tree = ET.parse(fd)
|
|
ns = 'http://finances.gouv.fr/dgme/pec/message/v1'
|
|
etat_node = tree.find('.//{%s}Etat' % ns)
|
|
if etat_node is None:
|
|
self.logger.error('pull responses: missing Etat node in "%s"', filename)
|
|
return
|
|
etat = etat_node.text
|
|
if etat in MDEL_STATUSES_BY_CODE:
|
|
self.status = MDEL_STATUSES_BY_CODE[etat].slug
|
|
else:
|
|
self.logger.error('pull responses: unknown etat in "%s", etat="%s"', filename, etat)
|
|
return
|
|
commentaire_node = tree.find('.//{%s}Etat' % ns)
|
|
if commentaire_node is not None:
|
|
commentaire = commentaire_node.text
|
|
self.data = self.data or {}
|
|
self.data.setdefault('commentaires', []).append(commentaire)
|
|
self.data['commentaire'] = commentaire
|
|
self.step = step + 1
|
|
self.save()
|
|
self.logger.info('pull responses: status of demand %s changed to %s', self, self.status)
|
|
except sftp.paramiko.SSHException as e:
|
|
self.logger.error(
|
|
'pull responses: failed to read response "%s", %s', filename, exception_to_text(e)
|
|
)
|
|
else:
|
|
try:
|
|
sftp_client.remove(filename)
|
|
except sftp.paramiko.SSHException as e:
|
|
self.logger.error(
|
|
'pull responses: failed to remove response "%s", %s', filename, exception_to_text(e)
|
|
)
|
|
|
|
def __str__(self):
|
|
return '<Demand %s reference:%s flow_type:%s>' % (self.id, self.reference, self.resource.flow_type)
|
|
|
|
class Meta:
|
|
abstract = True
|