917 lines
35 KiB
Python
917 lines
35 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
|
||
# passerelle - uniform access to multiple data sources and services
|
||
# Copyright (C) 2018 Entr'ouvert
|
||
#
|
||
# This program is free software: you can redistribute it and/or modify it
|
||
# under the terms of the GNU Affero General Public License as published
|
||
# by the Free Software Foundation, either version 3 of the License, or
|
||
# (at your option) any later version.
|
||
#
|
||
# This program is distributed in the hope that it will be useful,
|
||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
# GNU Affero General Public License for more details.
|
||
#
|
||
# You should have received a copy of the GNU Affero General Public License
|
||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
||
import json
|
||
import base64
|
||
import urlparse
|
||
import datetime
|
||
import os
|
||
import re
|
||
import magic
|
||
import hashlib
|
||
import copy
|
||
|
||
from HTMLParser import HTMLParser
|
||
|
||
from django.db import models
|
||
from django.http import Http404
|
||
from django.utils.translation import ugettext_lazy as _
|
||
from django.core.files import File
|
||
from django.core.files.base import ContentFile
|
||
|
||
from passerelle.base.models import BaseResource, HTTPResource
|
||
from passerelle.utils.api import endpoint
|
||
from passerelle.utils.jsonresponse import APIError
|
||
|
||
from .json_schemas import (
|
||
JSON_SCHEMA_CHECK_STATUS_OUT,
|
||
JSON_SCHEMA_CREATE_DOSSIER_IN,
|
||
JSON_SCHEMA_CREATE_DOSSIER_OUT,
|
||
JSON_SCHEMA_GET_DOSSIER_OUT,
|
||
JSON_SCHEMA_GET_FWD_FILES_OUT,
|
||
JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT,
|
||
JSON_SCHEMA_GET_COURRIER_OUT
|
||
)
|
||
|
||
|
||
class MLStripper(HTMLParser):
|
||
"""HTML parser that removes html tags."""
|
||
def __init__(self):
|
||
self.reset()
|
||
self.fed = []
|
||
def handle_data(self, d):
|
||
self.fed.append(d)
|
||
def get_data(self):
|
||
return ''.join(self.fed)
|
||
|
||
|
||
def strip_tags(html):
|
||
"""Remove html tags from a string."""
|
||
s = MLStripper()
|
||
s.feed(html)
|
||
return s.get_data()
|
||
|
||
|
||
def clean_spaces(text):
|
||
"""Remove extra spaces an line breaks from a string."""
|
||
text = text.replace('\n', ' ')
|
||
text = text.replace('\r', ' ')
|
||
text = text.replace('\t', ' ')
|
||
text = text.replace('\\n', ' ')
|
||
text = text.replace('\\r', ' ')
|
||
text = text.replace('\\t', ' ')
|
||
return re.sub(r' +', ' ', text).strip()
|
||
|
||
|
||
def normalize(value):
|
||
"""Normalize a value to be send to openADS.API."""
|
||
if value is None:
|
||
return ''
|
||
if not isinstance(value, unicode):
|
||
value = unicode(value)
|
||
return clean_spaces(value)
|
||
|
||
|
||
def get_file_data(path, b64=True):
|
||
"""Return the content of a file as a string, in base64 if specified."""
|
||
with open(path, 'r') as f:
|
||
if b64:
|
||
return base64.b64encode(f.read())
|
||
return f.read()
|
||
|
||
|
||
def get_upload_path(instance, filename=None):
|
||
"""Return a relative upload path for a file."""
|
||
# be careful:
|
||
# * openADS accept only filename less than 50 chars
|
||
# * name should be unique, even if the content is the same
|
||
return 'pass_openADS_up_%s_%s' % (
|
||
datetime.datetime.now().strftime('%Y-%b-%d_%Hh%Mm%Ss%f'),
|
||
instance.file_hash[:4]
|
||
)
|
||
|
||
|
||
def trunc_str_values(value, limit, visited=None, truncate_text=u'…'):
|
||
"""Truncate a string value (not dict keys) and append a truncate text."""
|
||
|
||
if visited is None:
|
||
visited = []
|
||
if not value in visited:
|
||
if isinstance(value, basestring) and len(value) > limit:
|
||
value = value[:limit] + truncate_text
|
||
elif isinstance(value, dict) or isinstance(value, list) or isinstance(value, tuple):
|
||
visited.append(value)
|
||
iterator = value.iteritems() if isinstance(value, dict) else enumerate(value)
|
||
for k,v in iterator:
|
||
value[k] = trunc_str_values(v, limit, visited, truncate_text)
|
||
return value
|
||
|
||
|
||
class DictDumper(object):
|
||
"""Helper to dump a dictionary to a string representation with lazy processing.
|
||
|
||
Only applied when dict is converted to string (lazy processing):
|
||
- long strings truncated (after the dict has been 'deep' copied)
|
||
- (optionaly) dict converted with json.dumps instead of unicode().
|
||
"""
|
||
|
||
def __init__(self, dic, max_str_len=255, use_json_dumps=True):
|
||
""" arguments:
|
||
- dic string the dict to dump
|
||
- max_str_len integer the maximul length of string values
|
||
- use_json_dumps boolean True to use json.dumps() else it uses unicode()
|
||
"""
|
||
self.dic = dic
|
||
self.max_str_len = max_str_len
|
||
self.use_json_dumps = use_json_dumps
|
||
|
||
def __str__(self):
|
||
dict_trunc = trunc_str_values(copy.deepcopy(self.dic), self.max_str_len)
|
||
dict_ref = json.dumps(dict_trunc) if self.use_json_dumps else dict_trunc
|
||
return unicode(dict_ref)
|
||
|
||
|
||
class ForwardFile(models.Model):
|
||
"""Represent a file uploaded by a user, to be forwarded to openADS.API."""
|
||
numero_demande = models.CharField(max_length=20)
|
||
numero_dossier = models.CharField(max_length=20)
|
||
type_fichier = models.CharField(max_length=10)
|
||
file_hash = models.CharField(max_length=100, default='', blank=True)
|
||
orig_filename = models.CharField(max_length=100, default='', blank=True)
|
||
content_type = models.CharField(max_length=100, default='', blank=True)
|
||
upload_file = models.FileField(upload_to=get_upload_path, null=True)
|
||
upload_attempt = models.PositiveIntegerField(default=0, blank=True)
|
||
upload_status = models.CharField(max_length=10, default='', blank=True)
|
||
upload_msg = models.CharField(max_length=255, default='', blank=True)
|
||
last_update_datetime = models.DateTimeField(auto_now=True)
|
||
|
||
|
||
class AtrealOpenads(BaseResource, HTTPResource):
|
||
"""API that proxy/relay communications with/to openADS."""
|
||
|
||
collectivite = models.CharField(_('Collectivity (identifier)'), max_length=255,
|
||
help_text=_('ex: Marseille, or ex: 3'), default='', blank=True)
|
||
openADS_API_url = models.URLField(_('openADS API URL'), max_length=255,
|
||
help_text=_('ex: https://openads.your_domain.net/api/'), default='')
|
||
|
||
openADS_API_timeout = 3600
|
||
|
||
category = _('Business Process Connectors')
|
||
|
||
api_description = _('''This API provides exchanges with openADS.''')
|
||
|
||
class Meta:
|
||
verbose_name = _('openADS')
|
||
|
||
|
||
def log_json_payload(self, payload, title='payload', max_str_len=100):
|
||
"""Log a json paylod surrounded by dashes and with file content filtered."""
|
||
self.logger.debug(u"----- %s (begining) -----", title)
|
||
self.logger.debug(u"%s", DictDumper(payload, max_str_len))
|
||
self.logger.debug(u"----- %s (end) -----", title)
|
||
|
||
|
||
def get_files_from_json_payload(self, payload, title='payload'):
|
||
"""Return files from a JSON payload with all checks and logging."""
|
||
|
||
# check the 'files' key
|
||
if 'files' not in payload:
|
||
self.log_json_payload(payload, title)
|
||
raise APIError(u"Expecting '%s' key in JSON %s" %
|
||
('files', title))
|
||
|
||
files = payload['files']
|
||
|
||
if not isinstance(files, list):
|
||
self.log_json_payload(payload, title)
|
||
raise APIError(
|
||
u"Expecting '%s' value in JSON %s to be a %s (not a %s)" %
|
||
('files', title, 'list', type(files)))
|
||
|
||
if len(files) <= 0:
|
||
self.log_json_payload(payload, title)
|
||
raise APIError(u"Expecting non-empty '%s' value in JSON %s" %
|
||
('files', title))
|
||
|
||
# log the response
|
||
self.log_json_payload(payload, title)
|
||
|
||
# return the files
|
||
return files
|
||
|
||
|
||
def check_file_dict(self, dict_file, title='payload', b64=True):
|
||
"""Ensure a file dict has all its required items."""
|
||
|
||
# key to get the content
|
||
content_key = 'content'
|
||
|
||
# if content is in base 64
|
||
if b64:
|
||
content_key = 'b64_content'
|
||
|
||
# check content existence
|
||
if content_key not in dict_file:
|
||
raise APIError(u"Expecting 'file.%s' key in JSON %s" % (content_key, title))
|
||
|
||
# get its content
|
||
file_content = dict_file[content_key]
|
||
|
||
if not isinstance(file_content, basestring):
|
||
raise APIError(
|
||
u"Expecting '%s' value in JSON %s in file dict to be a %s (not a %s)" %
|
||
('file.%s' % content_key, title, 'string', type(file_content)))
|
||
|
||
# check filename
|
||
if 'filename' in dict_file and not isinstance(dict_file['filename'], basestring):
|
||
raise APIError(
|
||
u"Expecting '%s' value in JSON %s in file dict to be a %s (not a %s)" %
|
||
('file.filename', title, 'string', type(dict_file['filename'])))
|
||
|
||
|
||
def get_first_file_from_json_payload(self, payload, title='payload', ensure_content=True, b64=True):
|
||
"""Return the first file from a JSON payload with all checks and logging."""
|
||
|
||
# get all files
|
||
files = self.get_files_from_json_payload(payload, title)
|
||
|
||
# get the first file
|
||
first = files[0]
|
||
|
||
# asked to check its content
|
||
if ensure_content:
|
||
self.check_file_dict(first, title=title, b64=b64)
|
||
|
||
# return the first file
|
||
return first
|
||
|
||
|
||
@endpoint(
|
||
description=_("Test an openADS 'connexion'")
|
||
#~ get={
|
||
#~ 'description': _("Test an openADS 'connexion'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_CHECK_STATUS_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def check_status(self, *args, **kwargs):
|
||
"""Check avaibility of the openADS.API service."""
|
||
url = urlparse.urljoin(self.openADS_API_url, '__api__')
|
||
response = self.requests.get(url)
|
||
response.raise_for_status()
|
||
return {'response': response.status_code}
|
||
|
||
|
||
@endpoint(
|
||
methods=['post'],
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier': {'description': _("Type of 'dossier'"), 'example_value': 'DIA'}
|
||
},
|
||
post={'description': _("Create an openADS 'dossier'"),
|
||
'request_body': {
|
||
'schema': {
|
||
'application/json': JSON_SCHEMA_CREATE_DOSSIER_IN
|
||
}
|
||
},
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_CREATE_DOSSIER_OUT
|
||
#~ }
|
||
#~ }
|
||
}
|
||
)
|
||
def create_dossier(self, request, type_dossier, *args, **kwargs):
|
||
|
||
# loads the request body as JSON content
|
||
json_data = json.loads(request.body)
|
||
|
||
# log the request body (filtering the files content)
|
||
self.log_json_payload(json_data, 'request')
|
||
|
||
# build the payload
|
||
payload = { "collectivite": self.collectivite }
|
||
|
||
payload["terrain"] = {
|
||
"numero_voie": normalize(json_data['fields']['terrain_numero_voie']),
|
||
"nom_voie" : normalize(json_data['fields']['terrain_nom_voie']),
|
||
"code_postal": normalize(json_data['fields']['terrain_code_postal']),
|
||
"localite" : normalize(json_data['fields']['terrain_localite']),
|
||
"references_cadastrales": []
|
||
}
|
||
if 'terrain_lieu_dit' in json_data['fields'] and json_data['fields']['terrain_lieu_dit']:
|
||
payload["terrain"]["lieu_dit"] = normalize(json_data['fields']['terrain_lieu_dit'])
|
||
|
||
for ref in json_data['fields']['reference_cadastrale']:
|
||
payload["terrain"]["references_cadastrales"].append({
|
||
"prefixe": normalize(ref[0]),
|
||
"section": normalize(ref[1]),
|
||
"numero" : normalize(ref[2])
|
||
})
|
||
if json_data['fields']['autres_parcelles']:
|
||
for ref in json_data['fields']['references_cadastrales']:
|
||
payload["terrain"]["references_cadastrales"].append({
|
||
"prefixe": normalize(ref[0]),
|
||
"section": normalize(ref[1]),
|
||
"numero" : normalize(ref[2])
|
||
})
|
||
|
||
# setup demandeur variable prefix
|
||
prefixes = {"demandeurs": ''}
|
||
if normalize(json_data['fields']['proprietaire']) != 'Oui':
|
||
prefixes["mandataires"] = 'mandataire_'
|
||
|
||
# for each type of demandeur with associated prefix
|
||
for key,prefix in prefixes.items():
|
||
|
||
# "qualité" of the demandeur
|
||
qualite = normalize(json_data['fields']['%squalite' % prefix])
|
||
|
||
# get the demandeur informations
|
||
demandeur = {
|
||
"type_personne": 'particulier' if qualite == 'Un particulier' else 'personne_morale',
|
||
"typologie" : 'petitionnaire' if key == 'demandeurs' else 'delegataire',
|
||
"nom" : normalize(json_data['fields']['%snom' % prefix]),
|
||
"prenom" : normalize(json_data['fields']['%sprenom' % prefix]),
|
||
"adresse": {
|
||
"numero_voie": normalize(json_data['fields']['%snumero_voie' % prefix]),
|
||
"nom_voie" : normalize(json_data['fields']['%snom_voie' % prefix]),
|
||
"code_postal": normalize(json_data['fields']['%scode_postal' % prefix]),
|
||
"localite" : normalize(json_data['fields']['%slocalite' % prefix])
|
||
}
|
||
}
|
||
|
||
# add fields if the demandeur is not an individual
|
||
if qualite != 'Un particulier':
|
||
demandeur["raison_sociale"] = normalize(json_data['fields']['%sraison_sociale' % prefix])
|
||
demandeur["denomination"] = normalize(json_data['fields']['%sdenomination' % prefix])
|
||
self.logger.debug("%s %s => '%s', '%s'", demandeur['prenom'], demandeur['nom'], demandeur['raison_sociale'], demandeur['denomination'])
|
||
|
||
# add optional lieu_dit field
|
||
if '%slieu_dit' % prefix in json_data['fields'] and json_data['fields']['%slieu_dit' % prefix]:
|
||
demandeur["adresse"]["lieu_dit"] = normalize(json_data['fields']['%slieu_dit' % prefix])
|
||
|
||
# add it to the payload
|
||
payload[key] = [demandeur]
|
||
|
||
self.logger.debug(u"Added '%s' to payload: %s %s", key, demandeur['prenom'], demandeur['nom'])
|
||
|
||
# log the payload
|
||
self.log_json_payload(payload)
|
||
|
||
# every field key that might contain a file content
|
||
file_keys = ['cerfa'] + ['plan_cadastral_%s' % i for i in range(1,5)] + ['pouvoir_mandat']
|
||
|
||
# prepare files that will be forwarded
|
||
files = []
|
||
for k in file_keys:
|
||
if (
|
||
k in json_data['fields']
|
||
and json_data['fields'][k]
|
||
and isinstance(json_data['fields'][k], dict)
|
||
and 'content' in json_data['fields'][k]
|
||
):
|
||
# get the content decoded from base 64
|
||
content = base64.b64decode(json_data['fields'][k]['content'])
|
||
|
||
# guess the mime type based on the begining of the content
|
||
content_type = magic.from_buffer(content, mime=True)
|
||
|
||
# set it as an upload
|
||
upload_file = ContentFile(content)
|
||
|
||
# build a hash from the upload
|
||
file_hash = self.file_digest(upload_file)
|
||
|
||
# build a filename (less than 50 chars)
|
||
filename = file_hash[45:] + '.pdf'
|
||
|
||
# get the content type if specified
|
||
if 'content_type' in json_data['fields'][k]:
|
||
content_type = json_data['fields'][k]['content_type']
|
||
|
||
# check the content type is PDF for file of type CERFA
|
||
if k == 'cerfa' and content_type != 'application/pdf':
|
||
self.logger.warning("CERFA content type is '%s' instead of '%s'", content_type, 'application/pdf')
|
||
|
||
# get the filename if specified
|
||
if 'filename' in json_data['fields'][k]:
|
||
filename = json_data['fields'][k]['filename']
|
||
|
||
# set the type fichier based on the key (less than 10 chars)
|
||
type_fichier = re.sub(r'_.*$', '', k)[:10]
|
||
|
||
# append the file to the list
|
||
files.append({
|
||
'type_fichier' : type_fichier,
|
||
'orig_filename': filename,
|
||
'content_type' : content_type,
|
||
'file_hash' : file_hash,
|
||
'upload_file' : upload_file
|
||
})
|
||
|
||
# log files to be forwarded
|
||
self.logger.debug("----- files (begining) -----")
|
||
self.logger.debug(files)
|
||
self.logger.debug("----- files (end) -----")
|
||
|
||
# make a request to openADS.API (with the payload)
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossiers/%s' % type_dossier)
|
||
response = self.requests.post(
|
||
url,
|
||
json=payload,
|
||
timeout=self.openADS_API_timeout
|
||
)
|
||
|
||
# response is an error code
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning(u"Request [POST] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response JSON content
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError(u'No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# get the recepisse
|
||
recepisse = self.get_first_file_from_json_payload(result, title='response')
|
||
|
||
# ensure recepisse content type is PDF
|
||
if (
|
||
'content_type' in recepisse
|
||
and recepisse['content_type']
|
||
and recepisse['content_type'] != 'application/pdf'
|
||
):
|
||
self.logger.debug(
|
||
u"Forcing 'recepisse' content type to '%s' instead of '%s'.",
|
||
'application/pdf',
|
||
recepisse['content_type']
|
||
)
|
||
recepisse['content_type'] = 'application/pdf'
|
||
|
||
# decode the recepisse from base 64
|
||
try:
|
||
recepisse_content = base64.b64decode(recepisse['b64_content'])
|
||
except TypeError:
|
||
raise APIError('Failed to decode recepisse content from base 64')
|
||
self.logger.debug("Successfully decoded recepisse from base 64")
|
||
|
||
# check/get the 'numero_dossier'
|
||
if 'numero_dossier' not in result:
|
||
raise APIError("Expecting 'numero_dossier' key in JSON response")
|
||
|
||
numero_dossier = result.get('numero_dossier')
|
||
|
||
if not isinstance(numero_dossier, basestring):
|
||
raise APIError(
|
||
u"Expecting '%s' value in JSON response to be a %s (not a %s)" %
|
||
('numero_dossier', 'string', type(numero_dossier)))
|
||
|
||
numero_dossier = normalize(numero_dossier)
|
||
self.logger.debug(u"Numéro dossier: %s", numero_dossier)
|
||
|
||
# save files to be forwarded to openADS.API
|
||
if files:
|
||
file_ids = []
|
||
for f in files:
|
||
rand_id = base64.urlsafe_b64encode(os.urandom(6))
|
||
FF = ForwardFile()
|
||
FF.numero_demande = rand_id
|
||
FF.numero_dossier = numero_dossier
|
||
for k in ['type_fichier', 'orig_filename', 'content_type', 'file_hash']:
|
||
setattr(FF, k, f[k])
|
||
FF.upload_file.save(FF.orig_filename, f['upload_file'])
|
||
FF.upload_status = 'pending'
|
||
FF.save()
|
||
self.logger.debug(
|
||
u"Created ForwardFile '%s' for file '%s' (%s)",
|
||
FF.id,
|
||
FF.orig_filename,
|
||
FF.upload_file.path
|
||
)
|
||
file_ids.append(FF.id)
|
||
|
||
job = self.add_job('upload_user_files',
|
||
natural_id=numero_dossier,
|
||
type_dossier=type_dossier,
|
||
numero_dossier=numero_dossier,
|
||
file_ids=file_ids)
|
||
self.logger.debug(
|
||
u"Added a job '%s' for dossier '%s' (%s) with file ids '%s'",
|
||
job.id,
|
||
numero_dossier,
|
||
type_dossier,
|
||
file_ids
|
||
)
|
||
|
||
# respond with the 'numero_dossier' and the recepisse file
|
||
return {
|
||
'numero_dossier': numero_dossier,
|
||
'recepisse' : recepisse
|
||
}
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about an openADS 'dossier'"),
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about an openADS 'dossier'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_DOSSIER_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_dossier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||
|
||
# make a request to openADS.API
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s' % (type_dossier, numero_dossier))
|
||
response = self.requests.get(url)
|
||
|
||
# response is an error
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning(u"Request [GET] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response as JSON
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError(u'No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# log the response
|
||
self.log_json_payload(result, 'response')
|
||
|
||
# return the response as-is
|
||
return response.json()
|
||
|
||
|
||
def upload2ForwardFile(self, path, numero_dossier, type_fichier):
|
||
"""Convert a file path to a ForwardFile."""
|
||
if path:
|
||
rand_id = base64.urlsafe_b64encode(os.urandom(6))
|
||
fwd_file = ForwardFile()
|
||
fwd_file.numero_demande = rand_id
|
||
fwd_file.numero_dossier = numero_dossier
|
||
fwd_file.type_fichier = type_fichier
|
||
fwd_file.orig_filename = os.path.basename(path)
|
||
fwd_file.content_type = magic.from_file(path, mime=True)
|
||
with open(path, 'r') as fp:
|
||
fwd_file.file_hash = self.file_digest(fp)
|
||
fwd_file.upload_file = File(open(path, 'r'))
|
||
fwd_file.upload_status = 'pending'
|
||
return fwd_file
|
||
return None
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about the forwarding of user files to openADS"),
|
||
parameters={
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about the forwarding of user files to openADS"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_fwd_files(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||
payload = []
|
||
fwd_files = []
|
||
|
||
# search for all files matching the 'numero_dossier' number
|
||
if not fichier_id:
|
||
fwd_files = ForwardFile.objects.filter(numero_dossier=numero_dossier)
|
||
|
||
# search for a single file
|
||
elif fichier_id:
|
||
try:
|
||
fichier_id = int(fichier_id)
|
||
except ValueError:
|
||
raise APIError('fichier_id must be an integer')
|
||
try:
|
||
fwd_files = [ForwardFile.objects.get(id=fichier_id)]
|
||
except ForwardFile.DoesNotExist:
|
||
raise Http404(u"No file matches 'numero_dossier=%s' and 'id=%s'." % (numero_dossier, fichier_id))
|
||
|
||
# append each file to the response payload
|
||
for fwd_file in fwd_files:
|
||
payload.append({
|
||
'id' : fwd_file.id,
|
||
'numero_demande': fwd_file.numero_demande,
|
||
'numero_dossier': fwd_file.numero_dossier,
|
||
'type_fichier' : fwd_file.type_fichier,
|
||
'file_hash' : fwd_file.file_hash,
|
||
'orig_filename' : fwd_file.orig_filename,
|
||
'content_type' : fwd_file.content_type,
|
||
'upload_status' : fwd_file.upload_status,
|
||
'upload_attempt': fwd_file.upload_attempt,
|
||
'upload_msg' : fwd_file.upload_msg,
|
||
'content_size' : fwd_file.upload_file.size if fwd_file.upload_file else 0,
|
||
'last_update_datetime' : fwd_file.last_update_datetime
|
||
})
|
||
|
||
# return the payload containing the list of files
|
||
return payload
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about the forwarding of a user file to openADS"),
|
||
parameters={
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about the forwarding of a user file to openADS"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_fwd_files_status(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||
|
||
# get all files matching 'numero_dossier' and 'fichier_id'
|
||
fwd_files = self.get_fwd_files(request, numero_dossier, fichier_id, *args, **kwargs)
|
||
|
||
# prepare the response payload
|
||
payload = {
|
||
'all_forwarded': True,
|
||
'pending' : [],
|
||
'uploading' : [],
|
||
'success' : [],
|
||
'failed' : []
|
||
}
|
||
|
||
# build a summary of all files statuses
|
||
for fwd_file in fwd_files:
|
||
status_msg = u'[%s] %s => %s' % (
|
||
fwd_file['id'],
|
||
fwd_file['orig_filename'],
|
||
fwd_file['upload_msg']
|
||
)
|
||
payload[fwd_file['upload_status']].append(status_msg)
|
||
if fwd_file['upload_status'] != 'success':
|
||
payload['all_forwarded'] = False
|
||
|
||
# respond with the payload
|
||
return payload
|
||
|
||
|
||
@endpoint(
|
||
description= _("Get a 'courrier' from an openADS 'dossier'"),
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get a 'courrier' from an openADS 'dossier'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_COURRIER_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_courrier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||
|
||
# make a request to openADS.API
|
||
url = urlparse.urljoin(
|
||
self.openADS_API_url,
|
||
'/dossier/%s/%s/courrier/%s' % (type_dossier, numero_dossier, 'dia_renonciation_preempter'))
|
||
response = self.requests.get(url)
|
||
|
||
# response is an error
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning(u"Request [GET] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response as JSON
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError(u'No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# log the response (filtering the file content)
|
||
self.log_json_payload(result, 'response')
|
||
|
||
# get the courrier
|
||
courrier = self.get_first_file_from_json_payload(result, title='response')
|
||
|
||
# decode the courrier from base 64
|
||
try:
|
||
courrier_content = base64.b64decode(courrier['b64_content'])
|
||
except TypeError:
|
||
raise APIError('Failed to decode courrier content from base 64')
|
||
|
||
# return the 'courrier' file
|
||
return {'courrier': courrier}
|
||
|
||
|
||
def get_response_error(self, response):
|
||
"""Return a error string from an HTTP response."""
|
||
try:
|
||
# load the response as JSON
|
||
result = response.json()
|
||
|
||
# collect errors and turn them into messages (multispaces are filtered)
|
||
errors = result.get('errors')
|
||
msg = []
|
||
if errors:
|
||
for error in errors:
|
||
location = error.get('location')
|
||
name = error.get('name')
|
||
desc = error.get('description')
|
||
msg.append(u'[%s] (%s) %s' % (location, normalize(name), normalize(desc)))
|
||
|
||
# if there are messages
|
||
if msg:
|
||
|
||
# return a string representing the HTTP error
|
||
return u"HTTP error: %s, %s" % (response.status_code, ','.join(msg))
|
||
|
||
except ValueError:
|
||
pass
|
||
|
||
# TODO ask for openADS.API to *always* send JSON formatted errors, not HTML ones
|
||
# return a string representing the HTTP error (filtering the HTML tags and multispaces)
|
||
detail = clean_spaces(strip_tags(response.content[:1000])) if response.content else ''
|
||
return u"HTTP error: %s%s" % (response.status_code, ', ' + detail if detail else '')
|
||
|
||
|
||
# @raise ForwareFile.DoesNotExist if not found
|
||
def upload_user_files(self, type_dossier, numero_dossier, file_ids):
|
||
"""A Job to forward user uploaded files to openADS."""
|
||
|
||
payload = []
|
||
fwd_files = []
|
||
|
||
# for every file ids specified (in parameters of this job)
|
||
for fid in file_ids:
|
||
self.logger.debug(u"upload_user_files() ForwardFile file_id: %s", fid)
|
||
|
||
# get the matching forward file
|
||
fwd_file = ForwardFile.objects.get(id=fid)
|
||
|
||
# found one
|
||
if fwd_file:
|
||
self.logger.debug("upload_user_files() got ForwardFile")
|
||
|
||
# add the file content and data to the payload
|
||
payload.append({
|
||
'filename' : fwd_file.orig_filename + ('.pdf' if fwd_file.orig_filename[-4:] != '.pdf' else ''),
|
||
'content_type' : fwd_file.content_type,
|
||
'b64_content' : base64.b64encode(fwd_file.upload_file.read()),
|
||
'file_type' : fwd_file.type_fichier
|
||
})
|
||
self.logger.debug("upload_user_files() payload added")
|
||
|
||
# update the file upload data (status and attempts)
|
||
fwd_file.upload_status = 'uploading'
|
||
fwd_file.upload_attempt += 1
|
||
fwd_file.upload_msg = 'attempt %s' % fwd_file.upload_attempt
|
||
self.logger.debug(u"upload_user_files() upload_msg: '%s'", fwd_file.upload_msg)
|
||
fwd_file.save()
|
||
self.logger.debug("upload_user_files() ForwardFile saved")
|
||
|
||
# append the forwarded file to the list
|
||
fwd_files.append(fwd_file)
|
||
|
||
# if files need to be forwarded
|
||
if payload:
|
||
|
||
self.logger.debug("upload_user_files() payload is not empty")
|
||
|
||
# log the payload
|
||
self.log_json_payload(payload, 'payload')
|
||
|
||
# make the request to openADS.API (with a specific timeout)
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s/files' % (type_dossier, numero_dossier))
|
||
response = self.requests.post(
|
||
url,
|
||
json=payload,
|
||
timeout=self.openADS_API_timeout
|
||
)
|
||
|
||
# reponse is an error
|
||
if response.status_code // 100 != 2:
|
||
|
||
# update every files status as 'failed' and save the error message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'failed'
|
||
fwd_file.upload_msg = self.get_response_error(response)
|
||
fwd_file.save()
|
||
|
||
# log (warning) the error message
|
||
self.logger.warning(
|
||
u"upload_user_files() openADS response is not OK (code: %s) for dossier '%s' and files '%s'",
|
||
response.status_code,
|
||
numero_dossier,
|
||
file_ids
|
||
)
|
||
|
||
# response is not an error
|
||
else:
|
||
|
||
# load the reponse as JSON
|
||
try:
|
||
result = response.json()
|
||
|
||
# in case of failure
|
||
except ValueError:
|
||
|
||
# update every files status as 'failed' and save the error message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'failed'
|
||
fwd_file.upload_msg = u'No JSON content returned: %r' % response.content[:1000]
|
||
fwd_file.save()
|
||
|
||
# log (warning) the error message
|
||
self.logger.warning(
|
||
u"upload_user_files() openADS response is not JSON valid for dossier '%s' and files '%s'",
|
||
numero_dossier,
|
||
fwd_files
|
||
)
|
||
|
||
# response correctly loaded as JSON
|
||
else:
|
||
|
||
# TODO handle response (now its just an informational sentence in key 'data')
|
||
|
||
# update every files status as 'success' and save the success message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'success'
|
||
fwd_file.upload_msg = 'uploaded successfuly'
|
||
|
||
# delete file content (on success)
|
||
fpath = fwd_file.upload_file.path
|
||
fwd_file.upload_file.delete()
|
||
|
||
# save the file
|
||
fwd_file.save()
|
||
|
||
# log the success message
|
||
self.logger.debug(
|
||
u"upload_user_files() flaging file '%s' has transfered (deleted '%s')",
|
||
fwd_file.id,
|
||
fpath
|
||
)
|
||
|
||
# no file need to be forwarded
|
||
else:
|
||
self.logger.warning(
|
||
u"upload_user_files() payload is empty for dossier '%s' and files '%s'",
|
||
numero_dossier,
|
||
file_ids
|
||
)
|
||
|
||
|
||
# copy-pasted from 'wcs/qommon/misc.py'
|
||
def file_digest(self, content, chunk_size=100000):
|
||
"""Return a hash for the content specified."""
|
||
digest = hashlib.sha256()
|
||
content.seek(0)
|
||
def read_chunk():
|
||
return content.read(chunk_size)
|
||
for chunk in iter(read_chunk, ''):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|