940 lines
36 KiB
Python
940 lines
36 KiB
Python
#!/usr/bin/env python
|
||
# -*- coding: utf-8 -*-
|
||
|
||
# passerelle - uniform access to multiple data sources and services
|
||
# Copyright (C) 2018 Entr'ouvert
|
||
#
|
||
# This program is free software: you can redistribute it and/or modify it
|
||
# under the terms of the GNU Affero General Public License as published
|
||
# by the Free Software Foundation, either version 3 of the License, or
|
||
# (at your option) any later version.
|
||
#
|
||
# This program is distributed in the hope that it will be useful,
|
||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
# GNU Affero General Public License for more details.
|
||
#
|
||
# You should have received a copy of the GNU Affero General Public License
|
||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||
|
||
import json
|
||
import base64
|
||
import urlparse
|
||
import datetime
|
||
import os
|
||
import re
|
||
import magic
|
||
import hashlib
|
||
import copy
|
||
|
||
from HTMLParser import HTMLParser
|
||
|
||
from django.db import models
|
||
from django.http import Http404
|
||
from django.utils.translation import ugettext_lazy as _
|
||
from django.core.files import File
|
||
from django.core.files.base import ContentFile
|
||
|
||
from passerelle.base.models import BaseResource, HTTPResource
|
||
from passerelle.utils.api import endpoint
|
||
from passerelle.utils.jsonresponse import APIError
|
||
|
||
from .json_schemas import (
|
||
JSON_SCHEMA_CHECK_STATUS_OUT,
|
||
JSON_SCHEMA_CREATE_DOSSIER_IN,
|
||
JSON_SCHEMA_CREATE_DOSSIER_OUT,
|
||
JSON_SCHEMA_GET_DOSSIER_OUT,
|
||
JSON_SCHEMA_GET_FWD_FILES_OUT,
|
||
JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT,
|
||
JSON_SCHEMA_GET_COURRIER_OUT
|
||
)
|
||
|
||
|
||
class MLStripper(HTMLParser):
|
||
"""HTML parser that removes html tags."""
|
||
def __init__(self):
|
||
self.reset()
|
||
self.fed = []
|
||
def handle_data(self, d):
|
||
self.fed.append(d)
|
||
def get_data(self):
|
||
return ''.join(self.fed)
|
||
|
||
|
||
def strip_tags(html):
|
||
"""Remove html tags from a string."""
|
||
s = MLStripper()
|
||
s.feed(html)
|
||
return s.get_data()
|
||
|
||
|
||
def clean_spaces(text):
|
||
"""Remove extra spaces an line breaks from a string."""
|
||
text = text.replace('\n', ' ')
|
||
text = text.replace('\r', ' ')
|
||
text = text.replace('\t', ' ')
|
||
text = text.replace('\\n', ' ')
|
||
text = text.replace('\\r', ' ')
|
||
text = text.replace('\\t', ' ')
|
||
return re.sub(r' +', ' ', text).strip()
|
||
|
||
|
||
def normalize(value):
|
||
"""Normalize a value to be send to openADS.API."""
|
||
if value is None:
|
||
return ''
|
||
return clean_spaces(str(value))
|
||
|
||
|
||
def get_file_data(path, b64=True):
|
||
"""Return the content of a file as a string, in base64 if specified."""
|
||
with open(path, 'r') as f:
|
||
if b64:
|
||
return base64.b64encode(f.read())
|
||
return f.read()
|
||
|
||
|
||
def get_upload_path(instance, filename):
|
||
"""Return a relative upload path for a file."""
|
||
# be careful:
|
||
# * openADS accept only filename less than 50 chars
|
||
# * name should be unique, even if the content is the same
|
||
return 'pass_openADS_up_%s_%s' % (
|
||
datetime.datetime.now().strftime('%Y-%b-%d_%Hh%Mm%Ss%f'),
|
||
instance.file_hash[:4]
|
||
)
|
||
|
||
|
||
def dict_deref_multi(data, keys):
|
||
"""Access a dict through a list of keys.
|
||
Inspired by: https://stackoverflow.com/a/47969823
|
||
"""
|
||
if keys:
|
||
key = int(keys[0]) if isinstance(data, list) else keys[0]
|
||
return dict_deref_multi(data[key], keys[1:]) if keys else data
|
||
|
||
# TODO implement wildcard filtering
|
||
def dict_replace_content_by_paths(dic, paths_to_replace=[], path_separator='/', replace_by='<file content>'):
|
||
"""Replace a dict content, with a string, from a list of paths."""
|
||
newdic = dic
|
||
if newdic and (isinstance(newdic, dict) or isinstance(newdic, list)):
|
||
for p in paths_to_replace:
|
||
if p:
|
||
if p.startswith(path_separator):
|
||
p = p[1:]
|
||
items = p.split(path_separator)
|
||
if items:
|
||
try:
|
||
value = dict_deref_multi(newdic, items)
|
||
except KeyError:
|
||
continue
|
||
except TypeError:
|
||
continue
|
||
if not isinstance(value, str) and not isinstance(value, unicode):
|
||
raise ValueError("path '%s' is not a string ([%s] %s)" % (p, str(type(value)), str(value)))
|
||
# newdic is still untouched
|
||
if newdic is dic:
|
||
# create a copy of it
|
||
newdic = copy.deepcopy(dic)
|
||
last_item = items.pop(-1)
|
||
value = dict_deref_multi(newdic, items)
|
||
value[last_item] = replace_by
|
||
return newdic
|
||
|
||
|
||
class LogJsonPayloadWithFileContent(object):
|
||
""" Return a string representation of a JSON payload with its file content emptyed."""
|
||
|
||
def __init__(self, payload, paths_to_replace=[], path_separator='/', replace_by='<data>'):
|
||
""" arguments:
|
||
- payload string the JSON payload to dump
|
||
- paths_to_replace array a list of strings either:
|
||
* in 'path' format (i.e.: ['/fields/file/content',...])
|
||
* in 'namespace' format (i.e.: ['fields.file.content',...])
|
||
- path_separator string the path separator used in 'paths_to_replace' format:
|
||
* '/' for 'path' format
|
||
* '.' for 'namespace' format
|
||
- replace_by string the new value of the content at the specified path
|
||
"""
|
||
self.payload = payload
|
||
self.paths_to_replace = paths_to_replace
|
||
self.path_separator = path_separator
|
||
self.replace_by = replace_by
|
||
|
||
def __str__(self):
|
||
return json.dumps(dict_replace_content_by_paths(
|
||
self.payload,
|
||
paths_to_replace = self.paths_to_replace,
|
||
path_separator = self.path_separator,
|
||
replace_by = self.replace_by
|
||
))
|
||
|
||
|
||
class ForwardFile(models.Model):
|
||
"""Represent a file uploaded by a user, to be forwarded to openADS.API."""
|
||
numero_demande = models.CharField(max_length=20)
|
||
numero_dossier = models.CharField(max_length=20)
|
||
type_fichier = models.CharField(max_length=10)
|
||
file_hash = models.CharField(max_length=100, default='', blank=True)
|
||
orig_filename = models.CharField(max_length=100, default='', blank=True)
|
||
content_type = models.CharField(max_length=100, default='', blank=True)
|
||
upload_file = models.FileField(upload_to=get_upload_path, null=True)
|
||
upload_attempt = models.PositiveIntegerField(default=0, blank=True)
|
||
upload_status = models.CharField(max_length=10, default='', blank=True)
|
||
upload_msg = models.CharField(max_length=255, default='', blank=True)
|
||
last_update_datetime = models.DateTimeField(auto_now=True)
|
||
|
||
|
||
class AtrealOpenads(BaseResource, HTTPResource):
|
||
"""API that proxy/relay communications with/to openADS."""
|
||
|
||
collectivite = models.CharField(_('Collectivity (identifier)'), max_length=255,
|
||
help_text=_('ex: Marseille, or ex: 3'), default='', blank=True)
|
||
openADS_API_url = models.URLField(_('openADS API URL'), max_length=255,
|
||
help_text=_('ex: https://openads.your_domain.net/api/'), default='')
|
||
|
||
openADS_API_timeout = 3600
|
||
|
||
category = _('Business Process Connectors')
|
||
|
||
api_description = _('''This API provides exchanges with openADS.''')
|
||
|
||
class Meta:
|
||
verbose_name = _('openADS')
|
||
|
||
|
||
@property
|
||
def extra_debug_enabled(self):
|
||
"""Return True if 'extra debug' is enabled."""
|
||
return bool(self.extra_debug)
|
||
|
||
|
||
def log_json_payload(self, payload, title='payload', paths_to_replace=[]):
|
||
"""Log a json paylod surrounded by dashes and with file content filtered."""
|
||
self.logger.debug("----- %s (begining) -----", title)
|
||
if paths_to_replace:
|
||
self.logger.debug("%s", LogJsonPayloadWithFileContent(
|
||
payload,
|
||
paths_to_replace = paths_to_replace,
|
||
path_separator = '.',
|
||
replace_by = '<b64 content>'
|
||
))
|
||
else:
|
||
self.logger.debug("%s", LogJsonPayloadWithFileContent(payload))
|
||
self.logger.debug("----- %s (end) -----", title)
|
||
|
||
|
||
def get_files_from_json_payload(self, payload, title='payload'):
|
||
"""Return files from a JSON payload with all checks and logging."""
|
||
|
||
# check the 'files' key
|
||
if 'files' not in payload:
|
||
self.log_json_payload(payload, title)
|
||
raise APIError("Expecting '%s' key in JSON %s" %
|
||
('files', title))
|
||
|
||
files = payload.get('files')
|
||
|
||
if not isinstance(files, list):
|
||
self.log_json_payload(payload, title)
|
||
raise APIError(
|
||
"Expecting '%s' value in JSON %s to be a %s (not a %s)" %
|
||
('files', title, 'list', str(type(files))))
|
||
|
||
if len(files) <= 0:
|
||
self.log_json_payload(payload, title)
|
||
raise APIError("Expecting non-empty '%s' value in JSON %s" %
|
||
('files', title))
|
||
|
||
# log the response
|
||
self.log_json_payload(
|
||
payload, title
|
||
,['files.%i.b64_content' % i for i in range(0, len(files))])
|
||
|
||
# return the files
|
||
return files
|
||
|
||
|
||
def check_file_dict(self, dict_file, b64=True, title='payload'):
|
||
"""Ensure a file dict has all its required items."""
|
||
|
||
# key to get the content
|
||
content_key = 'content'
|
||
|
||
# if content is in base 64
|
||
if b64:
|
||
content_key = 'b64_content'
|
||
|
||
# check content existence
|
||
if content_key not in dict_file:
|
||
raise APIError("Expecting 'files.0.%s' key in JSON %s" % (content_key, title))
|
||
|
||
# get its content
|
||
file_content = dict_file[content_key]
|
||
|
||
if not isinstance(file_content, str) and not isinstance(file_content, unicode):
|
||
raise APIError(
|
||
"Expecting '%s' value in JSON %s to be a %s (not a %s)" %
|
||
('file.%s' % content_key, title, 'string', str(type(file_content))))
|
||
|
||
# check filename
|
||
if (
|
||
'filename' in dict_file
|
||
and not isinstance(dict_file['filename'], str)
|
||
and not isinstance(dict_file['filename'], unicode)
|
||
):
|
||
raise APIError(
|
||
"Expecting '%s' value in file dict to be a %s (not a %s)" %
|
||
('file.filename', title, 'string', str(type(dict_file['filename']))))
|
||
|
||
|
||
def get_first_file_from_json_payload(self, payload, title='payload', ensure_content=True, b64=True):
|
||
"""Return the first file from a JSON payload with all checks and logging."""
|
||
|
||
# get all files
|
||
files = self.get_files_from_json_payload(payload, title)
|
||
|
||
# get the first file
|
||
first = files[0]
|
||
|
||
# asked to check its content
|
||
if ensure_content:
|
||
self.check_file_dict(first, title, b64)
|
||
|
||
# return the first file
|
||
return first
|
||
|
||
|
||
@endpoint(
|
||
description=_("Test an openADS 'connexion'")
|
||
#~ get={
|
||
#~ 'description': _("Test an openADS 'connexion'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_CHECK_STATUS_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def check_status(self, *args, **kwargs):
|
||
"""Check avaibility of the openADS.API service."""
|
||
url = urlparse.urljoin(self.openADS_API_url, '__api__')
|
||
response = self.requests.get(url)
|
||
response.raise_for_status()
|
||
return {'response': response.status_code}
|
||
|
||
|
||
@endpoint(
|
||
methods=['post'],
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier': {'description': _("Type of 'dossier'"), 'example_value': 'DIA'}
|
||
},
|
||
post={'description': _("Create an openADS 'dossier'"),
|
||
'request_body': {
|
||
'schema': {
|
||
'application/json': JSON_SCHEMA_CREATE_DOSSIER_IN
|
||
}
|
||
},
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_CREATE_DOSSIER_OUT
|
||
#~ }
|
||
#~ }
|
||
}
|
||
)
|
||
def create_dossier(self, request, type_dossier, *args, **kwargs):
|
||
|
||
# loads the request body as JSON content
|
||
json_data = json.loads(request.body)
|
||
|
||
# every field key that might contain a file content
|
||
file_keys = ['cerfa'] + ['plan_cadastral_%s' % i for i in range(1,5)]
|
||
|
||
# detect if there evolutions data to filter them out of logging
|
||
# TODO replace when wildcard filtering is implement in 'dict_replace_content_by_paths()'
|
||
evolution_to_filter = []
|
||
if 'evolution' in json_data and isinstance(json_data['evolution'], list):
|
||
for e, evol in enumerate(json_data['evolution']):
|
||
if isinstance(evol, dict) and 'parts' in evol and isinstance(evol['parts'], list):
|
||
for p, part in enumerate(evol['parts']):
|
||
if isinstance(part, dict) and 'data' in part:
|
||
evolution_to_filter.append('evolution.%s.parts.%s.data' % (e, p))
|
||
|
||
# log the request body (filtering the files content)
|
||
self.log_json_payload(
|
||
json_data, 'request',
|
||
['fields.%s.content' % k for k in file_keys] + evolution_to_filter)
|
||
|
||
# build the payload
|
||
payload = { "collectivite": self.collectivite }
|
||
|
||
payload["terrain"] = {
|
||
"numero_voie": normalize(json_data['fields']['terrain_numero_voie']),
|
||
"nom_voie" : normalize(json_data['fields']['terrain_nom_voie']),
|
||
"code_postal": normalize(json_data['fields']['terrain_code_postal']),
|
||
"localite" : normalize(json_data['fields']['terrain_localite']),
|
||
"references_cadastrales": []
|
||
}
|
||
if 'terrain_lieu_dit' in json_data['fields'] and json_data['fields']['terrain_lieu_dit']:
|
||
payload["terrain"]["lieu_dit"] = normalize(json_data['fields']['terrain_lieu_dit'])
|
||
|
||
for ref in json_data['fields']['reference_cadastrale']:
|
||
payload["terrain"]["references_cadastrales"].append({
|
||
"prefixe": normalize(ref[0]),
|
||
"section": normalize(ref[1]),
|
||
"numero" : normalize(ref[2])
|
||
})
|
||
if json_data['fields']['autres_parcelles']:
|
||
for ref in json_data['fields']['references_cadastrales']:
|
||
payload["terrain"]["references_cadastrales"].append({
|
||
"prefixe": normalize(ref[0]),
|
||
"section": normalize(ref[1]),
|
||
"numero" : normalize(ref[2])
|
||
})
|
||
|
||
prefix = ''
|
||
if json_data['fields']['proprietaire'] != 'Oui':
|
||
prefix = 'mandataire_'
|
||
|
||
demandeur = {
|
||
"type_personne": 'particulier' if normalize(json_data['fields']['proprietaire_qualite']) == 'Un particulier' else 'personne morale',
|
||
"typologie" : 'petitionnaire' if normalize(json_data['fields']['proprietaire']) == 'Oui' else 'mandataire',
|
||
"nom" : normalize(json_data['fields']['%snom' % prefix]),
|
||
"prenom" : normalize(json_data['fields']['%sprenom' % prefix]),
|
||
"adresse": {
|
||
"numero_voie": normalize(json_data['fields']['%snumero_voie' % prefix]),
|
||
"nom_voie" : normalize(json_data['fields']['%snom_voie' % prefix]),
|
||
"code_postal": normalize(json_data['fields']['%scode_postal' % prefix]),
|
||
"localite" : normalize(json_data['fields']['%slocalite' % prefix])
|
||
}
|
||
}
|
||
if '%slieu_dit' % prefix in json_data['fields'] and json_data['fields']['%slieu_dit' % prefix]:
|
||
demandeur["adresse"]["lieu_dit"] = normalize(json_data['fields']['%slieu_dit' % prefix])
|
||
|
||
payload["demandeurs"] = [demandeur]
|
||
|
||
# log the payload
|
||
self.log_json_payload(payload)
|
||
|
||
# prepare files that will be forwarded
|
||
files = []
|
||
for k in file_keys:
|
||
if (
|
||
k in json_data['fields']
|
||
and json_data['fields'][k]
|
||
and isinstance(json_data['fields'][k], dict)
|
||
and 'content' in json_data['fields'][k]
|
||
):
|
||
content = base64.b64decode(json_data['fields'][k]['content'])
|
||
content_type = magic.from_buffer(content, mime=True)
|
||
upload_file = ContentFile(content)
|
||
file_hash = self.file_digest(upload_file)
|
||
filename = file_hash[45:] + '.pdf'
|
||
if 'content_type' in json_data['fields'][k]:
|
||
content_type = json_data['fields'][k]['content_type']
|
||
if k == 'cerfa' and content_type != 'application/pdf':
|
||
self.logger.warning("CERFA content type is '%s' instead of '%s'", content_type, 'application/pdf')
|
||
if 'filename' in json_data['fields'][k]:
|
||
filename = json_data['fields'][k]['filename']
|
||
files.append({
|
||
'type_fichier' : 'CERFA' if k == 'cerfa' else 'plan',
|
||
'orig_filename': filename,
|
||
'content_type' : content_type,
|
||
'file_hash' : file_hash,
|
||
'upload_file' : upload_file
|
||
})
|
||
|
||
# log files to be forwarded
|
||
self.logger.debug("----- files (begining) -----")
|
||
self.logger.debug(files)
|
||
self.logger.debug("----- files (end) -----")
|
||
|
||
# make a request to openADS.API (with the payload)
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossiers/%s' % type_dossier)
|
||
response = self.requests.post(
|
||
url,
|
||
json=payload,
|
||
timeout=self.openADS_API_timeout
|
||
)
|
||
|
||
# response is an error code
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning("Request [POST] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response JSON content
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# get the recepisse
|
||
recepisse = self.get_first_file_from_json_payload(result, title='response')
|
||
|
||
# ensure recepisse content type is PDF
|
||
if (
|
||
'content_type' in recepisse
|
||
and recepisse['content_type']
|
||
and recepisse['content_type'] != 'application/pdf'
|
||
):
|
||
self.logger.debug(
|
||
"Forcing 'recepisse' content type to '%s' instead of '%s'.",
|
||
'application/pdf',
|
||
recepisse['content_type']
|
||
)
|
||
recepisse['content_type'] = 'application/pdf'
|
||
|
||
# decode the recepisse from base 64
|
||
try:
|
||
recepisse_content = base64.b64decode(recepisse['b64_content'])
|
||
except TypeError:
|
||
raise APIError('Failed to decode recepisse content from base 64')
|
||
self.logger.debug("Successfully decoded recepisse from base 64")
|
||
|
||
# check/get the 'numero_dossier'
|
||
if 'numero_dossier' not in result:
|
||
raise APIError("Expecting 'numero_dossier' key in JSON response")
|
||
|
||
numero_dossier = result.get('numero_dossier')
|
||
|
||
if not isinstance(numero_dossier, str) and not isinstance(numero_dossier, unicode):
|
||
raise APIError(
|
||
"Expecting '%s' value in JSON response to be a %s (not a %s)" %
|
||
('numero_dossier', 'string', str(type(numero_dossier))))
|
||
|
||
self.logger.debug("Numéro dossier: %s", str(numero_dossier))
|
||
|
||
# save files to be forwarded to openADS.API
|
||
if files:
|
||
file_ids = []
|
||
for f in files:
|
||
rand_id = base64.urlsafe_b64encode(os.urandom(6))
|
||
FF = ForwardFile()
|
||
FF.numero_demande = rand_id
|
||
FF.numero_dossier = numero_dossier
|
||
for k in ['type_fichier', 'orig_filename', 'content_type', 'file_hash']:
|
||
setattr(FF, k, f[k])
|
||
FF.upload_file.save(FF.orig_filename, f['upload_file'])
|
||
FF.upload_status = 'pending'
|
||
FF.save()
|
||
self.logger.debug(
|
||
"Created ForwardFile '%s' for file '%s' (%s)",
|
||
FF.id,
|
||
FF.orig_filename,
|
||
FF.upload_file.path
|
||
)
|
||
file_ids.append(FF.id)
|
||
|
||
job = self.add_job('upload_user_files',
|
||
natural_id=numero_dossier,
|
||
type_dossier=type_dossier,
|
||
numero_dossier=numero_dossier,
|
||
file_ids=file_ids)
|
||
self.logger.debug(
|
||
"Added a job '%s' for dossier '%s' (%s) with file ids '%s'",
|
||
job.id,
|
||
numero_dossier,
|
||
type_dossier,
|
||
','.join([str(fid) for fid in file_ids])
|
||
)
|
||
|
||
# respond with the 'numero_dossier' and the recepisse file
|
||
return {
|
||
'numero_dossier': numero_dossier,
|
||
'recepisse' : recepisse
|
||
}
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about an openADS 'dossier'"),
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about an openADS 'dossier'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_DOSSIER_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_dossier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||
|
||
# make a request to openADS.API
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s' % (type_dossier, numero_dossier))
|
||
response = self.requests.get(url)
|
||
|
||
# response is an error
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response as JSON
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# log the response
|
||
self.log_json_payload(result, 'response')
|
||
|
||
# return the response as-is
|
||
return response.json()
|
||
|
||
|
||
def upload2ForwardFile(self, path, numero_dossier, type_fichier='CERFA'):
|
||
"""Convert a file path to a ForwardFile."""
|
||
if path:
|
||
rand_id = base64.urlsafe_b64encode(os.urandom(6))
|
||
fwd_file = ForwardFile()
|
||
fwd_file.numero_demande = rand_id
|
||
fwd_file.numero_dossier = numero_dossier
|
||
fwd_file.type_fichier = type_fichier
|
||
fwd_file.orig_filename = os.path.basename(path)
|
||
fwd_file.content_type = magic.from_file(path, mime=True)
|
||
with open(path, 'r') as fp:
|
||
fwd_file.file_hash = self.file_digest(fp)
|
||
fwd_file.upload_file = File(open(path, 'r'))
|
||
fwd_file.upload_status = 'pending'
|
||
return fwd_file
|
||
return None
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about the forwarding of user files to openADS"),
|
||
parameters={
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about the forwarding of user files to openADS"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_fwd_files(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||
payload = []
|
||
fwd_files = []
|
||
|
||
# search for all files matching the 'numero_dossier' number
|
||
if not fichier_id:
|
||
fwd_files = ForwardFile.objects.filter(numero_dossier=numero_dossier)
|
||
|
||
# search for a single file
|
||
elif fichier_id:
|
||
try:
|
||
fichier_id = int(fichier_id)
|
||
except ValueError:
|
||
raise APIError('fichier_id must be an integer')
|
||
try:
|
||
fwd_files = [ForwardFile.objects.get(id=fichier_id)]
|
||
except ForwardFile.DoesNotExist:
|
||
raise Http404("No file matches 'numero_dossier=%s' and 'id=%s'." % (numero_dossier, fichier_id))
|
||
|
||
# append each file to the response payload
|
||
for fwd_file in fwd_files:
|
||
payload.append({
|
||
'id' : fwd_file.id,
|
||
'numero_demande': fwd_file.numero_demande,
|
||
'numero_dossier': fwd_file.numero_dossier,
|
||
'type_fichier' : fwd_file.type_fichier,
|
||
'file_hash' : fwd_file.file_hash,
|
||
'orig_filename' : fwd_file.orig_filename,
|
||
'content_type' : fwd_file.content_type,
|
||
'upload_status' : fwd_file.upload_status,
|
||
'upload_attempt': fwd_file.upload_attempt,
|
||
'upload_msg' : fwd_file.upload_msg,
|
||
'content_size' : fwd_file.upload_file.size if fwd_file.upload_file else 0,
|
||
'last_update_datetime' : fwd_file.last_update_datetime
|
||
})
|
||
|
||
# return the payload containing the list of files
|
||
return payload
|
||
|
||
|
||
@endpoint(
|
||
description=_("Get informations about the forwarding of a user file to openADS"),
|
||
parameters={
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get informations about the forwarding of a user file to openADS"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_fwd_files_status(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||
|
||
# get all files matching 'numero_dossier' and 'fichier_id'
|
||
fwd_files = self.get_fwd_files(request, numero_dossier, fichier_id, *args, **kwargs)
|
||
|
||
# prepare the response payload
|
||
payload = {
|
||
'all_forwarded': True,
|
||
'pending' : [],
|
||
'uploading' : [],
|
||
'success' : [],
|
||
'failed' : []
|
||
}
|
||
|
||
# build a summary of all files statuses
|
||
for fwd_file in fwd_files:
|
||
status_msg = '[%s] %s => %s' % (
|
||
fwd_file['id'],
|
||
fwd_file['orig_filename'],
|
||
fwd_file['upload_msg']
|
||
)
|
||
payload[fwd_file['upload_status']].append(status_msg)
|
||
if fwd_file['upload_status'] != 'success':
|
||
payload['all_forwarded'] = False
|
||
|
||
# respond with the payload
|
||
return payload
|
||
|
||
|
||
@endpoint(
|
||
description= _("Get a 'courrier' from an openADS 'dossier'"),
|
||
pattern='^(?P<type_dossier>\w+)/?$',
|
||
example_pattern='{type_dossier}/',
|
||
parameters={
|
||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||
},
|
||
#~ get={
|
||
#~ 'description': _("Get a 'courrier' from an openADS 'dossier'"),
|
||
#~ 'response_body': {
|
||
#~ 'schema': {
|
||
#~ 'application/json': JSON_SCHEMA_GET_COURRIER_OUT
|
||
#~ }
|
||
#~ }
|
||
#~ }
|
||
)
|
||
def get_courrier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||
|
||
# make a request to openADS.API
|
||
url = urlparse.urljoin(
|
||
self.openADS_API_url,
|
||
'/dossier/%s/%s/courrier/%s' % (type_dossier, numero_dossier, 'dia_renonciation_preempter'))
|
||
response = self.requests.get(url)
|
||
|
||
# response is an error
|
||
if response.status_code // 100 != 2:
|
||
error = self.get_response_error(response)
|
||
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
|
||
raise APIError(error)
|
||
|
||
# load the response as JSON
|
||
try:
|
||
result = response.json()
|
||
except ValueError:
|
||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||
|
||
# log the response (filtering the file content)
|
||
self.log_json_payload(
|
||
result, 'response',
|
||
['files.%i.b64_content' % i for i in range(0, len(result['files']))])
|
||
|
||
# get the courrier
|
||
courrier = self.get_first_file_from_json_payload(result, title='response')
|
||
|
||
# decode the courrier from base 64
|
||
try:
|
||
courrier_content = base64.b64decode(courrier['b64_content'])
|
||
except TypeError:
|
||
raise APIError('Invalid content for courrier')
|
||
|
||
# return the 'courrier' file
|
||
return {'courrier': courrier}
|
||
|
||
|
||
def get_response_error(self, response):
|
||
"""Return a error string from an HTTP response."""
|
||
try:
|
||
# load the response as JSON
|
||
result = response.json()
|
||
|
||
# collect errors and turn them into messages (multispaces are filtered)
|
||
errors = result.get('errors')
|
||
msg = []
|
||
if errors:
|
||
for error in errors:
|
||
location = error.get('location')
|
||
name = error.get('name')
|
||
desc = error.get('description')
|
||
msg.append('[%s] (%s) %s' % (location, normalize(name), normalize(desc)))
|
||
|
||
# if there are messages
|
||
if msg:
|
||
|
||
# return a string representing the HTTP error
|
||
return "HTTP error: %s, %s" % (response.status_code, ','.join(msg))
|
||
|
||
except ValueError:
|
||
pass
|
||
|
||
# TODO ask for openADS.API to *always* send JSON formatted errors, not HTML ones
|
||
# return a string representing the HTTP error (filtering the HTML tags and multispaces)
|
||
return "HTTP error: %s, %s" % \
|
||
(response.status_code,
|
||
clean_spaces(strip_tags(response.content[:1000])) if response.content else '')
|
||
|
||
|
||
def upload_user_files(self, type_dossier, numero_dossier, file_ids):
|
||
"""A Job to forward user uploaded files to openADS."""
|
||
|
||
payload = []
|
||
fwd_files = []
|
||
|
||
# for every file ids specified (in parameters of this job)
|
||
for fid in file_ids:
|
||
self.logger.debug("upload_user_files() ForwardFile file_id: %s", fid)
|
||
|
||
# get the matching forward file
|
||
fwd_file = ForwardFile.objects.get(id=fid)
|
||
|
||
# found one
|
||
if fwd_file:
|
||
self.logger.debug("upload_user_files() got ForwardFile")
|
||
|
||
# add the file content and data to the payload
|
||
payload.append({
|
||
'filename' : fwd_file.orig_filename + ('.pdf' if fwd_file.orig_filename[-4:] != '.pdf' else ''),
|
||
'content_type' : fwd_file.content_type,
|
||
'b64_content' : base64.b64encode(fwd_file.upload_file.read()),
|
||
'file_type' : fwd_file.type_fichier
|
||
})
|
||
self.logger.debug("upload_user_files() payload added")
|
||
|
||
# update the file upload data (status and attempts)
|
||
fwd_file.upload_status = 'uploading'
|
||
fwd_file.upload_attempt += 1
|
||
fwd_file.upload_msg = 'attempt %s' % fwd_file.upload_attempt
|
||
self.logger.debug("upload_user_files() upload_msg: '%s'", fwd_file.upload_msg)
|
||
fwd_file.save()
|
||
self.logger.debug("upload_user_files() ForwardFile saved")
|
||
|
||
# append the forwarded file to the list
|
||
fwd_files.append(fwd_file)
|
||
|
||
# not found the forwarded file specified: bug
|
||
else:
|
||
self.logger.warning("upload_user_files() failed to find ForwardFile file_id: %s", fid)
|
||
|
||
# if files need to be forwarded
|
||
if payload:
|
||
|
||
self.logger.debug("upload_user_files() payload is not empty")
|
||
|
||
# log the payload
|
||
self.log_json_payload(
|
||
payload, 'payload',
|
||
['%i.b64_content' % i for i in range(0, len(fwd_files))])
|
||
|
||
# make the request to openADS.API (with a specific timeout)
|
||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s/files' % (type_dossier, numero_dossier))
|
||
response = self.requests.post(
|
||
url,
|
||
json=payload,
|
||
timeout=self.openADS_API_timeout
|
||
)
|
||
|
||
# reponse is an error
|
||
if response.status_code // 100 != 2:
|
||
|
||
# update every files status as 'failed' and save the error message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'failed'
|
||
fwd_file.upload_msg = self.get_response_error(response)
|
||
fwd_file.save()
|
||
|
||
# log (warning) the error message
|
||
self.logger.warning(
|
||
"upload_user_files() openADS response is not OK (code: %s) for dossier '%s' and files '%s'",
|
||
response.status_code,
|
||
numero_dossier,
|
||
','.join(file_ids)
|
||
)
|
||
|
||
# response is not an error
|
||
else:
|
||
|
||
# load the reponse as JSON
|
||
try:
|
||
result = response.json()
|
||
|
||
# in case of failure
|
||
except ValueError:
|
||
|
||
# update every files status as 'failed' and save the error message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'failed'
|
||
fwd_file.upload_msg = 'No JSON content returned: %r' % response.content[:1000]
|
||
fwd_file.save()
|
||
|
||
# log (warning) the error message
|
||
self.logger.warning(
|
||
"upload_user_files() openADS response is not JSON valid for dossier '%s' and files '%s'",
|
||
numero_dossier,
|
||
','.join([f.id for f in fwd_files])
|
||
)
|
||
|
||
# response correctly loaded as JSON
|
||
else:
|
||
|
||
# TODO handle response (now its just an informational sentence in key 'data')
|
||
|
||
# update every files status as 'success' and save the success message
|
||
for fwd_file in fwd_files:
|
||
fwd_file.upload_status = 'success'
|
||
fwd_file.upload_msg = 'uploaded successfuly'
|
||
|
||
# delete file content (on success)
|
||
fpath = fwd_file.upload_file.path
|
||
fwd_file.upload_file.delete()
|
||
|
||
# save the file
|
||
fwd_file.save()
|
||
|
||
# log the success message
|
||
self.logger.debug(
|
||
"upload_user_files() flaging file '%s' has transfered (deleted '%s')",
|
||
fwd_file.id,
|
||
fpath
|
||
)
|
||
|
||
# no file need to be forwarded
|
||
else:
|
||
self.logger.warning(
|
||
"upload_user_files() payload is empty for dossier '%s' and files '%s'",
|
||
numero_dossier,
|
||
','.join(file_ids)
|
||
)
|
||
|
||
|
||
# copy-pasted from 'wcs/qommon/misc.py'
|
||
def file_digest(self, content, chunk_size=100000):
|
||
"""Return a hash for the content specified."""
|
||
digest = hashlib.sha256()
|
||
content.seek(0)
|
||
def read_chunk():
|
||
return content.read(chunk_size)
|
||
for chunk in iter(read_chunk, ''):
|
||
digest.update(chunk)
|
||
return digest.hexdigest()
|
||
|