Use HTTPResource, JSON schema for inputs, JSON lazy logging, added ForwardFile.attempt, and more
Features:
* class AtrealOpenads now extends HTTPResource for HTTP Basic credentials
* check_status is now also an endpoint (endpoint 'test_openads_connexion' deleted)
* added JSON schema to validate POST request input (ouput schemas are defined too but unused for now)
* added class 'LogJsonPayloadWithFileContent' to lazy log a json payload with content file filtered
* added 'attempt' field to class 'ForwardFile' that count the attempt of uploading to openADS.API
* added more controls over response data received
Refactoring:
* removed the file content in function 'get_fwd_files_status()' and added content size instead
* splitted the function 'get_fwd_files_status()' in two: one that retrieve detailed files, one only the summary
* added function 'log_json_payload()' to help to log json payloads
* added 2 functions to factorize the process of getting files from json payload
* added function 'check_file_dict()' to factorize the process of checking a file dictionary
Fixes:
* Default filename are now under 50 chars
* Make the function 'get_fwd_files_status()' consistent in its return
Tests:
* updated the test to reflect the use of HTTPResource and the split of 'get_fwd_files_status()'
More:
* Added a lot of comments
* Added endpoint anotation to validate response body, but its commented as the 'get' helper is not defined (not like the 'post' one)
This commit is contained in:
Michael Bideau
parent
85e815f72b
commit
02f4a071fc
|
|
@ -0,0 +1,339 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# passerelle - uniform access to multiple data sources and services
|
||||
# Copyright (C) 2018 Entr'ouvert
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU Affero General Public License as published
|
||||
# by the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
|
||||
|
||||
# TODO add string limits (maxLength)
|
||||
|
||||
|
||||
JSON_SCHEMA_FILE = {
|
||||
"description": "A file object",
|
||||
"$id" : "#file",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content" : { "type": "string" },
|
||||
"content_type": { "type": ["string","null"] },
|
||||
"filename" : { "type": "string" }
|
||||
},
|
||||
"required": ["content","filename"]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_FILE_B64 = {
|
||||
"description": "A file object encoded in base64",
|
||||
"$id" : "#file",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"b64_content" : { "type": "string" },
|
||||
"content_type": { "type": ["string","null"] },
|
||||
"filename" : { "type": "string" }
|
||||
},
|
||||
"required": ["b64_content","filename"]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_DATE_FRENCH = {
|
||||
"type": "string",
|
||||
"pattern": "^(0?[1-9]|[12][0-9]|3[01])/(0?[1-9]|1[012])/\d{4}$"
|
||||
}
|
||||
|
||||
JSON_SCHEMA_CHECK_STATUS_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of an openADS 'connexion' test",
|
||||
"type" : "object",
|
||||
"properties": {
|
||||
"response": { "type": "integer" }
|
||||
},
|
||||
"required": ["response"]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_CREATE_DOSSIER_IN = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Incoming request to create a 'dossier' in openADS.API",
|
||||
"definitions": {
|
||||
"refs-cadastrales": {
|
||||
"description": "The 3 items of a 'cadastrale' reference",
|
||||
"$id" : "#refs-cadastrales",
|
||||
"type" : "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"minItems": 3,
|
||||
"maxItems": 3
|
||||
},
|
||||
"file": JSON_SCHEMA_FILE,
|
||||
"file_plan_cadastral": {
|
||||
"description": "A 'plan cadastral' document file",
|
||||
"anyOf": [{ "$ref": "#/definitions/file" }, { "type": "null" }]
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"fields": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cerfa": {
|
||||
"description": "A 'CERFA' PDF document file",
|
||||
"type": "object",
|
||||
"allOf": [{"$ref": "#/definitions/file" }]
|
||||
},
|
||||
"plan_cadastral_1": { "$ref": "#/definitions/file_plan_cadastral" },
|
||||
"plan_cadastral_2": { "$ref": "#/definitions/file_plan_cadastral" },
|
||||
"plan_cadastral_3": { "$ref": "#/definitions/file_plan_cadastral" },
|
||||
"plan_cadastral_4": { "$ref": "#/definitions/file_plan_cadastral" },
|
||||
"plan_cadastral_5": { "$ref": "#/definitions/file_plan_cadastral" },
|
||||
"terrain_numero_voie" : { "type": "string" },
|
||||
"terrain_nom_voie" : { "type": "string" },
|
||||
"terrain_code_postal" : { "type": "string" },
|
||||
"terrain_localite" : { "type": "string" },
|
||||
"terrain_lieu_dit" : { "type": ["string","null"] },
|
||||
"reference_cadastrale": {
|
||||
"description": "A list of 'cadastrales' references",
|
||||
"type": "array",
|
||||
"items": { "$ref": "#/definitions/refs-cadastrales" }
|
||||
},
|
||||
"autres_parcelles": {
|
||||
#"enum": ["Oui","Non"]
|
||||
"type": "boolean"
|
||||
},
|
||||
"references_cadastrales": {
|
||||
"description": "A list of 'cadastrales' references",
|
||||
# conditionaly required and typed below
|
||||
},
|
||||
"proprietaire" : { "enum": ["Oui","Non"] },
|
||||
"proprietaire_qualite" : { "type": "string" },
|
||||
"nom" : { "type": "string" },
|
||||
"prenom" : { "type": "string" },
|
||||
"numero_voie" : { "type": "string" },
|
||||
"nom_voie" : { "type": "string" },
|
||||
"code_postal" : { "type": "string" },
|
||||
"localite" : { "type": "string" },
|
||||
"lieu_dit" : { "type": ["string","null"] },
|
||||
"mandataire_nom" : { }, # conditionaly required and typed below
|
||||
"mandataire_prenom" : { }, # conditionaly required and typed below
|
||||
"mandataire_numero_voie": { }, # conditionaly required and typed below
|
||||
"mandataire_nom_voie" : { }, # conditionaly required and typed below
|
||||
"mandataire_code_postal": { }, # conditionaly required and typed below
|
||||
"mandataire_localite" : { }, # conditionaly required and typed below
|
||||
"mandataire_lieu_dit" : { } # conditionaly required and typed below
|
||||
},
|
||||
|
||||
# requirements
|
||||
"required": [
|
||||
"cerfa",
|
||||
"terrain_numero_voie",
|
||||
"terrain_nom_voie",
|
||||
"terrain_code_postal",
|
||||
"terrain_localite",
|
||||
"reference_cadastrale",
|
||||
"proprietaire",
|
||||
"proprietaire_qualite"
|
||||
],
|
||||
|
||||
# conditional requirements
|
||||
"allOf": [
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"autres_parcelles": { "const": False },
|
||||
"references_cadastrales": { "type": "null" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"autres_parcelles": { "const": True },
|
||||
"references_cadastrales": {
|
||||
"type": "array",
|
||||
"items": { "$ref": "#/definitions/refs-cadastrales" }
|
||||
}
|
||||
},
|
||||
"required": ["autres_parcelles", "references_cadastrales"]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"anyOf": [
|
||||
{
|
||||
"properties": {
|
||||
"proprietaire": { "const": "Oui" },
|
||||
"mandataire_nom" : { "type": "null" },
|
||||
"mandataire_prenom" : { "type": "null" },
|
||||
"mandataire_numero_voie": { "type": "null" },
|
||||
"mandataire_nom_voie" : { "type": "null" },
|
||||
"mandataire_code_postal": { "type": "null" },
|
||||
"mandataire_localite" : { "type": "null" },
|
||||
"mandataire_lieu_dit" : { "type": "null" }
|
||||
},
|
||||
"required": [
|
||||
"nom",
|
||||
"prenom",
|
||||
"numero_voie",
|
||||
"nom_voie",
|
||||
"code_postal",
|
||||
"localite"
|
||||
]
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"proprietaire": { "const": "Non" },
|
||||
"mandataire_nom" : { "type": "string" },
|
||||
"mandataire_prenom" : { "type": "string" },
|
||||
"mandataire_numero_voie": { "type": "string" },
|
||||
"mandataire_nom_voie" : { "type": "string" },
|
||||
"mandataire_code_postal": { "type": "string" },
|
||||
"mandataire_localite" : { "type": "string" },
|
||||
"mandataire_lieu_dit" : { "type": ["string","null"] }
|
||||
},
|
||||
"required": [
|
||||
"mandataire_nom",
|
||||
"mandataire_prenom",
|
||||
"mandataire_numero_voie",
|
||||
"mandataire_nom_voie",
|
||||
"mandataire_code_postal",
|
||||
"mandataire_localite"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": ["fields"]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_CREATE_DOSSIER_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of a 'dossier' creation in openADS.API",
|
||||
"type" : "object",
|
||||
"properties": {
|
||||
"numero_dossier": { "type": "string" },
|
||||
"recepisse": JSON_SCHEMA_FILE_B64
|
||||
},
|
||||
"required": ["numero_dossier", "recepisse"]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_GET_DOSSIER_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of a 'dossier' creation in openADS.API",
|
||||
"type" : "object",
|
||||
"properties": {
|
||||
"etat" : { "type": "string" },
|
||||
"date_depot" : JSON_SCHEMA_DATE_FRENCH,
|
||||
"date_decision": JSON_SCHEMA_DATE_FRENCH,
|
||||
"date_limite_instruction": JSON_SCHEMA_DATE_FRENCH,
|
||||
"decision" : { "type": "string" }
|
||||
},
|
||||
"required": [
|
||||
"etat",
|
||||
"date_depot",
|
||||
"date_decision",
|
||||
"date_limite_instruction",
|
||||
"decision"
|
||||
]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_FORWARDFILE = {
|
||||
"description": "A ForwardFile object (PDF document that must be forwarded to openADS)",
|
||||
"$id" : "#forwardfile",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id" : { "type": "integer" },
|
||||
"numero_demande" : { "type": "string" },
|
||||
"numero_dossier" : { "type": "string" },
|
||||
"type_fichier" : { "type": "string" },
|
||||
"file_hash" : { "type": "string" },
|
||||
"orig_filename" : { "type": "string" },
|
||||
"content_type" : { "type": "string" },
|
||||
"upload_status" : { "type": "string" },
|
||||
"upload_attempt" : { "type": "integer" },
|
||||
"upload_msg" : { "type": "string" },
|
||||
"content_size" : { "type": "integer" },
|
||||
"last_update_datetime": { "type": "string", "format": "date-time" }
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"numero_demande",
|
||||
"numero_dossier",
|
||||
"type_fichier",
|
||||
"file_hash",
|
||||
"orig_filename",
|
||||
"content_type",
|
||||
"upload_status",
|
||||
"upload_attempt",
|
||||
"upload_msg",
|
||||
"content_size",
|
||||
"last_update_datetime"
|
||||
]
|
||||
}
|
||||
|
||||
JSON_SCHEMA_GET_FWD_FILES_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of a request about the forwarding (detailled) of user files to openADS",
|
||||
"type" : "array",
|
||||
"items": { "$ref": "#/definitions/forwardfile" },
|
||||
"definitions": {
|
||||
"forwardfile" : JSON_SCHEMA_FORWARDFILE
|
||||
}
|
||||
}
|
||||
|
||||
JSON_SCHEMA_GET_FWD_FILE_STATUS = {
|
||||
"description": "The status of a ForwardFile",
|
||||
"$id" : "#forwardfile-status",
|
||||
"type": "string",
|
||||
"pattern": "^\[\w+\] .+ => .+$"
|
||||
}
|
||||
|
||||
JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of a request about the forwarding (summarized) of user files to openADS",
|
||||
"type" : "object",
|
||||
"properties": {
|
||||
"all_forwarded": { "type": "boolean" },
|
||||
"pending" : { "type": "array", "items": { "$ref": "#/definitions/forwardfile-status" } },
|
||||
"uploading": { "type": "array", "items": { "$ref": "#/definitions/forwardfile-status" } },
|
||||
"success" : { "type": "array", "items": { "$ref": "#/definitions/forwardfile-status" } },
|
||||
"failed" : { "type": "array", "items": { "$ref": "#/definitions/forwardfile-status" } }
|
||||
},
|
||||
"required": ["all_forwarded","pending","uploading","success","failed"],
|
||||
"definitions": {
|
||||
"forwardfile-status" : JSON_SCHEMA_GET_FWD_FILE_STATUS
|
||||
}
|
||||
}
|
||||
|
||||
JSON_SCHEMA_GET_COURRIER_OUT = {
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title" : "Response of a 'courrier' from an openADS 'dossier'",
|
||||
"type" : "object",
|
||||
"properties": {
|
||||
"courrier": JSON_SCHEMA_FILE_B64
|
||||
},
|
||||
"required": ["courrier"]
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
fmt = '\n----- %s -----\n%s\n'
|
||||
print(fmt % ('JSON_SCHEMA_CHECK_STATUS_OUT' , json.dumps(JSON_SCHEMA_CHECK_STATUS_OUT)))
|
||||
print(fmt % ('JSON_SCHEMA_CREATE_DOSSIER_IN' , json.dumps(JSON_SCHEMA_CREATE_DOSSIER_IN)))
|
||||
print(fmt % ('JSON_SCHEMA_CREATE_DOSSIER_OUT' , json.dumps(JSON_SCHEMA_CREATE_DOSSIER_OUT)))
|
||||
print(fmt % ('JSON_SCHEMA_GET_DOSSIER_OUT' , json.dumps(JSON_SCHEMA_GET_DOSSIER_OUT)))
|
||||
print(fmt % ('JSON_SCHEMA_GET_FWD_FILES_OUT' , json.dumps(JSON_SCHEMA_GET_FWD_FILES_OUT)))
|
||||
print(fmt % ('JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT',
|
||||
json.dumps(JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT)))
|
||||
print(fmt % ('JSON_SCHEMA_GET_COURRIER_OUT' , json.dumps(JSON_SCHEMA_GET_COURRIER_OUT)))
|
||||
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Generated by Django 1.11.15 on 2019-07-11 16:20
|
||||
# Generated by Django 1.11.15 on 2019-07-18 15:52
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import migrations, models
|
||||
import atreal_openads.models
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
|
@ -22,9 +22,13 @@ class Migration(migrations.Migration):
|
|||
('title', models.CharField(max_length=50, verbose_name='Title')),
|
||||
('description', models.TextField(verbose_name='Description')),
|
||||
('slug', models.SlugField(unique=True, verbose_name='Identifier')),
|
||||
('basic_auth_username', models.CharField(blank=True, max_length=128, verbose_name='Basic authentication username')),
|
||||
('basic_auth_password', models.CharField(blank=True, max_length=128, verbose_name='Basic authentication password')),
|
||||
('client_certificate', models.FileField(blank=True, null=True, upload_to=b'', verbose_name='TLS client certificate')),
|
||||
('trusted_certificate_authorities', models.FileField(blank=True, null=True, upload_to=b'', verbose_name='TLS trusted CAs')),
|
||||
('verify_cert', models.BooleanField(default=True, verbose_name='TLS verify certificates')),
|
||||
('http_proxy', models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy')),
|
||||
('collectivite', models.CharField(blank=True, default=b'', help_text='ex: Marseille, or ex: 3', max_length=255, verbose_name='Collectivity (identifier)')),
|
||||
('openADS_API_login', models.CharField(default=b'', help_text='ex: user1234', max_length=255, verbose_name='openADS API login')),
|
||||
('openADS_API_password', models.CharField(default=b'', help_text='ex: ah9pGbKKHv5ToF3cPQuV', max_length=255, verbose_name='openADS API password')),
|
||||
('openADS_API_url', models.URLField(default=b'', help_text='ex: https://openads.your_domain.net/api/', max_length=255, verbose_name='openADS API URL')),
|
||||
('extra_debug', models.BooleanField(default=0, help_text='ex: True', verbose_name='Extra debug')),
|
||||
('users', models.ManyToManyField(blank=True, related_name='_atrealopenads_users_+', related_query_name='+', to='base.ApiUser')),
|
||||
|
|
@ -44,6 +48,7 @@ class Migration(migrations.Migration):
|
|||
('orig_filename', models.CharField(blank=True, default=b'', max_length=100)),
|
||||
('content_type', models.CharField(blank=True, default=b'', max_length=100)),
|
||||
('upload_file', models.FileField(null=True, upload_to=atreal_openads.models.get_upload_path)),
|
||||
('upload_attempt', models.PositiveIntegerField(blank=True, default=0)),
|
||||
('upload_status', models.CharField(blank=True, default=b'', max_length=10)),
|
||||
('upload_msg', models.CharField(blank=True, default=b'', max_length=255)),
|
||||
('last_update_datetime', models.DateTimeField(auto_now=True)),
|
||||
|
|
|
|||
|
|
@ -35,10 +35,20 @@ from django.utils.translation import ugettext_lazy as _
|
|||
from django.core.files import File
|
||||
from django.core.files.base import ContentFile
|
||||
|
||||
from passerelle.base.models import BaseResource
|
||||
from passerelle.base.models import BaseResource, HTTPResource
|
||||
from passerelle.utils.api import endpoint
|
||||
from passerelle.utils.jsonresponse import APIError
|
||||
|
||||
from .json_schemas import (
|
||||
JSON_SCHEMA_CHECK_STATUS_OUT,
|
||||
JSON_SCHEMA_CREATE_DOSSIER_IN,
|
||||
JSON_SCHEMA_CREATE_DOSSIER_OUT,
|
||||
JSON_SCHEMA_GET_DOSSIER_OUT,
|
||||
JSON_SCHEMA_GET_FWD_FILES_OUT,
|
||||
JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT,
|
||||
JSON_SCHEMA_GET_COURRIER_OUT
|
||||
)
|
||||
|
||||
|
||||
class MLStripper(HTMLParser):
|
||||
"""HTML parser that removes html tags."""
|
||||
|
|
@ -95,6 +105,71 @@ def get_upload_path(instance, filename):
|
|||
)
|
||||
|
||||
|
||||
def dict_deref_multi(data, keys):
|
||||
"""Access a dict through a list of keys.
|
||||
Inspired by: https://stackoverflow.com/a/47969823
|
||||
"""
|
||||
if keys:
|
||||
key = int(keys[0]) if isinstance(data, list) else keys[0]
|
||||
return dict_deref_multi(data[key], keys[1:]) if keys else data
|
||||
|
||||
# TODO implement wildcard filtering
|
||||
def dict_replace_content_by_paths(dic, paths_to_replace=[], path_separator='/', replace_by='<file content>'):
|
||||
"""Replace a dict content, with a string, from a list of paths."""
|
||||
newdic = dic
|
||||
if newdic and (isinstance(newdic, dict) or isinstance(newdic, list)):
|
||||
for p in paths_to_replace:
|
||||
if p:
|
||||
if p.startswith(path_separator):
|
||||
p = p[1:]
|
||||
items = p.split(path_separator)
|
||||
if items:
|
||||
try:
|
||||
value = dict_deref_multi(newdic, items)
|
||||
except KeyError:
|
||||
continue
|
||||
except TypeError:
|
||||
continue
|
||||
if not isinstance(value, str) and not isinstance(value, unicode):
|
||||
raise ValueError("path '%s' is not a string ([%s] %s)" % (p, str(type(value)), str(value)))
|
||||
# newdic is still untouched
|
||||
if newdic is dic:
|
||||
# create a copy of it
|
||||
newdic = copy.deepcopy(dic)
|
||||
last_item = items.pop(-1)
|
||||
value = dict_deref_multi(newdic, items)
|
||||
value[last_item] = replace_by
|
||||
return newdic
|
||||
|
||||
|
||||
class LogJsonPayloadWithFileContent(object):
|
||||
""" Return a string representation of a JSON payload with its file content emptyed."""
|
||||
|
||||
def __init__(self, payload, paths_to_replace=[], path_separator='/', replace_by='<data>'):
|
||||
""" arguments:
|
||||
- payload string the JSON payload to dump
|
||||
- paths_to_replace array a list of strings either:
|
||||
* in 'path' format (i.e.: ['/fields/file/content',...])
|
||||
* in 'namespace' format (i.e.: ['fields.file.content',...])
|
||||
- path_separator string the path separator used in 'paths_to_replace' format:
|
||||
* '/' for 'path' format
|
||||
* '.' for 'namespace' format
|
||||
- replace_by string the new value of the content at the specified path
|
||||
"""
|
||||
self.payload = payload
|
||||
self.paths_to_replace = paths_to_replace
|
||||
self.path_separator = path_separator
|
||||
self.replace_by = replace_by
|
||||
|
||||
def __str__(self):
|
||||
return json.dumps(dict_replace_content_by_paths(
|
||||
self.payload,
|
||||
paths_to_replace = self.paths_to_replace,
|
||||
path_separator = self.path_separator,
|
||||
replace_by = self.replace_by
|
||||
))
|
||||
|
||||
|
||||
class ForwardFile(models.Model):
|
||||
"""Represent a file uploaded by a user, to be forwarded to openADS.API."""
|
||||
numero_demande = models.CharField(max_length=20)
|
||||
|
|
@ -104,20 +179,17 @@ class ForwardFile(models.Model):
|
|||
orig_filename = models.CharField(max_length=100, default='', blank=True)
|
||||
content_type = models.CharField(max_length=100, default='', blank=True)
|
||||
upload_file = models.FileField(upload_to=get_upload_path, null=True)
|
||||
upload_attempt = models.PositiveIntegerField(default=0, blank=True)
|
||||
upload_status = models.CharField(max_length=10, default='', blank=True)
|
||||
upload_msg = models.CharField(max_length=255, default='', blank=True)
|
||||
last_update_datetime = models.DateTimeField(auto_now=True)
|
||||
|
||||
|
||||
class AtrealOpenads(BaseResource):
|
||||
class AtrealOpenads(BaseResource, HTTPResource):
|
||||
"""API that proxy/relay communications with/to openADS."""
|
||||
|
||||
collectivite = models.CharField(_('Collectivity (identifier)'), max_length=255,
|
||||
help_text=_('ex: Marseille, or ex: 3'), default='', blank=True)
|
||||
openADS_API_login = models.CharField(_('openADS API login'), max_length=255,
|
||||
help_text=_('ex: user1234'), default='')
|
||||
openADS_API_password = models.CharField(_('openADS API password'), max_length=255,
|
||||
help_text=_('ex: ah9pGbKKHv5ToF3cPQuV'), default='')
|
||||
openADS_API_url = models.URLField(_('openADS API URL'), max_length=255,
|
||||
help_text=_('ex: https://openads.your_domain.net/api/'), default='')
|
||||
extra_debug = models.BooleanField(_('Extra debug'),
|
||||
|
|
@ -127,9 +199,7 @@ class AtrealOpenads(BaseResource):
|
|||
|
||||
category = _('Business Process Connectors')
|
||||
|
||||
api_description = _('''
|
||||
This API provides exchanges with openADS.
|
||||
''')
|
||||
api_description = _('''This API provides exchanges with openADS.''')
|
||||
|
||||
class Meta:
|
||||
verbose_name = _('openADS')
|
||||
|
|
@ -147,41 +217,165 @@ class AtrealOpenads(BaseResource):
|
|||
self.logger.debug(*args, **kwargs)
|
||||
|
||||
|
||||
def check_status(self):
|
||||
def log_json_payload(self, payload, title='payload', paths_to_replace=[]):
|
||||
"""Log a json paylod surrounded by dashes and with file content filtered."""
|
||||
self.debug("----- %s (begining) -----", title)
|
||||
if paths_to_replace:
|
||||
self.debug("%s", LogJsonPayloadWithFileContent(
|
||||
payload,
|
||||
paths_to_replace = paths_to_replace,
|
||||
path_separator = '.',
|
||||
replace_by = '<b64 content>'
|
||||
))
|
||||
else:
|
||||
self.debug("%s", LogJsonPayloadWithFileContent(payload))
|
||||
self.debug("----- %s (end) -----", title)
|
||||
|
||||
|
||||
def get_files_from_json_payload(self, payload, title='payload'):
|
||||
"""Return files from a JSON payload with all checks and logging."""
|
||||
|
||||
# check the 'files' key
|
||||
if 'files' not in payload:
|
||||
self.log_json_payload(payload, title)
|
||||
raise APIError("Expecting '%s' key in JSON %s" %
|
||||
('files', title))
|
||||
|
||||
files = payload.get('files')
|
||||
|
||||
if not isinstance(files, list):
|
||||
self.log_json_payload(payload, title)
|
||||
raise APIError(
|
||||
"Expecting '%s' value in JSON %s to be a %s (not a %s)" %
|
||||
('files', title, 'list', str(type(files))))
|
||||
|
||||
if len(files) <= 0:
|
||||
self.log_json_payload(payload, title)
|
||||
raise APIError("Expecting non-empty '%s' value in JSON %s" %
|
||||
('files', title))
|
||||
|
||||
# log the response
|
||||
self.log_json_payload(
|
||||
payload, title
|
||||
,['files.%i.b64_content' % i for i in range(0, len(files))])
|
||||
|
||||
# return the files
|
||||
return files
|
||||
|
||||
|
||||
def check_file_dict(self, dict_file, b64=True, title='payload'):
|
||||
"""Ensure a file dict has all its required items."""
|
||||
|
||||
# key to get the content
|
||||
content_key = 'content'
|
||||
|
||||
# if content is in base 64
|
||||
if b64:
|
||||
content_key = 'b64_content'
|
||||
|
||||
# check content existence
|
||||
if content_key not in dict_file:
|
||||
raise APIError("Expecting 'files.0.%s' key in JSON %s" % (content_key, title))
|
||||
|
||||
# get its content
|
||||
file_content = dict_file[content_key]
|
||||
|
||||
if not isinstance(file_content, str) and not isinstance(file_content, unicode):
|
||||
raise APIError(
|
||||
"Expecting '%s' value in JSON %s to be a %s (not a %s)" %
|
||||
('file.%s' % content_key, title, 'string', str(type(file_content))))
|
||||
|
||||
# check filename
|
||||
if (
|
||||
'filename' in dict_file
|
||||
and not isinstance(dict_file['filename'], str)
|
||||
and not isinstance(dict_file['filename'], unicode)
|
||||
):
|
||||
raise APIError(
|
||||
"Expecting '%s' value in file dict to be a %s (not a %s)" %
|
||||
('file.filename', title, 'string', str(type(dict_file['filename']))))
|
||||
|
||||
|
||||
def get_first_file_from_json_payload(self, payload, title='payload', ensure_content=True, b64=True):
|
||||
"""Return the first file from a JSON payload with all checks and logging."""
|
||||
|
||||
# get all files
|
||||
files = self.get_files_from_json_payload(payload, title)
|
||||
|
||||
# get the first file
|
||||
first = files[0]
|
||||
|
||||
# asked to check its content
|
||||
if ensure_content:
|
||||
self.check_file_dict(first, title, b64)
|
||||
|
||||
# return the first file
|
||||
return first
|
||||
|
||||
|
||||
@endpoint(
|
||||
description=_("Test an openADS 'connexion'")
|
||||
#~ get={
|
||||
#~ 'description': _("Test an openADS 'connexion'"),
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_CHECK_STATUS_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
#~ }
|
||||
)
|
||||
def check_status(self, *args, **kwargs):
|
||||
"""Check avaibility of the openADS.API service."""
|
||||
url = urlparse.urljoin(self.openADS_API_url, '__api__')
|
||||
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
|
||||
response = self.requests.get(url)
|
||||
response.raise_for_status()
|
||||
return {'response': response.status_code}
|
||||
|
||||
|
||||
@endpoint(description="Test an openADS 'connexion'")
|
||||
def test_openads_connexion(self, request):
|
||||
return self.check_status()
|
||||
|
||||
|
||||
@endpoint(
|
||||
description="Create an openADS 'dossier'",
|
||||
methods=['post'],
|
||||
pattern='^(?P<type_dossier>\w+)/?$',
|
||||
example_pattern='{type_dossier}/',
|
||||
parameters={
|
||||
'type_dossier': {'description': _("Type of 'dossier'"), 'example_value': 'DIA'}
|
||||
})
|
||||
},
|
||||
post={'description': _("Create an openADS 'dossier'"),
|
||||
'request_body': {
|
||||
'schema': {
|
||||
'application/json': JSON_SCHEMA_CREATE_DOSSIER_IN
|
||||
}
|
||||
},
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_CREATE_DOSSIER_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
}
|
||||
)
|
||||
def create_dossier(self, request, type_dossier, *args, **kwargs):
|
||||
self.debug("----- request json (begining) -----")
|
||||
json_data = json.loads(request.body)
|
||||
debug_json = copy.deepcopy(json_data)
|
||||
file_keys = ['cerfa'] + ['plan_cadastral_%s' % i for i in range(1,5)]
|
||||
for k in file_keys:
|
||||
if k in debug_json['fields'] \
|
||||
and debug_json['fields'][k] \
|
||||
and isinstance(debug_json['fields'][k], dict) \
|
||||
and 'content' in debug_json['fields'][k]:
|
||||
debug_json['fields'][k]['content'] = '<b64 content>'
|
||||
self.debug(json.dumps(debug_json))
|
||||
self.debug("----- request json (end) -----")
|
||||
|
||||
# loads the request body as JSON content
|
||||
json_data = json.loads(request.body)
|
||||
|
||||
# every field key that might contain a file content
|
||||
file_keys = ['cerfa'] + ['plan_cadastral_%s' % i for i in range(1,5)]
|
||||
|
||||
# detect if there evolutions data to filter them out of logging
|
||||
# TODO replace when wildcard filtering is implement in 'dict_replace_content_by_paths()'
|
||||
evolution_to_filter = []
|
||||
if 'evolution' in json_data and isinstance(json_data['evolution'], list):
|
||||
for e, evol in enumerate(json_data['evolution']):
|
||||
if isinstance(evol, dict) and 'parts' in evol and isinstance(evol['parts'], list):
|
||||
for p, part in enumerate(evol['parts']):
|
||||
if isinstance(part, dict) and 'data' in part:
|
||||
evolution_to_filter.append('evolution.%s.parts.%s.data' % (e, p))
|
||||
|
||||
# log the request body (filtering the files content)
|
||||
self.log_json_payload(
|
||||
json_data, 'request',
|
||||
['fields.%s.content' % k for k in file_keys] + evolution_to_filter)
|
||||
|
||||
# build the payload
|
||||
payload = { "collectivite": self.collectivite }
|
||||
|
||||
payload["terrain"] = {
|
||||
|
|
@ -229,22 +423,23 @@ class AtrealOpenads(BaseResource):
|
|||
|
||||
payload["demandeurs"] = [demandeur]
|
||||
|
||||
self.debug("----- payload (begining) -----")
|
||||
self.debug(json.dumps(payload))
|
||||
self.debug("----- payload (end) -----")
|
||||
# log the payload
|
||||
self.log_json_payload(payload)
|
||||
|
||||
# prepare files that will be forwarded
|
||||
files = []
|
||||
file_keys = ['cerfa'] + ['plan_cadastral_%s' % i for i in range(1,5)]
|
||||
for k in file_keys:
|
||||
if k in json_data['fields'] \
|
||||
and json_data['fields'][k] \
|
||||
and isinstance(json_data['fields'][k], dict) \
|
||||
and 'content' in json_data['fields'][k]:
|
||||
if (
|
||||
k in json_data['fields']
|
||||
and json_data['fields'][k]
|
||||
and isinstance(json_data['fields'][k], dict)
|
||||
and 'content' in json_data['fields'][k]
|
||||
):
|
||||
content = base64.b64decode(json_data['fields'][k]['content'])
|
||||
content_type = magic.from_buffer(content, mime=True)
|
||||
upload_file = ContentFile(content)
|
||||
file_hash = self.file_digest(upload_file)
|
||||
filename = file_hash + '.pdf'
|
||||
filename = file_hash[45:] + '.pdf'
|
||||
if 'content_type' in json_data['fields'][k]:
|
||||
content_type = json_data['fields'][k]['content_type']
|
||||
if k == 'cerfa' and content_type != 'application/pdf':
|
||||
|
|
@ -259,49 +454,40 @@ class AtrealOpenads(BaseResource):
|
|||
'upload_file' : upload_file
|
||||
})
|
||||
|
||||
# log files to be forwarded
|
||||
self.debug("----- files (begining) -----")
|
||||
self.debug(files)
|
||||
self.debug("----- files (end) -----")
|
||||
|
||||
# make a request to openADS.API (with the payload)
|
||||
url = urlparse.urljoin(self.openADS_API_url, '/dossiers/%s' % type_dossier)
|
||||
response = self.requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
auth=(self.openADS_API_login, self.openADS_API_password),
|
||||
timeout=self.openADS_API_timeout
|
||||
)
|
||||
|
||||
# response is an error code
|
||||
if response.status_code // 100 != 2:
|
||||
error = self.get_response_error(response)
|
||||
self.logger.warning("Request [POST] '%s' failed with error: '%s'", url, error)
|
||||
raise APIError(error)
|
||||
|
||||
# load the response JSON content
|
||||
try:
|
||||
result = response.json()
|
||||
except ValueError:
|
||||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||||
|
||||
self.debug("----- response json (begining) -----")
|
||||
debug_json = copy.deepcopy(result)
|
||||
if 'files' in debug_json \
|
||||
and debug_json['files'] \
|
||||
and isinstance(debug_json['files'], list) \
|
||||
and len(debug_json['files']) > 0 \
|
||||
and isinstance(debug_json['files'][0], dict) \
|
||||
and 'b64_content' in debug_json['files'][0]:
|
||||
debug_json['files'][0]['b64_content'] = '<b64 content>'
|
||||
self.debug(json.dumps(debug_json))
|
||||
self.debug("----- response json (end) -----")
|
||||
# get the recepisse
|
||||
recepisse = self.get_first_file_from_json_payload(result, title='response')
|
||||
|
||||
numero_dossier = result.get('numero_dossier')
|
||||
self.debug("Numéro dossier: %s", str(numero_dossier))
|
||||
|
||||
recepisse = result['files'][0]
|
||||
try:
|
||||
recepisse_content = base64.b64decode(recepisse['b64_content'])
|
||||
except TypeError:
|
||||
raise APIError('Invalid content for recepisse')
|
||||
self.debug("Successfully decoded recepisse from base64")
|
||||
|
||||
if recepisse['content_type'] and recepisse['content_type'] != 'application/pdf':
|
||||
# ensure recepisse content type is PDF
|
||||
if (
|
||||
'content_type' in recepisse
|
||||
and recepisse['content_type']
|
||||
and recepisse['content_type'] != 'application/pdf'
|
||||
):
|
||||
self.debug(
|
||||
"Forcing 'recepisse' content type to '%s' instead of '%s'.",
|
||||
'application/pdf',
|
||||
|
|
@ -309,6 +495,27 @@ class AtrealOpenads(BaseResource):
|
|||
)
|
||||
recepisse['content_type'] = 'application/pdf'
|
||||
|
||||
# decode the recepisse from base 64
|
||||
try:
|
||||
recepisse_content = base64.b64decode(recepisse['b64_content'])
|
||||
except TypeError:
|
||||
raise APIError('Failed to decode recepisse content from base 64')
|
||||
self.debug("Successfully decoded recepisse from base 64")
|
||||
|
||||
# check/get the 'numero_dossier'
|
||||
if 'numero_dossier' not in result:
|
||||
raise APIError("Expecting 'numero_dossier' key in JSON response")
|
||||
|
||||
numero_dossier = result.get('numero_dossier')
|
||||
|
||||
if not isinstance(numero_dossier, str) and not isinstance(numero_dossier, unicode):
|
||||
raise APIError(
|
||||
"Expecting '%s' value in JSON response to be a %s (not a %s)" %
|
||||
('numero_dossier', 'string', str(type(numero_dossier))))
|
||||
|
||||
self.debug("Numéro dossier: %s", str(numero_dossier))
|
||||
|
||||
# save files to be forwarded to openADS.API
|
||||
if files:
|
||||
file_ids = []
|
||||
for f in files:
|
||||
|
|
@ -342,6 +549,7 @@ class AtrealOpenads(BaseResource):
|
|||
','.join([str(fid) for fid in file_ids])
|
||||
)
|
||||
|
||||
# respond with the 'numero_dossier' and the recepisse file
|
||||
return {
|
||||
'numero_dossier': numero_dossier,
|
||||
'recepisse' : recepisse
|
||||
|
|
@ -349,29 +557,44 @@ class AtrealOpenads(BaseResource):
|
|||
|
||||
|
||||
@endpoint(
|
||||
description="Get informations about an openADS 'dossier'",
|
||||
description=_("Get informations about an openADS 'dossier'"),
|
||||
pattern='^(?P<type_dossier>\w+)/?$',
|
||||
example_pattern='{type_dossier}/',
|
||||
parameters={
|
||||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||||
})
|
||||
},
|
||||
#~ get={
|
||||
#~ 'description': _("Get informations about an openADS 'dossier'"),
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_GET_DOSSIER_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
#~ }
|
||||
)
|
||||
def get_dossier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||||
|
||||
# make a request to openADS.API
|
||||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s' % (type_dossier, numero_dossier))
|
||||
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
|
||||
response = self.requests.get(url)
|
||||
|
||||
# response is an error
|
||||
if response.status_code // 100 != 2:
|
||||
error = self.get_response_error(response)
|
||||
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
|
||||
raise APIError(error)
|
||||
|
||||
# load the response as JSON
|
||||
try:
|
||||
result = response.json()
|
||||
except ValueError:
|
||||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||||
etat = result.get('etat')
|
||||
date_depot = result.get('date_depot')
|
||||
date_decision = result.get('date_decision')
|
||||
decision = result.get('decision')
|
||||
date_limite_instruction = result.get('date_limite_instruction')
|
||||
|
||||
# log the response
|
||||
self.log_json_payload(result, 'response')
|
||||
|
||||
# return the response as-is
|
||||
return response.json()
|
||||
|
||||
|
||||
|
|
@ -394,105 +617,166 @@ class AtrealOpenads(BaseResource):
|
|||
|
||||
|
||||
@endpoint(
|
||||
description="Get informations about the forwarding of a user file to openADS",
|
||||
methods=['get'],
|
||||
description=_("Get informations about the forwarding of user files to openADS"),
|
||||
parameters={
|
||||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'},
|
||||
'summary' : {'description': _("Summary (only)") , 'example_value': '1'}
|
||||
})
|
||||
def get_fwd_files_status(self, request, numero_dossier, fichier_id=None, summary=None, *args, **kwargs):
|
||||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||||
},
|
||||
#~ get={
|
||||
#~ 'description': _("Get informations about the forwarding of user files to openADS"),
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
#~ }
|
||||
)
|
||||
def get_fwd_files(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||||
payload = []
|
||||
fwd_files = []
|
||||
|
||||
# search for all files matching the 'numero_dossier' number
|
||||
if not fichier_id:
|
||||
try:
|
||||
fwd_files = ForwardFile.objects.filter(numero_dossier=numero_dossier)
|
||||
except ForwardFile.DoesNotExist:
|
||||
raise Http404("No file matches 'numero_dossier=%s'." % numero_dossier)
|
||||
fwd_files = ForwardFile.objects.filter(numero_dossier=numero_dossier)
|
||||
|
||||
# search for a single file
|
||||
elif fichier_id:
|
||||
try:
|
||||
fwd_file = ForwardFile.objects.get(id=fichier_id)
|
||||
fichier_id = int(fichier_id)
|
||||
except ValueError:
|
||||
raise APIError('fichier_id must be an integer')
|
||||
try:
|
||||
fwd_files = [ForwardFile.objects.get(id=fichier_id)]
|
||||
except ForwardFile.DoesNotExist:
|
||||
raise Http404("No file matches 'numero_dossier=%s' and 'id=%s'." % (numero_dossier, fichier_id))
|
||||
if fwd_file:
|
||||
fwd_files.append(fwd_file)
|
||||
|
||||
if fwd_files:
|
||||
|
||||
summary_enabled = summary and len(summary) and summary == '1'
|
||||
if summary_enabled:
|
||||
summary_data = {
|
||||
'all_forwarded': True,
|
||||
'pending' : [],
|
||||
'uploading' : [],
|
||||
'success' : [],
|
||||
'failed' : []
|
||||
}
|
||||
|
||||
for fwd_file in fwd_files:
|
||||
b64content = None
|
||||
if fwd_file.upload_file:
|
||||
b64content = base64.b64encode(fwd_file.upload_file.read())
|
||||
payload.append({
|
||||
'id' : fwd_file.id,
|
||||
'numero_demande': fwd_file.numero_demande,
|
||||
'numero_dossier': fwd_file.numero_dossier,
|
||||
'type_fichier' : fwd_file.type_fichier,
|
||||
'file_hash' : fwd_file.file_hash,
|
||||
'orig_filename' : fwd_file.orig_filename,
|
||||
'content_type' : fwd_file.content_type,
|
||||
'upload_status' : fwd_file.upload_status,
|
||||
'upload_msg' : fwd_file.upload_msg,
|
||||
'b64_content' : b64content,
|
||||
'last_update_datetime' : fwd_file.last_update_datetime
|
||||
})
|
||||
|
||||
if summary_enabled:
|
||||
status_msg = '[%s] %s => %s' % (fwd_file.id, fwd_file.orig_filename, fwd_file.upload_msg)
|
||||
summary_data[fwd_file.upload_status].append(status_msg)
|
||||
if fwd_file.upload_status != 'success':
|
||||
summary_data['all_forwarded'] = False
|
||||
|
||||
if summary_enabled:
|
||||
payload = summary_data
|
||||
# append each file to the response payload
|
||||
for fwd_file in fwd_files:
|
||||
payload.append({
|
||||
'id' : fwd_file.id,
|
||||
'numero_demande': fwd_file.numero_demande,
|
||||
'numero_dossier': fwd_file.numero_dossier,
|
||||
'type_fichier' : fwd_file.type_fichier,
|
||||
'file_hash' : fwd_file.file_hash,
|
||||
'orig_filename' : fwd_file.orig_filename,
|
||||
'content_type' : fwd_file.content_type,
|
||||
'upload_status' : fwd_file.upload_status,
|
||||
'upload_attempt': fwd_file.upload_attempt,
|
||||
'upload_msg' : fwd_file.upload_msg,
|
||||
'content_size' : fwd_file.upload_file.size if fwd_file.upload_file else 0,
|
||||
'last_update_datetime' : fwd_file.last_update_datetime
|
||||
})
|
||||
|
||||
# return the payload containing the list of files
|
||||
return payload
|
||||
|
||||
|
||||
@endpoint(
|
||||
description="Get a 'courrier' from an openADS 'dossier'",
|
||||
description=_("Get informations about the forwarding of a user file to openADS"),
|
||||
parameters={
|
||||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'},
|
||||
'fichier_id' : {'description': _("File identifier") , 'example_value': '78'}
|
||||
},
|
||||
#~ get={
|
||||
#~ 'description': _("Get informations about the forwarding of a user file to openADS"),
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_GET_FWD_FILES_STATUS_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
#~ }
|
||||
)
|
||||
def get_fwd_files_status(self, request, numero_dossier, fichier_id=None, *args, **kwargs):
|
||||
|
||||
# get all files matching 'numero_dossier' and 'fichier_id'
|
||||
fwd_files = self.get_fwd_files(request, numero_dossier, fichier_id, *args, **kwargs)
|
||||
|
||||
# prepare the response payload
|
||||
payload = {
|
||||
'all_forwarded': True,
|
||||
'pending' : [],
|
||||
'uploading' : [],
|
||||
'success' : [],
|
||||
'failed' : []
|
||||
}
|
||||
|
||||
# build a summary of all files statuses
|
||||
for fwd_file in fwd_files:
|
||||
status_msg = '[%s] %s => %s' % (
|
||||
fwd_file['id'],
|
||||
fwd_file['orig_filename'],
|
||||
fwd_file['upload_msg']
|
||||
)
|
||||
payload[fwd_file['upload_status']].append(status_msg)
|
||||
if fwd_file['upload_status'] != 'success':
|
||||
payload['all_forwarded'] = False
|
||||
|
||||
# respond with the payload
|
||||
return payload
|
||||
|
||||
|
||||
@endpoint(
|
||||
description= _("Get a 'courrier' from an openADS 'dossier'"),
|
||||
pattern='^(?P<type_dossier>\w+)/?$',
|
||||
example_pattern='{type_dossier}/',
|
||||
parameters={
|
||||
'type_dossier' : {'description': _("Type of 'dossier'") , 'example_value': 'DIA'},
|
||||
'numero_dossier': {'description': _("Identifier for 'dossier'"), 'example_value': 'DIA0130551900001'}
|
||||
})
|
||||
},
|
||||
#~ get={
|
||||
#~ 'description': _("Get a 'courrier' from an openADS 'dossier'"),
|
||||
#~ 'response_body': {
|
||||
#~ 'schema': {
|
||||
#~ 'application/json': JSON_SCHEMA_GET_COURRIER_OUT
|
||||
#~ }
|
||||
#~ }
|
||||
#~ }
|
||||
)
|
||||
def get_courrier(self, request, type_dossier, numero_dossier, *args, **kwargs):
|
||||
|
||||
# make a request to openADS.API
|
||||
url = urlparse.urljoin(
|
||||
self.openADS_API_url,
|
||||
'/dossier/%s/%s/courrier/%s' % (type_dossier, numero_dossier, 'dia_renonciation_preempter'))
|
||||
response = self.requests.get(url, auth=(self.openADS_API_login, self.openADS_API_password))
|
||||
response = self.requests.get(url)
|
||||
|
||||
# response is an error
|
||||
if response.status_code // 100 != 2:
|
||||
error = self.get_response_error(response)
|
||||
self.logger.warning("Request [GET] '%s' failed with error: '%s'", url, error)
|
||||
raise APIError(error)
|
||||
|
||||
# load the response as JSON
|
||||
try:
|
||||
result = response.json()
|
||||
except ValueError:
|
||||
raise APIError('No JSON content returned: %r' % response.content[:1000])
|
||||
courrier = result['files'][0]
|
||||
|
||||
# log the response (filtering the file content)
|
||||
self.log_json_payload(
|
||||
result, 'response',
|
||||
['files.%i.b64_content' % i for i in range(0, len(result['files']))])
|
||||
|
||||
# get the courrier
|
||||
courrier = self.get_first_file_from_json_payload(result, title='response')
|
||||
|
||||
# decode the courrier from base 64
|
||||
try:
|
||||
courrier_content = base64.b64decode(courrier['b64_content'])
|
||||
except TypeError:
|
||||
raise APIError('Invalid content for courrier')
|
||||
|
||||
# return the 'courrier' file
|
||||
return {'courrier': courrier}
|
||||
|
||||
|
||||
def get_response_error(self, response):
|
||||
"""Return a error string from an HTTP response."""
|
||||
try:
|
||||
# load the response as JSON
|
||||
result = response.json()
|
||||
|
||||
# collect errors and turn them into messages (multispaces are filtered)
|
||||
errors = result.get('errors')
|
||||
msg = []
|
||||
if errors:
|
||||
|
|
@ -501,10 +785,18 @@ class AtrealOpenads(BaseResource):
|
|||
name = error.get('name')
|
||||
desc = error.get('description')
|
||||
msg.append('[%s] (%s) %s' % (location, normalize(name), normalize(desc)))
|
||||
|
||||
# if there are messages
|
||||
if msg:
|
||||
|
||||
# return a string representing the HTTP error
|
||||
return "HTTP error: %s, %s" % (response.status_code, ','.join(msg))
|
||||
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# TODO ask for openADS.API to *always* send JSON formatted errors, not HTML ones
|
||||
# return a string representing the HTTP error (filtering the HTML tags and multispaces)
|
||||
return "HTTP error: %s, %s" % \
|
||||
(response.status_code,
|
||||
clean_spaces(strip_tags(response.content[:1000])) if response.content else '')
|
||||
|
|
@ -512,14 +804,22 @@ class AtrealOpenads(BaseResource):
|
|||
|
||||
def upload_user_files(self, type_dossier, numero_dossier, file_ids):
|
||||
"""A Job to forward user uploaded files to openADS."""
|
||||
|
||||
payload = []
|
||||
fwd_files = []
|
||||
|
||||
# for every file ids specified (in parameters of this job)
|
||||
for fid in file_ids:
|
||||
self.debug("upload_user_files() ForwardFile file_id: %s", fid)
|
||||
|
||||
# get the matching forward file
|
||||
fwd_file = ForwardFile.objects.get(id=fid)
|
||||
|
||||
# found one
|
||||
if fwd_file:
|
||||
self.debug("upload_user_files() got ForwardFile")
|
||||
|
||||
# add the file content and data to the payload
|
||||
payload.append({
|
||||
'filename' : fwd_file.orig_filename + ('.pdf' if fwd_file.orig_filename[-4:] != '.pdf' else ''),
|
||||
'content_type' : fwd_file.content_type,
|
||||
|
|
@ -527,72 +827,105 @@ class AtrealOpenads(BaseResource):
|
|||
'file_type' : fwd_file.type_fichier
|
||||
})
|
||||
self.debug("upload_user_files() payload added")
|
||||
fwd_file.upload_status = 'uploading'
|
||||
if fwd_file.upload_msg and re.search(r'^attempt \d+$', fwd_file.upload_msg):
|
||||
self.debug("upload_user_files() upload_msg: '%s'", fwd_file.upload_msg)
|
||||
attempt_num = fwd_file.upload_msg.replace('attempt ', '').strip()
|
||||
self.debug("upload_user_files() attempt_num: '%s'", attempt_num)
|
||||
fwd_file.upload_msg = 'attempt %s' % attempt_num
|
||||
else:
|
||||
fwd_file.upload_msg = 'attempt 1'
|
||||
self.debug("upload_user_files() ForwardFile ready to be saved")
|
||||
fwd_file.save()
|
||||
fwd_files.append(fwd_file)
|
||||
else:
|
||||
self.logger.warning("upload_user_files() failed to find ForwardFile file_id: %s", fid);
|
||||
|
||||
# update the file upload data (status and attempts)
|
||||
fwd_file.upload_status = 'uploading'
|
||||
fwd_file.upload_attempt += 1
|
||||
fwd_file.upload_msg = 'attempt %s' % fwd_file.upload_attempt
|
||||
self.debug("upload_user_files() upload_msg: '%s'", fwd_file.upload_msg)
|
||||
fwd_file.save()
|
||||
self.debug("upload_user_files() ForwardFile saved")
|
||||
|
||||
# append the forwarded file to the list
|
||||
fwd_files.append(fwd_file)
|
||||
|
||||
# not found the forwarded file specified: bug
|
||||
else:
|
||||
self.logger.warning("upload_user_files() failed to find ForwardFile file_id: %s", fid)
|
||||
|
||||
# if files need to be forwarded
|
||||
if payload:
|
||||
|
||||
self.debug("upload_user_files() payload is not empty")
|
||||
debug_payload = copy.deepcopy(payload)
|
||||
for p in debug_payload:
|
||||
if 'b64_content' in p:
|
||||
p['b64_content'] = '<b64 content>'
|
||||
self.debug("upload_user_files() payload is: %s", str(debug_payload))
|
||||
|
||||
# log the payload
|
||||
self.log_json_payload(
|
||||
payload, 'payload',
|
||||
['%i.b64_content' % i for i in range(0, len(fwd_files))])
|
||||
|
||||
# make the request to openADS.API (with a specific timeout)
|
||||
url = urlparse.urljoin(self.openADS_API_url, '/dossier/%s/%s/files' % (type_dossier, numero_dossier))
|
||||
response = self.requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
auth=(self.openADS_API_login, self.openADS_API_password),
|
||||
timeout=self.openADS_API_timeout
|
||||
)
|
||||
|
||||
# reponse is an error
|
||||
if response.status_code // 100 != 2:
|
||||
|
||||
# update every files status as 'failed' and save the error message
|
||||
for fwd_file in fwd_files:
|
||||
fwd_file.upload_status = 'failed'
|
||||
fwd_file.upload_msg = self.get_response_error(response)
|
||||
fwd_file.save()
|
||||
|
||||
# log (warning) the error message
|
||||
self.logger.warning(
|
||||
"upload_user_files() openADS response is not OK (code: %s) for dossier '%s' and files '%s'",
|
||||
response.status_code,
|
||||
numero_dossier,
|
||||
','.join(file_ids)
|
||||
)
|
||||
|
||||
# response is not an error
|
||||
else:
|
||||
|
||||
# load the reponse as JSON
|
||||
try:
|
||||
result = response.json()
|
||||
|
||||
# in case of failure
|
||||
except ValueError:
|
||||
|
||||
# update every files status as 'failed' and save the error message
|
||||
for fwd_file in fwd_files:
|
||||
fwd_file.upload_status = 'failed'
|
||||
fwd_file.upload_msg = 'No JSON content returned: %r' % response.content[:1000]
|
||||
fwd_file.save()
|
||||
|
||||
# log (warning) the error message
|
||||
self.logger.warning(
|
||||
"upload_user_files() openADS response is not JSON valid for dossier '%s' and files '%s'",
|
||||
numero_dossier,
|
||||
','.join([f.id for f in fwd_files])
|
||||
)
|
||||
|
||||
# response correctly loaded as JSON
|
||||
else:
|
||||
|
||||
# TODO handle response (now its just an informational sentence in key 'data')
|
||||
|
||||
# update every files status as 'success' and save the success message
|
||||
for fwd_file in fwd_files:
|
||||
fwd_file.upload_status = 'success'
|
||||
fwd_file.upload_msg = 'uploaded successfuly'
|
||||
|
||||
# delete file content (on success)
|
||||
fpath = fwd_file.upload_file.path
|
||||
fwd_file.upload_file.delete()
|
||||
|
||||
# save the file
|
||||
fwd_file.save()
|
||||
|
||||
# log the success message
|
||||
self.debug(
|
||||
"upload_user_files() flaging file '%s' has transfered (deleted '%s')",
|
||||
fwd_file.id,
|
||||
fpath
|
||||
)
|
||||
|
||||
# no file need to be forwarded
|
||||
else:
|
||||
self.logger.warning(
|
||||
"upload_user_files() payload is empty for dossier '%s' and files '%s'",
|
||||
|
|
|
|||
|
|
@ -54,9 +54,9 @@ def atreal_openads(db):
|
|||
return AtrealOpenads.objects.create(
|
||||
slug = CONNECTOR_SLUG,
|
||||
collectivite = COLLECTIVITE,
|
||||
openADS_API_login = OPENADS_API_LOGIN,
|
||||
openADS_API_password = OPENADS_API_PASSWORD,
|
||||
openADS_API_url = OPENADS_API_URL
|
||||
openADS_API_url = OPENADS_API_URL,
|
||||
basic_auth_username = OPENADS_API_LOGIN,
|
||||
basic_auth_password = OPENADS_API_PASSWORD
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -67,7 +67,7 @@ def test_openads_check_status(app, atreal_openads):
|
|||
fake_resp = JsonResponse(fake_resp_json)
|
||||
with mock.patch('passerelle.utils.Request.get') as requests_get:
|
||||
requests_get.return_value = mock.Mock(content=fake_resp, status_code=200)
|
||||
jresp = atreal_openads.check_status()
|
||||
jresp = atreal_openads.check_status(None)
|
||||
assert jresp['response'] == 200
|
||||
|
||||
|
||||
|
|
@ -261,12 +261,12 @@ def test_openads_upload2ForwardFile(app, atreal_openads):
|
|||
assert FF.upload_status == 'pending'
|
||||
|
||||
|
||||
def test_openads_get_fwd_files_status(app, atreal_openads):
|
||||
def test_openads_get_fwd_files(app, atreal_openads):
|
||||
with pytest.raises(Http404) as e:
|
||||
resp404 = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=18, summary=None)
|
||||
atreal_openads.get_fwd_files(None, FAKE_NUMERO_DOSSIER, fichier_id=18)
|
||||
assert re.search(r"^No file matches 'numero_dossier=[^']+' and 'id=[^']+'.$", str(e.value))
|
||||
|
||||
resp404 = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=None, summary=None)
|
||||
resp404 = atreal_openads.get_fwd_files(None, FAKE_NUMERO_DOSSIER, fichier_id=None)
|
||||
assert resp404 is not None
|
||||
assert len(resp404) == 0
|
||||
|
||||
|
|
@ -274,25 +274,35 @@ def test_openads_get_fwd_files_status(app, atreal_openads):
|
|||
FF.save()
|
||||
assert isinstance(FF, ForwardFile)
|
||||
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=None, summary=None)
|
||||
jresp = atreal_openads.get_fwd_files(None, FAKE_NUMERO_DOSSIER, fichier_id=None)
|
||||
assert jresp is not None
|
||||
assert len(jresp) == 1
|
||||
assert jresp[0]['id'] == FF.id
|
||||
for k in ['numero_dossier', 'type_fichier', 'file_hash', 'orig_filename', 'content_type', 'upload_status', 'upload_msg']:
|
||||
assert jresp[0][k] == getattr(FF, k)
|
||||
assert jresp[0]['b64_content'] == get_file_data(FF.upload_file.path)
|
||||
assert jresp[0]['content_size'] == len(get_file_data(FF.upload_file.path, b64=False))
|
||||
assert jresp[0]['last_update_datetime'] == FF.last_update_datetime
|
||||
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=FF.id, summary=None)
|
||||
jresp = atreal_openads.get_fwd_files(None, FAKE_NUMERO_DOSSIER, fichier_id=FF.id)
|
||||
assert jresp is not None
|
||||
assert len(jresp) == 1
|
||||
assert jresp[0]['id'] == FF.id
|
||||
for k in ['numero_dossier', 'type_fichier', 'file_hash', 'orig_filename', 'content_type', 'upload_status', 'upload_msg']:
|
||||
assert jresp[0][k] == getattr(FF, k)
|
||||
assert jresp[0]['b64_content'] == get_file_data(FF.upload_file.path)
|
||||
assert jresp[0]['content_size'] == len(get_file_data(FF.upload_file.path, b64=False))
|
||||
assert jresp[0]['last_update_datetime'] == FF.last_update_datetime
|
||||
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=None, summary='1')
|
||||
|
||||
def test_openads_get_fwd_files_status(app, atreal_openads):
|
||||
with pytest.raises(Http404) as e:
|
||||
atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=18)
|
||||
assert re.search(r"^No file matches 'numero_dossier=[^']+' and 'id=[^']+'.$", str(e.value))
|
||||
|
||||
FF = atreal_openads.upload2ForwardFile(TEST_FILE_CERFA_DIA, FAKE_NUMERO_DOSSIER)
|
||||
FF.save()
|
||||
assert isinstance(FF, ForwardFile)
|
||||
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=None)
|
||||
assert jresp is not None
|
||||
assert jresp['all_forwarded'] == False
|
||||
status_msg = '[%s] %s => %s' % (FF.id, FF.orig_filename, FF.upload_msg)
|
||||
|
|
@ -302,7 +312,7 @@ def test_openads_get_fwd_files_status(app, atreal_openads):
|
|||
assert len(jresp['success']) == 0
|
||||
assert len(jresp['failed']) == 0
|
||||
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=FF.id, summary='1')
|
||||
jresp = atreal_openads.get_fwd_files_status(None, FAKE_NUMERO_DOSSIER, fichier_id=FF.id)
|
||||
assert jresp is not None
|
||||
assert jresp['all_forwarded'] == False
|
||||
status_msg = '[%s] %s => %s' % (FF.id, FF.orig_filename, FF.upload_msg)
|
||||
|
|
|
|||
Reference in New Issue