entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 15 Releases Activity

toulouse_smart: add an add-media endpoint (#57875)

This commit is contained in:
Nicolas Roche 2021-10-31 17:31:08 +01:00
parent 8736651b4c
commit 8afd77f697
6 changed files with 346 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
passerelle/contrib/toulouse_smart/migrations/0004_wcsrequestfile.py Normal file
View File

@ -0,0 +1,36 @@
# Generated by Django 2.2.19 on 2021-10-29 15:45
import django.db.models.deletion
from django.db import migrations, models
import passerelle.contrib.toulouse_smart.models
class Migration(migrations.Migration):
dependencies = [
('toulouse_smart', '0003_smartrequest'),
]
operations = [
migrations.CreateModel(
name='WcsRequestFile',
fields=[
(
'id',
models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
('filename', models.CharField(max_length=256)),
('content_type', models.CharField(max_length=256)),
('content', models.FileField(upload_to=passerelle.contrib.toulouse_smart.models.upload_to)),
(
'resource',
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name='files',
to='toulouse_smart.WcsRequest',
),
),
],
),
]

91
passerelle/contrib/toulouse_smart/models.py
View File

@ -14,6 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import datetime
import json
from uuid import uuid4
@ -21,6 +22,7 @@ from uuid import uuid4
import lxml.etree as ET
from django.conf import settings
from django.contrib.postgres.fields import JSONField
from django.core.files.base import ContentFile
from django.db import models
from django.db.transaction import atomic
from django.urls import reverse
@ -190,12 +192,14 @@ class ToulouseSmartResource(BaseResource, HTTPResource):
wcs_form_api_url=post_data['form_api_url'],
wcs_form_number=post_data['external_number'],
)
update_intervention_endpoint_url = request.build_absolute_uri(
reverse(
'generic-endpoint',
kwargs={'connector': 'toulouse-smart', 'endpoint': 'update-intervention', 'slug': self.slug},
endpoint_url = {}
for endpoint_name in 'update-intervention', 'add-media':
endpoint_url[endpoint_name] = request.build_absolute_uri(
reverse(
'generic-endpoint',
kwargs={'connector': 'toulouse-smart', 'endpoint': endpoint_name, 'slug': self.slug},
)
)
)
wcs_request.payload = {
'description': post_data['description'],
'cityId': post_data['cityId'],
@ -217,7 +221,8 @@ class ToulouseSmartResource(BaseResource, HTTPResource):
'crs': 'EPSG:4326',
},
'interventionTypeId': intervention_type['id'],
'notificationUrl': '%s?uuid=%s' % (update_intervention_endpoint_url, wcs_request.uuid),
'notificationUrl': '%s?uuid=%s' % (endpoint_url['update-intervention'], wcs_request.uuid),
'add_media_url': '%s?uuid=%s' % (endpoint_url['add-media'], wcs_request.uuid),
}
for label in 'checkDuplicated', 'onPrivateLand', 'safeguardRequired':
if str(post_data.get(label)).lower() in ['true', 'oui', '1']:
@ -261,6 +266,50 @@ class ToulouseSmartResource(BaseResource, HTTPResource):
natural_id='smart-request-%s' % smart_request.id,
)
@endpoint(
name='add-media',
methods=['post'],
description=_('Add a media'),
parameters={
'uuid': {'description': _('Notification identifier')},
},
perm='can_access',
post={'request_body': {'schema': {'application/json': schemas.MEDIA_SCHEMA}}},
)
def add_media(self, request, uuid, post_data):
try:
wcs_request = self.wcs_requests.get(uuid=uuid)
except WcsRequest.DoesNotExist:
raise APIError("Cannot find intervention '%s'" % uuid, http_status=400)
nb_registered = 0
for media in post_data['files']:
if not media:
# silently ignore empty payload value
continue
wcs_request_file = wcs_request.files.create(
filename=media['filename'], content_type=media['content_type']
)
with ContentFile(base64.b64decode(media['content'])) as media_content:
wcs_request_file.content.save(media['filename'], media_content)
self.add_job(
'add_media_job',
id=wcs_request_file.id,
natural_id='wcs-request-file-%s' % wcs_request_file.id,
)
nb_registered += 1
return {'data': {'uuid': wcs_request.uuid, 'nb_registered': nb_registered}}
def add_media_job(self, *args, **kwargs):
wcs_request_file = WcsRequestFile.objects.get(id=kwargs['id'])
wcs_request = wcs_request_file.resource
if not wcs_request.result or not wcs_request.result.get('id'):
raise SkipJob(datetime.timedelta(minutes=10))
if not wcs_request_file.push():
raise SkipJob()
@atomic
@endpoint(
name='update-intervention',
@ -352,6 +401,36 @@ class WcsRequest(models.Model):
return True
def upload_to(wcs_request_file, filename):
instance = wcs_request_file.resource.resource
uuid = wcs_request_file.resource.uuid
return '%s/%s/%s/%s' % (instance.get_connector_slug(), instance.slug, uuid, filename)
class WcsRequestFile(models.Model):
resource = models.ForeignKey(
to=WcsRequest,
on_delete=models.CASCADE,
related_name='files',
)
filename = models.CharField(max_length=256)
content_type = models.CharField(max_length=256)
content = models.FileField(upload_to=upload_to)
def push(self):
wcs_request = self.resource
intervention_id = wcs_request.result.get('id')
instance = wcs_request.resource
url = '%sv1/intervention/%s/media' % (instance.webservice_base_url, intervention_id)
files = {'media': (self.filename, self.content.open('rb'), self.content_type)}
try:
instance.request(url, method='PUT', files=files)
except APIError as e:
return False
self.content.delete()
return True
class SmartRequest(models.Model):
resource = models.ForeignKey(
to=WcsRequest,

38
passerelle/contrib/toulouse_smart/schemas.py
View File

@ -177,3 +177,41 @@ UPDATE_SCHEMA = {
},
'required': ['data'],
}
MEDIA_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'object',
'properties': {
'files': {
'type': 'array',
'items': {
'oneOf': [
{
'type': 'null',
},
{
'type': 'object',
'properties': {
'filename': {
'description': "Nom du ficher",
'type': 'string',
},
'content_type': {
'description': "Type MIME",
'type': 'string',
},
'content': {
'description': "Contenu",
'type': 'string',
},
},
'required': ['filename', 'content_type', 'content'],
},
],
},
},
},
'required': ['files'],
'unflatten': True,
}

2
passerelle/views.py
View File

@ -386,7 +386,6 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
d[parameter] = float(d[parameter])
except ValueError:
raise InvalidParameterValue(parameter)
if request.method == 'POST' and self.endpoint.endpoint_info.post:
request_body = self.endpoint.endpoint_info.post.get('request_body', {})
if 'application/json' in request_body.get('schema', {}):
@ -409,7 +408,6 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
validator = validators.validator_for(json_schema)
validator.META_SCHEMA['properties'].pop('description', None)
validator.META_SCHEMA['properties'].pop('title', None)
try:
validate(data, json_schema)
except ValidationError as e:

BIN
tests/data/toulouse_smart/201x201.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 795 B

190
tests/test_toulouse_smart.py
View File

@ -14,6 +14,8 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import cgi
import functools
import io
import json
@ -27,11 +29,17 @@ import lxml.etree as ET
import mock
import pytest
import utils
from django.utils.encoding import force_text
from requests.exceptions import ReadTimeout
from test_manager import login
from passerelle.base.models import Job
from passerelle.contrib.toulouse_smart.models import SmartRequest, ToulouseSmartResource, WcsRequest
from passerelle.contrib.toulouse_smart.models import (
SmartRequest,
ToulouseSmartResource,
WcsRequest,
WcsRequestFile,
)
TEST_BASE_DIR = os.path.join(os.path.dirname(__file__), 'data', 'toulouse_smart')
@ -72,8 +80,21 @@ def mock_response(*path_contents):
def register(path, payload, content, status_code=200, exception=None):
@httmock.urlmatch(path=path)
def handler(url, request):
if payload and json.loads(request.body) != payload:
assert False, 'wrong payload sent to request to %s' % url.geturl()
if payload:
ctype, pdict = cgi.parse_header(request.headers["content-type"])
if ctype == 'multipart/form-data':
# here payload is an expected multipart contents list
pdict["boundary"] = bytes(pdict["boundary"], "utf-8")
pdict['CONTENT-LENGTH'] = request.headers['Content-Length']
postvars = cgi.parse_multipart(io.BytesIO(request.body), pdict)
for i, media_content in enumerate(postvars['media']):
assert media_content == payload[i], (
'wrong multipart content sent to %s' % url.geturl()
)
else:
assert json.loads(request.body) == payload, (
'wrong payload sent to request to %s' % url.geturl()
)
if exception:
raise exception
return httmock.response(status_code, content)
@ -100,6 +121,11 @@ def get_json_file(filename):
return desc.read()
def get_media_file(filename):
with open(os.path.join(TEST_BASE_DIR, "%s" % filename), 'rb') as desc:
return desc.read()
@mock_response(['/v1/type-intervention', None, b'<List></List>'])
def test_empty_intervention_types(smart):
assert smart.get_intervention_types() == []
@ -307,6 +333,7 @@ CREATE_INTERVENTION_PAYLOAD = {
UUID = uuid.UUID('12345678123456781234567812345678')
CREATE_INTERVENTION_QUERY = {
'add_media_url': 'http://testserver/toulouse-smart/test/add-media?uuid=%s' % str(UUID),
'description': 'coin coin',
'cityId': '12345',
'interventionCreated': '2021-06-30T16:08:05Z',
@ -657,6 +684,121 @@ def test_update_intervention_job_transport_error(mocked_uuid, app, freezer, smar
assert smart_request.result == None
ADD_MEDIA_PAYLOAD = {
'files/0': {
'filename': '201x201.jpg',
'content_type': 'image/jpeg',
'content': force_text(base64.b64encode(get_media_file('201x201.jpg'))),
},
'files/1': None,
}
ADD_MEDIA_QUERY = [get_media_file('201x201.jpg')]
@mock_response(
['/v1/type-intervention', None, INTERVENTION_TYPES],
['/v1/intervention/%s/media' % INTERVENTION_ID, ADD_MEDIA_QUERY, 200],
['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
)
@mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
def test_add_media(mocked_uuid, app, smart):
resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
assert not resp.json['err']
url = resp.json['data']['payload']['add_media_url']
url = URL + 'add-media?uuid=%s' % str(UUID)
resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
assert not resp.json['err']
assert resp.json['data']['uuid'] == str(UUID)
assert resp.json['data']['nb_registered'] == 1
assert Job.objects.count() == 1
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
wcs_request = smart.wcs_requests.get(uuid=UUID)
wcs_request_file = wcs_request.files.get(**job.parameters)
path = wcs_request_file.content.path
assert os.path.isfile(path)
with wcs_request_file.content.open('rb') as desc:
assert desc.read() == get_media_file('201x201.jpg')
# smart not responding
mocked_push = mock.patch(
"passerelle.contrib.toulouse_smart.models.WcsRequestFile.push",
return_value=False,
)
mocked_push.start()
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
# smart responding
mocked_push.stop()
smart.jobs()
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'completed'
wcs_request_file = wcs_request.files.get(**job.parameters)
with pytest.raises(ValueError, match='no file associated'):
assert not wcs_request_file.content.path
assert not os.path.isfile(path)
def test_add_media_wrong_uuid(app, smart):
with pytest.raises(WcsRequest.DoesNotExist):
smart.wcs_requests.get(uuid=UUID)
url = URL + 'add-media?uuid=%s' % str(UUID)
resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD, status=400)
assert resp.json['err']
assert 'Cannot find intervention' in resp.json['err_desc']
assert WcsRequestFile.objects.count() == 0
@mock_response(
['/v1/type-intervention', None, INTERVENTION_TYPES],
['/v1/intervention/%s/media' % json.loads(get_json_file('create_intervention'))['id'], None, None, 500],
['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
)
@mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
def test_add_media_error(mocked_uuid, app, freezer, smart):
resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
assert not resp.json['err']
freezer.move_to('2021-10-30 00:00:00')
url = resp.json['data']['payload']['add_media_url']
resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
freezer.move_to('2021-10-30 00:00:03')
smart.jobs()
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
assert job.update_timestamp > job.creation_timestamp
@mock_response(
['/v1/type-intervention', None, INTERVENTION_TYPES],
['/v1/intervention/%s/media' % INTERVENTION_ID, None, None, None, ReadTimeout('timeout')],
['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
)
@mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
def test_add_media_timeout_error(mocked_uuid, app, freezer, smart):
resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
assert not resp.json['err']
freezer.move_to('2021-10-30 00:00:00')
url = resp.json['data']['payload']['add_media_url']
resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
freezer.move_to('2021-10-30 00:00:03')
smart.jobs()
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
assert job.update_timestamp > job.creation_timestamp
UPDATE_INTERVENTION_QUERY_ON_ASYNC_CREATION = {
'creation_response': {
'wcs_form_api_url': CREATE_INTERVENTION_PAYLOAD_EXTRA['form_api_url'],
@ -732,3 +874,45 @@ def test_create_intervention_async(mocked_uuid4, app, smart, wcs_service):
smart_request = wcs_request.smart_requests.get()
assert smart_request.payload['creation_response']['uuid'] == str(UUID)
assert smart_request.payload['creation_response']['result']['id'] == INTERVENTION_ID
@mock_response(
['/v1/type-intervention', None, INTERVENTION_TYPES],
['/v1/intervention/%s/media' % INTERVENTION_ID, ADD_MEDIA_QUERY, 200],
['/v1/intervention', CREATE_INTERVENTION_QUERY, get_json_file('create_intervention')],
)
@mock.patch("django.db.models.fields.UUIDField.get_default", return_value=UUID)
def test_add_media_async(mocked_uuid4, app, smart, freezer):
mocked_wcs_request_push = mock.patch(
"passerelle.contrib.toulouse_smart.models.WcsRequest.push",
return_value=False,
)
# smart is down
freezer.move_to('2021-10-30 00:00:00')
mocked_wcs_request_push.start()
resp = app.post_json(URL + 'create-intervention/', params=CREATE_INTERVENTION_PAYLOAD)
assert not resp.json['err']
url = resp.json['data']['payload']['add_media_url']
resp = app.post_json(url, params=ADD_MEDIA_PAYLOAD)
smart.jobs()
job = Job.objects.get(method_name='create_intervention_job')
assert job.status == 'registered'
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
assert str(job.after_timestamp) == '2021-10-30 00:10:00+00:00'
# smart is up
freezer.move_to('2021-10-30 00:00:03')
mocked_wcs_request_push.stop()
smart.jobs()
job = Job.objects.get(method_name='create_intervention_job')
assert job.status == 'completed'
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'registered'
# 10 minutes later
freezer.move_to('2021-10-30 00:10:03')
smart.jobs()
job = Job.objects.get(method_name='add_media_job')
assert job.status == 'completed'