519 lines
19 KiB
Python
519 lines
19 KiB
Python
# passerelle - uniform access to multiple data sources and services
|
|
# Copyright (C) 2021 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import base64
|
|
import datetime
|
|
import json
|
|
from urllib import parse as urlparse
|
|
from uuid import uuid4
|
|
|
|
import lxml.etree as ET
|
|
from django.conf import settings
|
|
from django.core.exceptions import ValidationError
|
|
from django.core.files.base import ContentFile
|
|
from django.db import models
|
|
from django.db.models import JSONField
|
|
from django.db.transaction import atomic
|
|
from django.urls import reverse
|
|
from django.utils.text import slugify
|
|
from django.utils.timezone import now
|
|
from django.utils.translation import gettext_lazy as _
|
|
from requests import RequestException
|
|
|
|
from passerelle.base.models import BaseResource, HTTPResource, SkipJob
|
|
from passerelle.utils import xml
|
|
from passerelle.utils.api import endpoint
|
|
from passerelle.utils.jsonresponse import APIError
|
|
from passerelle.utils.wcs import WcsApi, WcsApiError
|
|
|
|
from . import schemas, utils
|
|
|
|
|
|
class ToulouseSmartResource(BaseResource, HTTPResource):
|
|
category = _('Business Process Connectors')
|
|
|
|
webservice_base_url = models.URLField(_('Webservice Base URL'))
|
|
|
|
log_requests_errors = False
|
|
|
|
class Meta:
|
|
verbose_name = _('Toulouse Smart')
|
|
|
|
def get_intervention_types(self):
|
|
try:
|
|
return self.get('intervention-types', max_age=120)
|
|
except KeyError:
|
|
pass
|
|
|
|
try:
|
|
url = self.webservice_base_url + 'v1/type-intervention'
|
|
response = self.requests.get(url, timeout=10)
|
|
doc = ET.fromstring(response.content)
|
|
intervention_types = []
|
|
for xml_item in doc:
|
|
item = xml.to_json(xml_item)
|
|
properties = item.pop('properties', [])
|
|
for prop in properties:
|
|
prop['required'] = prop.get('required') == 'true'
|
|
if prop.get('restrictedValues'):
|
|
prop['type'] = 'item'
|
|
if prop.get('type') in ('string', 'int', 'boolean', 'item'):
|
|
if 'properties' not in item:
|
|
item['properties'] = []
|
|
item['properties'].append(prop)
|
|
intervention_types.append(item)
|
|
intervention_types.sort(key=lambda x: x['name'])
|
|
for i, intervention_type in enumerate(intervention_types):
|
|
intervention_type['order'] = i + 1
|
|
except Exception:
|
|
return self.get('intervention-types')
|
|
self.set('intervention-types', intervention_types)
|
|
return intervention_types
|
|
|
|
def get(self, key, max_age=None):
|
|
cache_entries = self.cache_entries
|
|
if max_age:
|
|
cache_entries = cache_entries.filter(timestamp__gt=now() - datetime.timedelta(seconds=max_age))
|
|
try:
|
|
return cache_entries.get(key=key).value
|
|
except Cache.DoesNotExist:
|
|
raise KeyError(key)
|
|
|
|
def set(self, key, value):
|
|
self.cache_entries.update_or_create(key=key, defaults={'value': value})
|
|
|
|
@endpoint(
|
|
name='type-intervention',
|
|
description=_('Get intervention types'),
|
|
perm='can_access',
|
|
)
|
|
def type_intervention(self, request):
|
|
try:
|
|
return {
|
|
'data': [
|
|
{
|
|
'id': slugify(intervention_type['name']),
|
|
'text': intervention_type['name'],
|
|
'uuid': intervention_type['id'],
|
|
}
|
|
for intervention_type in self.get_intervention_types()
|
|
]
|
|
}
|
|
except Exception:
|
|
return {
|
|
'data': [
|
|
{
|
|
'id': '',
|
|
'text': _('Service is unavailable'),
|
|
'disabled': True,
|
|
}
|
|
]
|
|
}
|
|
|
|
def request(self, url, method='GET', json=None, files=None, timeout=None):
|
|
headers = {'Accept': 'application/json'}
|
|
if method == 'POST':
|
|
headers['Content-Type'] = 'application/json'
|
|
kwargs = {
|
|
'headers': headers,
|
|
'json': json,
|
|
'files': files,
|
|
}
|
|
if timeout:
|
|
kwargs['timeout'] = timeout
|
|
try:
|
|
response = self.requests.request(method=method, url=url, **kwargs)
|
|
response.raise_for_status()
|
|
except RequestException as e:
|
|
raise APIError('failed to %s %s: %s' % (method.lower(), url, e))
|
|
return response
|
|
|
|
@endpoint(
|
|
name='get-intervention',
|
|
methods=['get'],
|
|
description=_('Retrieve an intervention'),
|
|
perm='can_access',
|
|
parameters={
|
|
'id': {'description': _('Intervention identifier')},
|
|
},
|
|
)
|
|
def get_intervention(self, request, id):
|
|
url = self.webservice_base_url + 'v1/intervention/%s' % id
|
|
response = self.request(url)
|
|
data = response.json()
|
|
for label in 'interventionCreated', 'interventionDesired':
|
|
data[label] = utils.utc_to_localtz(data[label])
|
|
return {'data': data}
|
|
|
|
@atomic
|
|
@endpoint(
|
|
name='create-intervention',
|
|
methods=['post'],
|
|
description=_('Create an intervention'),
|
|
perm='can_access',
|
|
post={'request_body': {'schema': {'application/json': schemas.CREATE_SCHEMA}}},
|
|
)
|
|
def create_intervention(self, request, post_data):
|
|
slug = post_data['slug']
|
|
try:
|
|
types = [x for x in self.get_intervention_types() if slugify(x['name']) == slug]
|
|
except KeyError:
|
|
raise APIError('Service is unavailable')
|
|
if len(types) == 0:
|
|
raise APIError("unknown '%s' block slug" % slug, http_status=400)
|
|
intervention_type = types[0]
|
|
wcs_block_varname = slugify(intervention_type['name']).replace('-', '_')
|
|
try:
|
|
block = post_data['fields']['%s_raw' % wcs_block_varname][0]
|
|
except (KeyError, TypeError):
|
|
block = {}
|
|
data = {}
|
|
cast = {'string': str, 'int': int, 'boolean': bool, 'item': str}
|
|
for prop in intervention_type.get('properties') or []:
|
|
varname = slugify(prop['name']).replace('-', '_')
|
|
if block.get(varname):
|
|
try:
|
|
data[prop['name']] = cast[prop['type']](block[varname])
|
|
except ValueError:
|
|
raise APIError(
|
|
"cannot cast '%s' field to %s : '%s'" % (varname, cast[prop['type']], block[varname]),
|
|
http_status=400,
|
|
)
|
|
elif prop['required']:
|
|
raise APIError("'%s' field is required on '%s' block" % (varname, slug), http_status=400)
|
|
|
|
wcs_request, created = self.wcs_requests.get_or_create(
|
|
wcs_form_api_url=post_data['form_api_url'],
|
|
wcs_form_step=post_data.get('form_step', 'initial'),
|
|
defaults={'wcs_form_number': post_data['external_number']},
|
|
)
|
|
wcs_request = self.wcs_requests.select_for_update().get(pk=wcs_request.pk)
|
|
if not created:
|
|
if wcs_request.status != 'failed':
|
|
return {'data': wcs_request.reply}
|
|
wcs_request.tries = 0
|
|
wcs_request.status = 'registered'
|
|
wcs_request.result = None
|
|
|
|
endpoint_url = {}
|
|
for endpoint_name in 'update-intervention', 'add-media':
|
|
endpoint_url[endpoint_name] = request.build_absolute_uri(
|
|
reverse(
|
|
'generic-endpoint',
|
|
kwargs={'connector': 'toulouse-smart', 'endpoint': endpoint_name, 'slug': self.slug},
|
|
)
|
|
)
|
|
wcs_request.payload = {
|
|
'description': post_data['description'],
|
|
'cityId': post_data['cityId'],
|
|
'externalReferences': post_data['externalReferences'],
|
|
'submitterFirstName': post_data['submitterFirstName'],
|
|
'submitterLastName': post_data['submitterLastName'],
|
|
'submitterMail': post_data['submitterMail'],
|
|
'submitterPhone': post_data['submitterPhone'],
|
|
'submitterAddress': post_data['submitterAddress'],
|
|
'submitterType': post_data['submitterType'],
|
|
'external_number': post_data['external_number'],
|
|
'external_status': post_data['external_status'],
|
|
'address': post_data['address'],
|
|
'interventionData': data,
|
|
'geom': {
|
|
'type': 'Point',
|
|
'coordinates': [post_data['lon'], post_data['lat']],
|
|
'crs': 'EPSG:4326',
|
|
},
|
|
'interventionTypeId': intervention_type['id'],
|
|
'notificationUrl': '%s?uuid=%s' % (endpoint_url['update-intervention'], wcs_request.uuid),
|
|
'notification_url': '%s?uuid=%s' % (endpoint_url['update-intervention'], wcs_request.uuid),
|
|
'add_media_url': '%s?uuid=%s' % (endpoint_url['add-media'], wcs_request.uuid),
|
|
}
|
|
for label in 'checkDuplicated', 'onPrivateLand', 'safeguardRequired':
|
|
if str(post_data.get(label)).lower() in ['true', 'oui', '1']:
|
|
wcs_request.payload[label] = 'true'
|
|
for label in 'interventionCreated', 'interventionDesired':
|
|
wcs_request.payload[label] = utils.localtz_to_utc(post_data[label])
|
|
wcs_request.save()
|
|
if not wcs_request.push(timeout=10):
|
|
self.add_job(
|
|
'create_intervention_job',
|
|
pk=str(wcs_request.pk),
|
|
natural_id='wcs-request-%s' % wcs_request.pk,
|
|
)
|
|
return {'data': wcs_request.reply}
|
|
|
|
def create_intervention_job(self, *args, **kwargs):
|
|
with atomic():
|
|
wcs_request = self.wcs_requests.select_for_update().get(pk=kwargs['pk'])
|
|
after_timestamp = None
|
|
if not wcs_request.push():
|
|
if wcs_request.tries < 5:
|
|
if wcs_request.tries == 3:
|
|
after_timestamp = datetime.timedelta(hours=1)
|
|
if wcs_request.tries == 4:
|
|
after_timestamp = datetime.timedelta(days=1)
|
|
else:
|
|
wcs_request.status = 'failed'
|
|
wcs_request.save()
|
|
payload = {'creation_response': wcs_request.reply}
|
|
smart_request = wcs_request.smart_requests.create(payload=payload)
|
|
self.add_job(
|
|
'update_intervention_job',
|
|
id=smart_request.id,
|
|
natural_id='smart-request-%s' % smart_request.id,
|
|
)
|
|
if wcs_request.status == 'registered':
|
|
raise SkipJob(after_timestamp)
|
|
if wcs_request.status == 'failed':
|
|
raise Exception(wcs_request.result)
|
|
|
|
@endpoint(
|
|
name='add-media',
|
|
methods=['post'],
|
|
description=_('Add a media'),
|
|
parameters={
|
|
'uuid': {'description': _('Notification identifier')},
|
|
},
|
|
perm='can_access',
|
|
post={'request_body': {'schema': {'application/json': schemas.MEDIA_SCHEMA}}},
|
|
)
|
|
def add_media(self, request, uuid, post_data):
|
|
try:
|
|
wcs_request = self.wcs_requests.get(uuid=uuid)
|
|
except WcsRequest.DoesNotExist:
|
|
raise APIError("Cannot find intervention '%s'" % uuid, http_status=400)
|
|
|
|
nb_registered = 0
|
|
for media in post_data['files']:
|
|
if not media:
|
|
# silently ignore empty payload value
|
|
continue
|
|
wcs_request_file = wcs_request.files.create(
|
|
filename=media['filename'], content_type=media['content_type']
|
|
)
|
|
with ContentFile(base64.b64decode(media['content'])) as media_content:
|
|
wcs_request_file.content.save(media['filename'], media_content)
|
|
self.add_job(
|
|
'add_media_job',
|
|
id=wcs_request_file.id,
|
|
natural_id='wcs-request-file-%s' % wcs_request_file.id,
|
|
)
|
|
nb_registered += 1
|
|
|
|
return {'data': {'uuid': wcs_request.uuid, 'nb_registered': nb_registered}}
|
|
|
|
def add_media_job(self, *args, **kwargs):
|
|
wcs_request_file = WcsRequestFile.objects.get(id=kwargs['id'])
|
|
wcs_request = wcs_request_file.resource
|
|
if wcs_request.status == 'failed':
|
|
raise Exception('related wcs request failed')
|
|
if wcs_request.status == 'registered':
|
|
raise SkipJob(datetime.timedelta(minutes=10))
|
|
|
|
if not wcs_request_file.push():
|
|
raise SkipJob()
|
|
|
|
@atomic
|
|
@endpoint(
|
|
name='update-intervention',
|
|
methods=['post'],
|
|
description=_('Update an intervention status'),
|
|
parameters={
|
|
'uuid': {'description': _('Notification identifier')},
|
|
},
|
|
perm='can_access',
|
|
post={'request_body': {'schema': {'application/json': schemas.UPDATE_SCHEMA}}},
|
|
)
|
|
def update_intervention(self, request, uuid, post_data):
|
|
try:
|
|
wcs_request = self.wcs_requests.get(uuid=uuid)
|
|
except ValidationError as e:
|
|
raise APIError(str(e), http_status=400)
|
|
except WcsRequest.DoesNotExist:
|
|
raise APIError("Cannot find intervention '%s'" % uuid, http_status=400)
|
|
smart_request = wcs_request.smart_requests.create(payload=post_data)
|
|
self.add_job(
|
|
'update_intervention_job',
|
|
id=smart_request.id,
|
|
natural_id='smart-request-%s' % smart_request.id,
|
|
)
|
|
return {
|
|
'data': {
|
|
'wcs_form_api_url': wcs_request.wcs_form_api_url,
|
|
'wcs_form_number': wcs_request.wcs_form_number,
|
|
'uuid': wcs_request.uuid,
|
|
'payload': smart_request.payload,
|
|
}
|
|
}
|
|
|
|
def update_intervention_job(self, *args, **kwargs):
|
|
smart_request = SmartRequest.objects.get(id=kwargs['id'])
|
|
if not smart_request.push():
|
|
raise SkipJob()
|
|
|
|
|
|
class Cache(models.Model):
|
|
resource = models.ForeignKey(
|
|
verbose_name=_('Resource'),
|
|
to=ToulouseSmartResource,
|
|
on_delete=models.CASCADE,
|
|
related_name='cache_entries',
|
|
)
|
|
|
|
key = models.CharField(_('Key'), max_length=64)
|
|
|
|
timestamp = models.DateTimeField(_('Timestamp'), auto_now=True)
|
|
|
|
value = JSONField(_('Value'), default=dict)
|
|
|
|
|
|
class WcsRequest(models.Model):
|
|
resource = models.ForeignKey(
|
|
to=ToulouseSmartResource,
|
|
on_delete=models.CASCADE,
|
|
related_name='wcs_requests',
|
|
)
|
|
wcs_form_api_url = models.CharField(max_length=256)
|
|
wcs_form_number = models.CharField(max_length=16)
|
|
wcs_form_step = models.CharField(default='initial', max_length=32)
|
|
uuid = models.UUIDField(default=uuid4, primary_key=True, editable=False)
|
|
payload = JSONField(null=True)
|
|
result = JSONField(null=True)
|
|
status = models.CharField(
|
|
max_length=20,
|
|
default='registered',
|
|
choices=(
|
|
('registered', _('Registered')),
|
|
('sent', _('Sent')),
|
|
('failed', _('Failed')),
|
|
),
|
|
)
|
|
tries = models.IntegerField(default=0)
|
|
|
|
def push(self, timeout=None):
|
|
self.tries += 1
|
|
url = self.resource.webservice_base_url + 'v1/intervention'
|
|
try:
|
|
response = self.resource.request(url, method='POST', json=self.payload, timeout=timeout)
|
|
except APIError as e:
|
|
self.result = str(e)
|
|
self.save()
|
|
return False
|
|
try:
|
|
self.result = response.json()
|
|
except ValueError:
|
|
err_desc = 'invalid json, got: %s' % response.text
|
|
self.result = err_desc
|
|
self.save()
|
|
return False
|
|
for label in 'interventionCreated', 'interventionDesired':
|
|
self.result[label] = utils.utc_to_localtz(self.result[label])
|
|
self.status = 'sent'
|
|
self.save()
|
|
return True
|
|
|
|
@property
|
|
def reply(self):
|
|
return {
|
|
'wcs_form_api_url': self.wcs_form_api_url,
|
|
'wcs_form_number': self.wcs_form_number,
|
|
'uuid': str(self.uuid),
|
|
'payload': self.payload,
|
|
'result': self.result,
|
|
'status': self.status,
|
|
'tries': self.tries,
|
|
}
|
|
|
|
class Meta:
|
|
unique_together = [
|
|
('wcs_form_api_url', 'wcs_form_step'),
|
|
]
|
|
|
|
|
|
def upload_to(wcs_request_file, filename):
|
|
instance = wcs_request_file.resource.resource
|
|
uuid = wcs_request_file.resource.uuid
|
|
return '%s/%s/%s/%s' % (instance.get_connector_slug(), instance.slug, uuid, filename)
|
|
|
|
|
|
class WcsRequestFile(models.Model):
|
|
resource = models.ForeignKey(
|
|
to=WcsRequest,
|
|
on_delete=models.CASCADE,
|
|
related_name='files',
|
|
)
|
|
filename = models.CharField(max_length=256)
|
|
content_type = models.CharField(max_length=256)
|
|
content = models.FileField(upload_to=upload_to)
|
|
|
|
def push(self):
|
|
wcs_request = self.resource
|
|
intervention_id = wcs_request.result.get('id')
|
|
instance = wcs_request.resource
|
|
url = '%sv1/intervention/%s/media' % (instance.webservice_base_url, intervention_id)
|
|
files = {'media': (self.filename, self.content.open('rb'), self.content_type)}
|
|
try:
|
|
instance.request(url, method='PUT', files=files)
|
|
except APIError:
|
|
return False
|
|
self.content.delete()
|
|
return True
|
|
|
|
|
|
class SmartRequest(models.Model):
|
|
resource = models.ForeignKey(
|
|
to=WcsRequest,
|
|
on_delete=models.CASCADE,
|
|
related_name='smart_requests',
|
|
)
|
|
payload = JSONField()
|
|
result = JSONField(null=True)
|
|
|
|
def get_wcs_api(self, base_url):
|
|
scheme, netloc, dummy, dummy, dummy, dummy = urlparse.urlparse(base_url)
|
|
services = settings.KNOWN_SERVICES.get('wcs', {})
|
|
service = None
|
|
for service in services.values():
|
|
remote_url = service.get('url')
|
|
r_scheme, r_netloc, dummy, dummy, dummy, dummy = urlparse.urlparse(remote_url)
|
|
if r_scheme == scheme and r_netloc == netloc:
|
|
break
|
|
else:
|
|
return None
|
|
return WcsApi(base_url, orig=service.get('orig'), key=service.get('secret'))
|
|
|
|
def push(self):
|
|
headers = {
|
|
'Content-Type': 'application/json',
|
|
'Accept': 'application/json',
|
|
}
|
|
base_url = '%shooks/update_intervention/' % (self.resource.wcs_form_api_url)
|
|
wcs_api = self.get_wcs_api(base_url)
|
|
if not wcs_api:
|
|
err_desc = 'Cannot find wcs service for %s' % base_url
|
|
self.result = err_desc
|
|
self.save()
|
|
return True
|
|
try:
|
|
result = wcs_api.post_json(self.payload, [], headers=headers)
|
|
except WcsApiError as e:
|
|
try:
|
|
result = json.loads(e.args[3])
|
|
except ValueError:
|
|
return False
|
|
self.result = result
|
|
self.save()
|
|
return True
|