entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 15 Releases Activity

esabora: initial connector implementation (#67779)

This commit is contained in:
Agate 2022-08-01 16:48:58 +02:00 committed by Agate Berriot
parent 4e5c746582
commit 917e827592
6 changed files with 502 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
passerelle/apps/esabora/__init__.py Normal file
View File

74
passerelle/apps/esabora/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,74 @@
# Generated by Django 2.2.26 on 2022-08-08 09:24
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('base', '0029_auto_20210202_1627'),
]
operations = [
migrations.CreateModel(
name='Esabora',
fields=[
(
'id',
models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
('title', models.CharField(max_length=50, verbose_name='Title')),
('slug', models.SlugField(unique=True, verbose_name='Identifier')),
('description', models.TextField(verbose_name='Description')),
(
'basic_auth_username',
models.CharField(
blank=True, max_length=128, verbose_name='Basic authentication username'
),
),
(
'basic_auth_password',
models.CharField(
blank=True, max_length=128, verbose_name='Basic authentication password'
),
),
(
'client_certificate',
models.FileField(
blank=True, null=True, upload_to='', verbose_name='TLS client certificate'
),
),
(
'trusted_certificate_authorities',
models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'),
),
(
'verify_cert',
models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'),
),
(
'http_proxy',
models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'),
),
(
'service_url',
models.URLField(
help_text='Base Web Service URL, such as https://example.domain/ws/rest/',
verbose_name='Service URL',
),
),
('api_key', models.CharField(blank=True, default='', max_length=256, verbose_name='API key')),
(
'users',
models.ManyToManyField(
blank=True, related_name='_esabora_users_+', related_query_name='+', to='base.ApiUser'
),
),
],
options={
'verbose_name': 'Esabora',
},
),
]

0
passerelle/apps/esabora/migrations/__init__.py Normal file
View File

182
passerelle/apps/esabora/models.py Normal file
View File

@ -0,0 +1,182 @@
import urllib.parse
import requests
from django.db import models
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from passerelle.base.models import BaseResource, HTTPResource
from passerelle.utils.api import endpoint
from passerelle.utils.conversion import exception_to_text
from passerelle.utils.jsonresponse import APIError
MULT_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'title': 'Multi-criterion search',
'unflatten': True,
'description': '',
'type': 'object',
'required': ['search_name', 'criterions'],
'properties': {
'search_name': {
'description': _('Search name, as specified in the Esabora webservice'),
'type': 'string',
'examples': ['WS_ETAT_DOSSIER_SAS'],
},
'criterions': {
'description': _('A mapping of criterions'),
'type': 'object',
'examples': [{'SAS_Référence': 'HISTO0001'}],
},
},
}
DO_TREATMENT_SCHEMA = {
'$schema': 'http://json-schema.org/draft-04/schema#',
'unflatten': True,
'title': 'Treatment creation',
'description': 'Additional fields in the payload will be transmitted to the Esabora service.',
'type': 'object',
'required': ['treatment_name'],
'properties': {
'endpoint': {
'description': _('Endpoint name, such as modbdd or addevt. Defaults to modbdd'),
'type': 'string',
'examples': ['modbdd'],
},
'treatment_name': {
'description': _('Treatment name, as specified in the Esabora service'),
'type': 'string',
'examples': ['IMPORT HISTOLOGE'],
},
},
}
class Esabora(BaseResource, HTTPResource):
service_url = models.URLField(
blank=False,
verbose_name=_('Service URL'),
help_text=_('Base Web Service URL, such as https://example.domain/ws/rest/'),
)
api_key = models.CharField(max_length=256, default='', blank=True, verbose_name=_('API key'))
category = _('Business Process Connectors')
class Meta:
verbose_name = _('Esabora')
def post(self, path, payload, **kwargs):
url = urllib.parse.urljoin(self.service_url, path)
headers = {'Authorization': f'Bearer {self.api_key}'}
try:
return self.requests.post(url, json=payload, headers=headers, timeout=5, **kwargs)
except requests.RequestException as e:
raise APIError(
'Esabora platform "%s" connection error: %s' % (self.service_url, exception_to_text(e)),
log_error=True,
data={
'code': 'connection-error',
'service_url': self.service_url,
'error': str(e),
},
)
@endpoint(
name='do-search',
description=_('Multi-criterion search'),
perm='can_access',
methods=['post'],
post={'request_body': {'schema': {'application/json': MULT_SCHEMA}}},
json_schema_response={},
)
def do_search(self, request, post_data):
payload = {
'searchName': post_data['search_name'],
'criterionList': [
{'criterionName': name, 'criterionValueList': [value]}
for name, value in post_data['criterions'].items()
],
}
response = self.post('mult/', payload, params={'task': 'doSearch'})
response.raise_for_status()
data = response.json()
columns = {slugify(c).replace('-', '_'): c for c in data['columnList']}
keys = {slugify(c).replace('-', '_'): c for c in data['keyList']}
cleaned_data = {
'meta': {
'nbResults': data['nbResults'],
'searchId': data['searchId'],
'columns_name': columns,
'keys_name': keys,
},
'data': [
esabora_row_to_object(list(columns.keys()), list(keys.keys()), row) for row in data['rowList']
],
}
return cleaned_data
@endpoint(
name='do-treatment',
description=_('Create a new treatment'),
perm='can_access',
methods=['post'],
post={'request_body': {'schema': {'application/json': DO_TREATMENT_SCHEMA}}},
json_schema_response={},
)
def do_treatment(self, request, post_data):
endpoint = post_data.pop('endpoint', None) or 'modbdd'
payload = get_treatment_payload(post_data)
response = self.post(f'{endpoint}/', payload, params={'task': 'doTreatment'})
response.raise_for_status()
data = response.json()
keys = [slugify(c).replace('-', '_') for c in data['keyList']]
cleaned_data = esabora_row_to_object([], keys, data)
cleaned_data['action'] = data['action']
return cleaned_data
def esabora_row_to_object(columns, key_list, row):
key_data_list = row.get('keyDataList', [])
all_values = row.get('columnDataList', []) + key_data_list
obj = dict(zip(columns + key_list, all_values))
obj['text'] = all_values[0] if all_values else None
if key_data_list:
obj['id'] = key_data_list[0]
return obj
def get_treatment_payload(post_data):
payload = {'treatmentName': post_data.pop('treatment_name'), 'fieldList': []}
for key, value in post_data.items():
field = {
'fieldName': key,
}
if isinstance(value, list):
populate_document_field(field, value)
elif isinstance(value, dict):
populate_document_field(field, [value])
else:
field['fieldValue'] = value
payload['fieldList'].append(field)
return payload
def populate_document_field(field, value):
field['fieldDocumentUpdate'] = 1
field['fieldValue'] = []
# one or more files were sent,
# let's compute their size in bytes based on the base64 payload (without = padding)
# cf https://stackoverflow.com/a/45401395/2844093
for f in value:
if not value:
continue
size = (len(f['content']) * 3) / 4 - f['content'].count('=', -2)
field['fieldValue'].append(
{'documentContent': f['content'], 'documentName': f['filename'], 'documentSize': size}
)

1
passerelle/settings.py
View File

@ -141,6 +141,7 @@ INSTALLED_APPS = (
'passerelle.apps.cmis',
'passerelle.apps.cryptor',
'passerelle.apps.csvdatasource',
'passerelle.apps.esabora',
'passerelle.apps.esirius',
'passerelle.apps.family',
'passerelle.apps.feeds',

245
tests/test_esabora.py Normal file
View File

@ -0,0 +1,245 @@
import base64
import json
import pytest
import responses
from django.contrib.contenttypes.models import ContentType
import tests.utils
from passerelle.apps.esabora.models import Esabora
from passerelle.base.models import AccessRight, ApiUser
ESABORA_DO_SEARCH_RESPONSE = {
'searchId': '23568',
'nbResults': 2,
'columnList': [
'Column 1',
'Column 2',
'Column 3',
],
'keyList': ['internal.id'],
'rowList': [
{
'columnDataList': [
'Foo 1',
'Foo 2',
'Foo 3',
],
'keyDataList': ['id1'],
},
{
'columnDataList': [
'Bar 1',
'Bar 2',
'Bar 3',
],
'keyDataList': ['id2'],
},
],
}
ESABORA_DO_TREATMENT_RESPONSE = {
'action': 'insert',
'keyList': ['internal.id'],
'keyDataList': ['14'],
}
@pytest.fixture()
def connector(db):
api = ApiUser.objects.create(username='all', keytype='', key='')
connector = Esabora.objects.create(
service_url='http://example.esabora/ws/rest/', api_key='1234', slug='test'
)
obj_type = ContentType.objects.get_for_model(connector)
AccessRight.objects.create(
codename='can_access', apiuser=api, resource_type=obj_type, resource_pk=connector.pk
)
return connector
@responses.activate
def test_authentication(app, connector):
url = tests.utils.generic_endpoint_url('esabora', 'do-search')
responses.add(
responses.POST,
f'{connector.service_url}mult/',
json=ESABORA_DO_SEARCH_RESPONSE,
status=200,
)
app.post_json(url, params={'search_name': 'foo', 'criterions/bar': 'noop'})
assert len(responses.calls) == 1
assert responses.calls[0].request.headers['authorization'] == f'Bearer {connector.api_key}'
@responses.activate
def test_do_treatment(app, connector):
url = tests.utils.generic_endpoint_url('esabora', 'do-treatment')
responses.add(
responses.POST,
f'{connector.service_url}modbdd/',
json=ESABORA_DO_TREATMENT_RESPONSE,
status=200,
)
file_content1 = b'this is a test file'
file_content2 = b'this is another test file'
file_content3 = b'this is yet another test file'
payload = {
'treatment_name': 'Import HISTOLOGE',
'Adresse_Latitude': 12.3,
'Adresse_Ville': 'Marseille',
# documents that will be unflattened
'PJ_Documents/0': {
'filename': 'test1.pdf',
'content_type': 'application/pdf',
'content': base64.b64encode(file_content1).decode(),
},
'PJ_Documents/1': {
'filename': 'test2.pdf',
'content_type': 'application/pdf',
'content': base64.b64encode(file_content2).decode(),
},
# ensure we handle single documents as well
'PJ_Documents_Autre': {
'filename': 'test3.pdf',
'content_type': 'application/pdf',
'content': base64.b64encode(file_content3).decode(),
},
}
expected_payload = {
'treatmentName': 'Import HISTOLOGE',
'fieldList': [
{'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3},
{'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'},
{
'fieldName': 'PJ_Documents',
'fieldDocumentUpdate': 1,
'fieldValue': [
{
'documentName': 'test1.pdf',
'documentContent': base64.b64encode(file_content1).decode(),
# len(file_content1)
'documentSize': 19,
},
{
'documentName': 'test2.pdf',
'documentContent': base64.b64encode(file_content2).decode(),
'documentSize': 25,
},
],
},
{
'fieldName': 'PJ_Documents_Autre',
'fieldDocumentUpdate': 1,
'fieldValue': [
{
'documentName': 'test3.pdf',
'documentContent': base64.b64encode(file_content3).decode(),
'documentSize': 29,
}
],
},
],
}
response = app.post_json(url, params=payload)
assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'}
assert len(responses.calls) == 1
assert responses.calls[0].request.params['task'] == 'doTreatment'
response_data = json.loads(responses.calls[0].request.body)
assert response_data == expected_payload
@responses.activate
def test_do_search(app, connector):
url = tests.utils.generic_endpoint_url('esabora', 'do-search')
responses.add(
responses.POST,
f'{connector.service_url}mult/',
json=ESABORA_DO_SEARCH_RESPONSE,
status=200,
)
payload = {'search_name': 'WS_ETAT_SAS', 'criterions/foo': 'bar'}
expected_payload = {
'searchName': 'WS_ETAT_SAS',
'criterionList': [{'criterionName': 'foo', 'criterionValueList': ['bar']}],
}
response = app.post_json(url, params=payload)
expected = {
'err': 0,
'data': [
{
'id': 'id1',
'text': 'Foo 1',
'internalid': 'id1',
"column_1": 'Foo 1',
"column_2": 'Foo 2',
"column_3": 'Foo 3',
},
{
'id': 'id2',
'text': 'Bar 1',
'internalid': 'id2',
"column_1": 'Bar 1',
"column_2": 'Bar 2',
"column_3": 'Bar 3',
},
],
'meta': {
'searchId': '23568',
'nbResults': 2,
'columns_name': {
"column_1": 'Column 1',
"column_2": 'Column 2',
"column_3": 'Column 3',
},
'keys_name': {
"internalid": 'internal.id',
},
},
}
assert response.json == expected
assert responses.calls[0].request.params['task'] == 'doSearch'
assert json.loads(responses.calls[0].request.body) == expected_payload
@responses.activate
def test_do_treatment_arbitrary_endpoint(app, connector):
url = tests.utils.generic_endpoint_url('esabora', 'do-treatment')
responses.add(
responses.POST,
f'{connector.service_url}addevt/',
json=ESABORA_DO_TREATMENT_RESPONSE,
status=200,
)
payload = {
'endpoint': 'addevt',
'treatment_name': 'Import Event',
'Adresse_Latitude': 12.3,
'Adresse_Ville': 'Marseille',
}
expected_payload = {
'treatmentName': 'Import Event',
'fieldList': [
{'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3},
{'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'},
],
}
response = app.post_json(url, params=payload)
assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'}
assert len(responses.calls) == 1
assert responses.calls[0].request.params['task'] == 'doTreatment'
response_data = json.loads(responses.calls[0].request.body)
assert response_data == expected_payload