esabora: initial connector implementation (#67779)
This commit is contained in:
parent
4e5c746582
commit
917e827592
|
@ -0,0 +1,74 @@
|
|||
# Generated by Django 2.2.26 on 2022-08-08 09:24
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
('base', '0029_auto_20210202_1627'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='Esabora',
|
||||
fields=[
|
||||
(
|
||||
'id',
|
||||
models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
('title', models.CharField(max_length=50, verbose_name='Title')),
|
||||
('slug', models.SlugField(unique=True, verbose_name='Identifier')),
|
||||
('description', models.TextField(verbose_name='Description')),
|
||||
(
|
||||
'basic_auth_username',
|
||||
models.CharField(
|
||||
blank=True, max_length=128, verbose_name='Basic authentication username'
|
||||
),
|
||||
),
|
||||
(
|
||||
'basic_auth_password',
|
||||
models.CharField(
|
||||
blank=True, max_length=128, verbose_name='Basic authentication password'
|
||||
),
|
||||
),
|
||||
(
|
||||
'client_certificate',
|
||||
models.FileField(
|
||||
blank=True, null=True, upload_to='', verbose_name='TLS client certificate'
|
||||
),
|
||||
),
|
||||
(
|
||||
'trusted_certificate_authorities',
|
||||
models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'),
|
||||
),
|
||||
(
|
||||
'verify_cert',
|
||||
models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'),
|
||||
),
|
||||
(
|
||||
'http_proxy',
|
||||
models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'),
|
||||
),
|
||||
(
|
||||
'service_url',
|
||||
models.URLField(
|
||||
help_text='Base Web Service URL, such as https://example.domain/ws/rest/',
|
||||
verbose_name='Service URL',
|
||||
),
|
||||
),
|
||||
('api_key', models.CharField(blank=True, default='', max_length=256, verbose_name='API key')),
|
||||
(
|
||||
'users',
|
||||
models.ManyToManyField(
|
||||
blank=True, related_name='_esabora_users_+', related_query_name='+', to='base.ApiUser'
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Esabora',
|
||||
},
|
||||
),
|
||||
]
|
|
@ -0,0 +1,182 @@
|
|||
import urllib.parse
|
||||
|
||||
import requests
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from passerelle.base.models import BaseResource, HTTPResource
|
||||
from passerelle.utils.api import endpoint
|
||||
from passerelle.utils.conversion import exception_to_text
|
||||
from passerelle.utils.jsonresponse import APIError
|
||||
|
||||
MULT_SCHEMA = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'title': 'Multi-criterion search',
|
||||
'unflatten': True,
|
||||
'description': '',
|
||||
'type': 'object',
|
||||
'required': ['search_name', 'criterions'],
|
||||
'properties': {
|
||||
'search_name': {
|
||||
'description': _('Search name, as specified in the Esabora webservice'),
|
||||
'type': 'string',
|
||||
'examples': ['WS_ETAT_DOSSIER_SAS'],
|
||||
},
|
||||
'criterions': {
|
||||
'description': _('A mapping of criterions'),
|
||||
'type': 'object',
|
||||
'examples': [{'SAS_Référence': 'HISTO0001'}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
DO_TREATMENT_SCHEMA = {
|
||||
'$schema': 'http://json-schema.org/draft-04/schema#',
|
||||
'unflatten': True,
|
||||
'title': 'Treatment creation',
|
||||
'description': 'Additional fields in the payload will be transmitted to the Esabora service.',
|
||||
'type': 'object',
|
||||
'required': ['treatment_name'],
|
||||
'properties': {
|
||||
'endpoint': {
|
||||
'description': _('Endpoint name, such as modbdd or addevt. Defaults to modbdd'),
|
||||
'type': 'string',
|
||||
'examples': ['modbdd'],
|
||||
},
|
||||
'treatment_name': {
|
||||
'description': _('Treatment name, as specified in the Esabora service'),
|
||||
'type': 'string',
|
||||
'examples': ['IMPORT HISTOLOGE'],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Esabora(BaseResource, HTTPResource):
|
||||
|
||||
service_url = models.URLField(
|
||||
blank=False,
|
||||
verbose_name=_('Service URL'),
|
||||
help_text=_('Base Web Service URL, such as https://example.domain/ws/rest/'),
|
||||
)
|
||||
|
||||
api_key = models.CharField(max_length=256, default='', blank=True, verbose_name=_('API key'))
|
||||
|
||||
category = _('Business Process Connectors')
|
||||
|
||||
class Meta:
|
||||
verbose_name = _('Esabora')
|
||||
|
||||
def post(self, path, payload, **kwargs):
|
||||
url = urllib.parse.urljoin(self.service_url, path)
|
||||
headers = {'Authorization': f'Bearer {self.api_key}'}
|
||||
try:
|
||||
return self.requests.post(url, json=payload, headers=headers, timeout=5, **kwargs)
|
||||
except requests.RequestException as e:
|
||||
raise APIError(
|
||||
'Esabora platform "%s" connection error: %s' % (self.service_url, exception_to_text(e)),
|
||||
log_error=True,
|
||||
data={
|
||||
'code': 'connection-error',
|
||||
'service_url': self.service_url,
|
||||
'error': str(e),
|
||||
},
|
||||
)
|
||||
|
||||
@endpoint(
|
||||
name='do-search',
|
||||
description=_('Multi-criterion search'),
|
||||
perm='can_access',
|
||||
methods=['post'],
|
||||
post={'request_body': {'schema': {'application/json': MULT_SCHEMA}}},
|
||||
json_schema_response={},
|
||||
)
|
||||
def do_search(self, request, post_data):
|
||||
payload = {
|
||||
'searchName': post_data['search_name'],
|
||||
'criterionList': [
|
||||
{'criterionName': name, 'criterionValueList': [value]}
|
||||
for name, value in post_data['criterions'].items()
|
||||
],
|
||||
}
|
||||
response = self.post('mult/', payload, params={'task': 'doSearch'})
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
columns = {slugify(c).replace('-', '_'): c for c in data['columnList']}
|
||||
keys = {slugify(c).replace('-', '_'): c for c in data['keyList']}
|
||||
cleaned_data = {
|
||||
'meta': {
|
||||
'nbResults': data['nbResults'],
|
||||
'searchId': data['searchId'],
|
||||
'columns_name': columns,
|
||||
'keys_name': keys,
|
||||
},
|
||||
'data': [
|
||||
esabora_row_to_object(list(columns.keys()), list(keys.keys()), row) for row in data['rowList']
|
||||
],
|
||||
}
|
||||
return cleaned_data
|
||||
|
||||
@endpoint(
|
||||
name='do-treatment',
|
||||
description=_('Create a new treatment'),
|
||||
perm='can_access',
|
||||
methods=['post'],
|
||||
post={'request_body': {'schema': {'application/json': DO_TREATMENT_SCHEMA}}},
|
||||
json_schema_response={},
|
||||
)
|
||||
def do_treatment(self, request, post_data):
|
||||
endpoint = post_data.pop('endpoint', None) or 'modbdd'
|
||||
payload = get_treatment_payload(post_data)
|
||||
|
||||
response = self.post(f'{endpoint}/', payload, params={'task': 'doTreatment'})
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
keys = [slugify(c).replace('-', '_') for c in data['keyList']]
|
||||
cleaned_data = esabora_row_to_object([], keys, data)
|
||||
cleaned_data['action'] = data['action']
|
||||
return cleaned_data
|
||||
|
||||
|
||||
def esabora_row_to_object(columns, key_list, row):
|
||||
key_data_list = row.get('keyDataList', [])
|
||||
all_values = row.get('columnDataList', []) + key_data_list
|
||||
obj = dict(zip(columns + key_list, all_values))
|
||||
obj['text'] = all_values[0] if all_values else None
|
||||
if key_data_list:
|
||||
obj['id'] = key_data_list[0]
|
||||
return obj
|
||||
|
||||
|
||||
def get_treatment_payload(post_data):
|
||||
payload = {'treatmentName': post_data.pop('treatment_name'), 'fieldList': []}
|
||||
for key, value in post_data.items():
|
||||
field = {
|
||||
'fieldName': key,
|
||||
}
|
||||
if isinstance(value, list):
|
||||
populate_document_field(field, value)
|
||||
elif isinstance(value, dict):
|
||||
populate_document_field(field, [value])
|
||||
else:
|
||||
field['fieldValue'] = value
|
||||
|
||||
payload['fieldList'].append(field)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def populate_document_field(field, value):
|
||||
field['fieldDocumentUpdate'] = 1
|
||||
field['fieldValue'] = []
|
||||
# one or more files were sent,
|
||||
# let's compute their size in bytes based on the base64 payload (without = padding)
|
||||
# cf https://stackoverflow.com/a/45401395/2844093
|
||||
for f in value:
|
||||
if not value:
|
||||
continue
|
||||
size = (len(f['content']) * 3) / 4 - f['content'].count('=', -2)
|
||||
field['fieldValue'].append(
|
||||
{'documentContent': f['content'], 'documentName': f['filename'], 'documentSize': size}
|
||||
)
|
|
@ -141,6 +141,7 @@ INSTALLED_APPS = (
|
|||
'passerelle.apps.cmis',
|
||||
'passerelle.apps.cryptor',
|
||||
'passerelle.apps.csvdatasource',
|
||||
'passerelle.apps.esabora',
|
||||
'passerelle.apps.esirius',
|
||||
'passerelle.apps.family',
|
||||
'passerelle.apps.feeds',
|
||||
|
|
|
@ -0,0 +1,245 @@
|
|||
import base64
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
|
||||
import tests.utils
|
||||
from passerelle.apps.esabora.models import Esabora
|
||||
from passerelle.base.models import AccessRight, ApiUser
|
||||
|
||||
ESABORA_DO_SEARCH_RESPONSE = {
|
||||
'searchId': '23568',
|
||||
'nbResults': 2,
|
||||
'columnList': [
|
||||
'Column 1',
|
||||
'Column 2',
|
||||
'Column 3',
|
||||
],
|
||||
'keyList': ['internal.id'],
|
||||
'rowList': [
|
||||
{
|
||||
'columnDataList': [
|
||||
'Foo 1',
|
||||
'Foo 2',
|
||||
'Foo 3',
|
||||
],
|
||||
'keyDataList': ['id1'],
|
||||
},
|
||||
{
|
||||
'columnDataList': [
|
||||
'Bar 1',
|
||||
'Bar 2',
|
||||
'Bar 3',
|
||||
],
|
||||
'keyDataList': ['id2'],
|
||||
},
|
||||
],
|
||||
}
|
||||
ESABORA_DO_TREATMENT_RESPONSE = {
|
||||
'action': 'insert',
|
||||
'keyList': ['internal.id'],
|
||||
'keyDataList': ['14'],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def connector(db):
|
||||
api = ApiUser.objects.create(username='all', keytype='', key='')
|
||||
connector = Esabora.objects.create(
|
||||
service_url='http://example.esabora/ws/rest/', api_key='1234', slug='test'
|
||||
)
|
||||
obj_type = ContentType.objects.get_for_model(connector)
|
||||
AccessRight.objects.create(
|
||||
codename='can_access', apiuser=api, resource_type=obj_type, resource_pk=connector.pk
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_authentication(app, connector):
|
||||
url = tests.utils.generic_endpoint_url('esabora', 'do-search')
|
||||
responses.add(
|
||||
responses.POST,
|
||||
f'{connector.service_url}mult/',
|
||||
json=ESABORA_DO_SEARCH_RESPONSE,
|
||||
status=200,
|
||||
)
|
||||
|
||||
app.post_json(url, params={'search_name': 'foo', 'criterions/bar': 'noop'})
|
||||
assert len(responses.calls) == 1
|
||||
assert responses.calls[0].request.headers['authorization'] == f'Bearer {connector.api_key}'
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_do_treatment(app, connector):
|
||||
url = tests.utils.generic_endpoint_url('esabora', 'do-treatment')
|
||||
responses.add(
|
||||
responses.POST,
|
||||
f'{connector.service_url}modbdd/',
|
||||
json=ESABORA_DO_TREATMENT_RESPONSE,
|
||||
status=200,
|
||||
)
|
||||
|
||||
file_content1 = b'this is a test file'
|
||||
file_content2 = b'this is another test file'
|
||||
file_content3 = b'this is yet another test file'
|
||||
|
||||
payload = {
|
||||
'treatment_name': 'Import HISTOLOGE',
|
||||
'Adresse_Latitude': 12.3,
|
||||
'Adresse_Ville': 'Marseille',
|
||||
# documents that will be unflattened
|
||||
'PJ_Documents/0': {
|
||||
'filename': 'test1.pdf',
|
||||
'content_type': 'application/pdf',
|
||||
'content': base64.b64encode(file_content1).decode(),
|
||||
},
|
||||
'PJ_Documents/1': {
|
||||
'filename': 'test2.pdf',
|
||||
'content_type': 'application/pdf',
|
||||
'content': base64.b64encode(file_content2).decode(),
|
||||
},
|
||||
# ensure we handle single documents as well
|
||||
'PJ_Documents_Autre': {
|
||||
'filename': 'test3.pdf',
|
||||
'content_type': 'application/pdf',
|
||||
'content': base64.b64encode(file_content3).decode(),
|
||||
},
|
||||
}
|
||||
|
||||
expected_payload = {
|
||||
'treatmentName': 'Import HISTOLOGE',
|
||||
'fieldList': [
|
||||
{'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3},
|
||||
{'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'},
|
||||
{
|
||||
'fieldName': 'PJ_Documents',
|
||||
'fieldDocumentUpdate': 1,
|
||||
'fieldValue': [
|
||||
{
|
||||
'documentName': 'test1.pdf',
|
||||
'documentContent': base64.b64encode(file_content1).decode(),
|
||||
# len(file_content1)
|
||||
'documentSize': 19,
|
||||
},
|
||||
{
|
||||
'documentName': 'test2.pdf',
|
||||
'documentContent': base64.b64encode(file_content2).decode(),
|
||||
'documentSize': 25,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
'fieldName': 'PJ_Documents_Autre',
|
||||
'fieldDocumentUpdate': 1,
|
||||
'fieldValue': [
|
||||
{
|
||||
'documentName': 'test3.pdf',
|
||||
'documentContent': base64.b64encode(file_content3).decode(),
|
||||
'documentSize': 29,
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
response = app.post_json(url, params=payload)
|
||||
assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'}
|
||||
|
||||
assert len(responses.calls) == 1
|
||||
assert responses.calls[0].request.params['task'] == 'doTreatment'
|
||||
response_data = json.loads(responses.calls[0].request.body)
|
||||
assert response_data == expected_payload
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_do_search(app, connector):
|
||||
url = tests.utils.generic_endpoint_url('esabora', 'do-search')
|
||||
responses.add(
|
||||
responses.POST,
|
||||
f'{connector.service_url}mult/',
|
||||
json=ESABORA_DO_SEARCH_RESPONSE,
|
||||
status=200,
|
||||
)
|
||||
|
||||
payload = {'search_name': 'WS_ETAT_SAS', 'criterions/foo': 'bar'}
|
||||
|
||||
expected_payload = {
|
||||
'searchName': 'WS_ETAT_SAS',
|
||||
'criterionList': [{'criterionName': 'foo', 'criterionValueList': ['bar']}],
|
||||
}
|
||||
|
||||
response = app.post_json(url, params=payload)
|
||||
|
||||
expected = {
|
||||
'err': 0,
|
||||
'data': [
|
||||
{
|
||||
'id': 'id1',
|
||||
'text': 'Foo 1',
|
||||
'internalid': 'id1',
|
||||
"column_1": 'Foo 1',
|
||||
"column_2": 'Foo 2',
|
||||
"column_3": 'Foo 3',
|
||||
},
|
||||
{
|
||||
'id': 'id2',
|
||||
'text': 'Bar 1',
|
||||
'internalid': 'id2',
|
||||
"column_1": 'Bar 1',
|
||||
"column_2": 'Bar 2',
|
||||
"column_3": 'Bar 3',
|
||||
},
|
||||
],
|
||||
'meta': {
|
||||
'searchId': '23568',
|
||||
'nbResults': 2,
|
||||
'columns_name': {
|
||||
"column_1": 'Column 1',
|
||||
"column_2": 'Column 2',
|
||||
"column_3": 'Column 3',
|
||||
},
|
||||
'keys_name': {
|
||||
"internalid": 'internal.id',
|
||||
},
|
||||
},
|
||||
}
|
||||
assert response.json == expected
|
||||
assert responses.calls[0].request.params['task'] == 'doSearch'
|
||||
assert json.loads(responses.calls[0].request.body) == expected_payload
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_do_treatment_arbitrary_endpoint(app, connector):
|
||||
url = tests.utils.generic_endpoint_url('esabora', 'do-treatment')
|
||||
responses.add(
|
||||
responses.POST,
|
||||
f'{connector.service_url}addevt/',
|
||||
json=ESABORA_DO_TREATMENT_RESPONSE,
|
||||
status=200,
|
||||
)
|
||||
|
||||
payload = {
|
||||
'endpoint': 'addevt',
|
||||
'treatment_name': 'Import Event',
|
||||
'Adresse_Latitude': 12.3,
|
||||
'Adresse_Ville': 'Marseille',
|
||||
}
|
||||
|
||||
expected_payload = {
|
||||
'treatmentName': 'Import Event',
|
||||
'fieldList': [
|
||||
{'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3},
|
||||
{'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'},
|
||||
],
|
||||
}
|
||||
|
||||
response = app.post_json(url, params=payload)
|
||||
assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'}
|
||||
|
||||
assert len(responses.calls) == 1
|
||||
assert responses.calls[0].request.params['task'] == 'doTreatment'
|
||||
response_data = json.loads(responses.calls[0].request.body)
|
||||
assert response_data == expected_payload
|
Loading…
Reference in New Issue