From 917e827592df4a818f559658086ffec479c5abcf Mon Sep 17 00:00:00 2001 From: Agate Date: Mon, 1 Aug 2022 16:48:58 +0200 Subject: [PATCH] esabora: initial connector implementation (#67779) --- passerelle/apps/esabora/__init__.py | 0 .../apps/esabora/migrations/0001_initial.py | 74 ++++++ .../apps/esabora/migrations/__init__.py | 0 passerelle/apps/esabora/models.py | 182 +++++++++++++ passerelle/settings.py | 1 + tests/test_esabora.py | 245 ++++++++++++++++++ 6 files changed, 502 insertions(+) create mode 100644 passerelle/apps/esabora/__init__.py create mode 100644 passerelle/apps/esabora/migrations/0001_initial.py create mode 100644 passerelle/apps/esabora/migrations/__init__.py create mode 100644 passerelle/apps/esabora/models.py create mode 100644 tests/test_esabora.py diff --git a/passerelle/apps/esabora/__init__.py b/passerelle/apps/esabora/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/esabora/migrations/0001_initial.py b/passerelle/apps/esabora/migrations/0001_initial.py new file mode 100644 index 00000000..10a5f79d --- /dev/null +++ b/passerelle/apps/esabora/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# Generated by Django 2.2.26 on 2022-08-08 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('base', '0029_auto_20210202_1627'), + ] + + operations = [ + migrations.CreateModel( + name='Esabora', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('title', models.CharField(max_length=50, verbose_name='Title')), + ('slug', models.SlugField(unique=True, verbose_name='Identifier')), + ('description', models.TextField(verbose_name='Description')), + ( + 'basic_auth_username', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication username' + ), + ), + ( + 'basic_auth_password', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication password' + ), + ), + ( + 'client_certificate', + models.FileField( + blank=True, null=True, upload_to='', verbose_name='TLS client certificate' + ), + ), + ( + 'trusted_certificate_authorities', + models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'), + ), + ( + 'verify_cert', + models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'), + ), + ( + 'http_proxy', + models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'), + ), + ( + 'service_url', + models.URLField( + help_text='Base Web Service URL, such as https://example.domain/ws/rest/', + verbose_name='Service URL', + ), + ), + ('api_key', models.CharField(blank=True, default='', max_length=256, verbose_name='API key')), + ( + 'users', + models.ManyToManyField( + blank=True, related_name='_esabora_users_+', related_query_name='+', to='base.ApiUser' + ), + ), + ], + options={ + 'verbose_name': 'Esabora', + }, + ), + ] diff --git a/passerelle/apps/esabora/migrations/__init__.py b/passerelle/apps/esabora/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/esabora/models.py b/passerelle/apps/esabora/models.py new file mode 100644 index 00000000..773d0ead --- /dev/null +++ b/passerelle/apps/esabora/models.py @@ -0,0 +1,182 @@ +import urllib.parse + +import requests +from django.db import models +from django.utils.text import slugify +from django.utils.translation import ugettext_lazy as _ + +from passerelle.base.models import BaseResource, HTTPResource +from passerelle.utils.api import endpoint +from passerelle.utils.conversion import exception_to_text +from passerelle.utils.jsonresponse import APIError + +MULT_SCHEMA = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'title': 'Multi-criterion search', + 'unflatten': True, + 'description': '', + 'type': 'object', + 'required': ['search_name', 'criterions'], + 'properties': { + 'search_name': { + 'description': _('Search name, as specified in the Esabora webservice'), + 'type': 'string', + 'examples': ['WS_ETAT_DOSSIER_SAS'], + }, + 'criterions': { + 'description': _('A mapping of criterions'), + 'type': 'object', + 'examples': [{'SAS_Référence': 'HISTO0001'}], + }, + }, +} + +DO_TREATMENT_SCHEMA = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'unflatten': True, + 'title': 'Treatment creation', + 'description': 'Additional fields in the payload will be transmitted to the Esabora service.', + 'type': 'object', + 'required': ['treatment_name'], + 'properties': { + 'endpoint': { + 'description': _('Endpoint name, such as modbdd or addevt. Defaults to modbdd'), + 'type': 'string', + 'examples': ['modbdd'], + }, + 'treatment_name': { + 'description': _('Treatment name, as specified in the Esabora service'), + 'type': 'string', + 'examples': ['IMPORT HISTOLOGE'], + }, + }, +} + + +class Esabora(BaseResource, HTTPResource): + + service_url = models.URLField( + blank=False, + verbose_name=_('Service URL'), + help_text=_('Base Web Service URL, such as https://example.domain/ws/rest/'), + ) + + api_key = models.CharField(max_length=256, default='', blank=True, verbose_name=_('API key')) + + category = _('Business Process Connectors') + + class Meta: + verbose_name = _('Esabora') + + def post(self, path, payload, **kwargs): + url = urllib.parse.urljoin(self.service_url, path) + headers = {'Authorization': f'Bearer {self.api_key}'} + try: + return self.requests.post(url, json=payload, headers=headers, timeout=5, **kwargs) + except requests.RequestException as e: + raise APIError( + 'Esabora platform "%s" connection error: %s' % (self.service_url, exception_to_text(e)), + log_error=True, + data={ + 'code': 'connection-error', + 'service_url': self.service_url, + 'error': str(e), + }, + ) + + @endpoint( + name='do-search', + description=_('Multi-criterion search'), + perm='can_access', + methods=['post'], + post={'request_body': {'schema': {'application/json': MULT_SCHEMA}}}, + json_schema_response={}, + ) + def do_search(self, request, post_data): + payload = { + 'searchName': post_data['search_name'], + 'criterionList': [ + {'criterionName': name, 'criterionValueList': [value]} + for name, value in post_data['criterions'].items() + ], + } + response = self.post('mult/', payload, params={'task': 'doSearch'}) + response.raise_for_status() + data = response.json() + columns = {slugify(c).replace('-', '_'): c for c in data['columnList']} + keys = {slugify(c).replace('-', '_'): c for c in data['keyList']} + cleaned_data = { + 'meta': { + 'nbResults': data['nbResults'], + 'searchId': data['searchId'], + 'columns_name': columns, + 'keys_name': keys, + }, + 'data': [ + esabora_row_to_object(list(columns.keys()), list(keys.keys()), row) for row in data['rowList'] + ], + } + return cleaned_data + + @endpoint( + name='do-treatment', + description=_('Create a new treatment'), + perm='can_access', + methods=['post'], + post={'request_body': {'schema': {'application/json': DO_TREATMENT_SCHEMA}}}, + json_schema_response={}, + ) + def do_treatment(self, request, post_data): + endpoint = post_data.pop('endpoint', None) or 'modbdd' + payload = get_treatment_payload(post_data) + + response = self.post(f'{endpoint}/', payload, params={'task': 'doTreatment'}) + response.raise_for_status() + data = response.json() + keys = [slugify(c).replace('-', '_') for c in data['keyList']] + cleaned_data = esabora_row_to_object([], keys, data) + cleaned_data['action'] = data['action'] + return cleaned_data + + +def esabora_row_to_object(columns, key_list, row): + key_data_list = row.get('keyDataList', []) + all_values = row.get('columnDataList', []) + key_data_list + obj = dict(zip(columns + key_list, all_values)) + obj['text'] = all_values[0] if all_values else None + if key_data_list: + obj['id'] = key_data_list[0] + return obj + + +def get_treatment_payload(post_data): + payload = {'treatmentName': post_data.pop('treatment_name'), 'fieldList': []} + for key, value in post_data.items(): + field = { + 'fieldName': key, + } + if isinstance(value, list): + populate_document_field(field, value) + elif isinstance(value, dict): + populate_document_field(field, [value]) + else: + field['fieldValue'] = value + + payload['fieldList'].append(field) + + return payload + + +def populate_document_field(field, value): + field['fieldDocumentUpdate'] = 1 + field['fieldValue'] = [] + # one or more files were sent, + # let's compute their size in bytes based on the base64 payload (without = padding) + # cf https://stackoverflow.com/a/45401395/2844093 + for f in value: + if not value: + continue + size = (len(f['content']) * 3) / 4 - f['content'].count('=', -2) + field['fieldValue'].append( + {'documentContent': f['content'], 'documentName': f['filename'], 'documentSize': size} + ) diff --git a/passerelle/settings.py b/passerelle/settings.py index cbb54802..1c385137 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -141,6 +141,7 @@ INSTALLED_APPS = ( 'passerelle.apps.cmis', 'passerelle.apps.cryptor', 'passerelle.apps.csvdatasource', + 'passerelle.apps.esabora', 'passerelle.apps.esirius', 'passerelle.apps.family', 'passerelle.apps.feeds', diff --git a/tests/test_esabora.py b/tests/test_esabora.py new file mode 100644 index 00000000..31686a17 --- /dev/null +++ b/tests/test_esabora.py @@ -0,0 +1,245 @@ +import base64 +import json + +import pytest +import responses +from django.contrib.contenttypes.models import ContentType + +import tests.utils +from passerelle.apps.esabora.models import Esabora +from passerelle.base.models import AccessRight, ApiUser + +ESABORA_DO_SEARCH_RESPONSE = { + 'searchId': '23568', + 'nbResults': 2, + 'columnList': [ + 'Column 1', + 'Column 2', + 'Column 3', + ], + 'keyList': ['internal.id'], + 'rowList': [ + { + 'columnDataList': [ + 'Foo 1', + 'Foo 2', + 'Foo 3', + ], + 'keyDataList': ['id1'], + }, + { + 'columnDataList': [ + 'Bar 1', + 'Bar 2', + 'Bar 3', + ], + 'keyDataList': ['id2'], + }, + ], +} +ESABORA_DO_TREATMENT_RESPONSE = { + 'action': 'insert', + 'keyList': ['internal.id'], + 'keyDataList': ['14'], +} + + +@pytest.fixture() +def connector(db): + api = ApiUser.objects.create(username='all', keytype='', key='') + connector = Esabora.objects.create( + service_url='http://example.esabora/ws/rest/', api_key='1234', slug='test' + ) + obj_type = ContentType.objects.get_for_model(connector) + AccessRight.objects.create( + codename='can_access', apiuser=api, resource_type=obj_type, resource_pk=connector.pk + ) + return connector + + +@responses.activate +def test_authentication(app, connector): + url = tests.utils.generic_endpoint_url('esabora', 'do-search') + responses.add( + responses.POST, + f'{connector.service_url}mult/', + json=ESABORA_DO_SEARCH_RESPONSE, + status=200, + ) + + app.post_json(url, params={'search_name': 'foo', 'criterions/bar': 'noop'}) + assert len(responses.calls) == 1 + assert responses.calls[0].request.headers['authorization'] == f'Bearer {connector.api_key}' + + +@responses.activate +def test_do_treatment(app, connector): + url = tests.utils.generic_endpoint_url('esabora', 'do-treatment') + responses.add( + responses.POST, + f'{connector.service_url}modbdd/', + json=ESABORA_DO_TREATMENT_RESPONSE, + status=200, + ) + + file_content1 = b'this is a test file' + file_content2 = b'this is another test file' + file_content3 = b'this is yet another test file' + + payload = { + 'treatment_name': 'Import HISTOLOGE', + 'Adresse_Latitude': 12.3, + 'Adresse_Ville': 'Marseille', + # documents that will be unflattened + 'PJ_Documents/0': { + 'filename': 'test1.pdf', + 'content_type': 'application/pdf', + 'content': base64.b64encode(file_content1).decode(), + }, + 'PJ_Documents/1': { + 'filename': 'test2.pdf', + 'content_type': 'application/pdf', + 'content': base64.b64encode(file_content2).decode(), + }, + # ensure we handle single documents as well + 'PJ_Documents_Autre': { + 'filename': 'test3.pdf', + 'content_type': 'application/pdf', + 'content': base64.b64encode(file_content3).decode(), + }, + } + + expected_payload = { + 'treatmentName': 'Import HISTOLOGE', + 'fieldList': [ + {'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3}, + {'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'}, + { + 'fieldName': 'PJ_Documents', + 'fieldDocumentUpdate': 1, + 'fieldValue': [ + { + 'documentName': 'test1.pdf', + 'documentContent': base64.b64encode(file_content1).decode(), + # len(file_content1) + 'documentSize': 19, + }, + { + 'documentName': 'test2.pdf', + 'documentContent': base64.b64encode(file_content2).decode(), + 'documentSize': 25, + }, + ], + }, + { + 'fieldName': 'PJ_Documents_Autre', + 'fieldDocumentUpdate': 1, + 'fieldValue': [ + { + 'documentName': 'test3.pdf', + 'documentContent': base64.b64encode(file_content3).decode(), + 'documentSize': 29, + } + ], + }, + ], + } + + response = app.post_json(url, params=payload) + assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'} + + assert len(responses.calls) == 1 + assert responses.calls[0].request.params['task'] == 'doTreatment' + response_data = json.loads(responses.calls[0].request.body) + assert response_data == expected_payload + + +@responses.activate +def test_do_search(app, connector): + url = tests.utils.generic_endpoint_url('esabora', 'do-search') + responses.add( + responses.POST, + f'{connector.service_url}mult/', + json=ESABORA_DO_SEARCH_RESPONSE, + status=200, + ) + + payload = {'search_name': 'WS_ETAT_SAS', 'criterions/foo': 'bar'} + + expected_payload = { + 'searchName': 'WS_ETAT_SAS', + 'criterionList': [{'criterionName': 'foo', 'criterionValueList': ['bar']}], + } + + response = app.post_json(url, params=payload) + + expected = { + 'err': 0, + 'data': [ + { + 'id': 'id1', + 'text': 'Foo 1', + 'internalid': 'id1', + "column_1": 'Foo 1', + "column_2": 'Foo 2', + "column_3": 'Foo 3', + }, + { + 'id': 'id2', + 'text': 'Bar 1', + 'internalid': 'id2', + "column_1": 'Bar 1', + "column_2": 'Bar 2', + "column_3": 'Bar 3', + }, + ], + 'meta': { + 'searchId': '23568', + 'nbResults': 2, + 'columns_name': { + "column_1": 'Column 1', + "column_2": 'Column 2', + "column_3": 'Column 3', + }, + 'keys_name': { + "internalid": 'internal.id', + }, + }, + } + assert response.json == expected + assert responses.calls[0].request.params['task'] == 'doSearch' + assert json.loads(responses.calls[0].request.body) == expected_payload + + +@responses.activate +def test_do_treatment_arbitrary_endpoint(app, connector): + url = tests.utils.generic_endpoint_url('esabora', 'do-treatment') + responses.add( + responses.POST, + f'{connector.service_url}addevt/', + json=ESABORA_DO_TREATMENT_RESPONSE, + status=200, + ) + + payload = { + 'endpoint': 'addevt', + 'treatment_name': 'Import Event', + 'Adresse_Latitude': 12.3, + 'Adresse_Ville': 'Marseille', + } + + expected_payload = { + 'treatmentName': 'Import Event', + 'fieldList': [ + {'fieldName': 'Adresse_Latitude', 'fieldValue': 12.3}, + {'fieldName': 'Adresse_Ville', 'fieldValue': 'Marseille'}, + ], + } + + response = app.post_json(url, params=payload) + assert response.json == {'err': 0, 'action': 'insert', 'id': '14', 'internalid': '14', 'text': '14'} + + assert len(responses.calls) == 1 + assert responses.calls[0].request.params['task'] == 'doTreatment' + response_data = json.loads(responses.calls[0].request.body) + assert response_data == expected_payload