filr_rest: start connector (#73226)

passerelle/apps/filr_rest/migrations/0001_initial.py

@@ -0,0 +1,67 @@
+# Generated by Django 2.2.26 on 2023-01-10 13:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('base', '0030_resourcelog_base_resour_appname_298cbc_idx'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Filr',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=50, verbose_name='Title')),
+                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
+                ('description', models.TextField(verbose_name='Description')),
+                (
+                    'basic_auth_username',
+                    models.CharField(
+                        blank=True, max_length=128, verbose_name='Basic authentication username'
+                    ),
+                ),
+                (
+                    'basic_auth_password',
+                    models.CharField(
+                        blank=True, max_length=128, verbose_name='Basic authentication password'
+                    ),
+                ),
+                (
+                    'client_certificate',
+                    models.FileField(
+                        blank=True, null=True, upload_to='', verbose_name='TLS client certificate'
+                    ),
+                ),
+                (
+                    'trusted_certificate_authorities',
+                    models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'),
+                ),
+                (
+                    'verify_cert',
+                    models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'),
+                ),
+                (
+                    'http_proxy',
+                    models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'),
+                ),
+                ('base_url', models.URLField(verbose_name='Webservice Base URL')),
+                (
+                    'users',
+                    models.ManyToManyField(
+                        blank=True, related_name='_filr_users_+', related_query_name='+', to='base.ApiUser'
+                    ),
+                ),
+            ],
+            options={
+                'verbose_name': 'Filr REST API',
+            },
+        ),
+    ]

passerelle/apps/filr_rest/models.py

@@ -0,0 +1,156 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2023  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import binascii
+import json
+import urllib
+
+import requests
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.base.models import BaseResource, HTTPResource
+from passerelle.utils.api import endpoint
+from passerelle.utils.jsonresponse import APIError
+
+from . import schemas
+
+
+class Filr(BaseResource, HTTPResource):
+    base_url = models.URLField(
+        verbose_name=_('Webservice Base URL'),
+    )
+
+    category = _('File Storage')
+
+    class Meta:
+        verbose_name = _('Filr REST API')
+
+    def _call(self, path, method='get', json_data=None, data=None, headers=None, params=None):
+        kwargs = {}
+        if data:
+            kwargs['data'] = data
+        if json_data:
+            kwargs['json'] = json_data
+        if headers:
+            kwargs['headers'] = headers
+        if params:
+            kwargs['params'] = params
+        try:
+            resp = self.requests.request(
+                method=method, url=urllib.parse.urljoin(self.base_url, path), **kwargs
+            )
+        except (requests.Timeout, requests.RequestException) as e:
+            raise APIError(str(e))
+        try:
+            resp.raise_for_status()
+        except requests.RequestException as main_exc:
+            try:
+                err_data = resp.json()
+            except (json.JSONDecodeError, requests.exceptions.RequestException):
+                err_data = {'response_text': resp.text}
+            raise APIError(str(main_exc), data=err_data)
+
+        content_type = resp.headers.get('Content-Type')
+        if content_type and content_type.startswith('application/json') and resp.status_code != 204:
+            try:
+                return resp.json()
+            except (json.JSONDecodeError, requests.exceptions.JSONDecodeError) as e:
+                raise APIError(str(e))
+
+        return resp.text
+
+    @endpoint(
+        perm='can_access',
+        description=_('Upload a file'),
+        post={
+            'request_body': {'schema': {'application/json': schemas.UPLOAD}},
+        },
+    )
+    def upload(self, request, post_data):
+        root_folder_id, folder_name = post_data['root_folder_id'], post_data['folder_name']
+        try:
+            file_content = base64.b64decode(post_data['file']['content'])
+        except (TypeError, binascii.Error):
+            raise APIError('invalid base64 string', http_status=400)
+        filename = post_data.get('filename') or post_data['file']['filename']
+
+        # get or create folder
+        folder_id = None
+        root_folder_info = self._call('rest/folders/%s/library_folders' % root_folder_id)
+        for folder in root_folder_info.get('items', []):
+            if folder.get('title') == folder_name:
+                folder_id = folder.get('id')
+                break
+        else:
+            folder_info = self._call(
+                'rest/folders/%s/library_folders' % root_folder_id,
+                method='post',
+                json_data={'title': folder_name},
+            )
+            folder_id = folder_info['id']
+
+        # upload file
+        file_info = self._call(
+            'rest/folders/%s/library_files' % folder_id,
+            method='post',
+            params={'file_name': filename},
+            headers={'Content-Type': 'application/octet-stream'},
+            data=file_content,
+        )
+        return {'data': {'folder_id': folder_id, 'file_info': file_info}}
+
+    @endpoint(
+        name='share-folder',
+        perm='can_access',
+        description=_('Share a folder to external users'),
+        post={
+            'request_body': {'schema': {'application/json': schemas.SHARE_FOLDER}},
+            'input_example': {
+                'folder_id': '1234',
+                'emails/0': 'foo@invalid',
+                'emails/1': 'bar@invalid',
+                'days_to_expire': '30',
+            },
+        },
+    )
+    def share_folder(self, request, post_data):
+        data = []
+        for email in post_data['emails']:
+            share_info = self._call(
+                'rest/folders/%s/shares' % post_data['folder_id'],
+                method='post',
+                params={'notify': 'true'},
+                json_data={
+                    'days_to_expire': post_data['days_to_expire'],
+                    'recipient': {'type': 'external_user', 'email': email},
+                    'access': {'role': 'VIEWER'},
+                },
+            )
+            data.append(share_info)
+        return {'data': data}
+
+    @endpoint(
+        name="delete-folder",
+        perm='can_access',
+        methods=['post'],
+        description=_('Delete a folder'),
+        post={'request_body': {'schema': {'application/json': schemas.DELETE_FOLDER}}},
+    )
+    def delete_folder(self, request, post_data):
+        delete_infos = self._call('rest/folders/%s' % post_data['folder_id'], method='delete')
+        return {'data': delete_infos}

passerelle/apps/filr_rest/schemas.py

@@ -0,0 +1,91 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2023  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.utils.translation import gettext_lazy as _
+
+UPLOAD = {
+    'type': 'object',
+    'title': _('Upload file'),
+    'properties': {
+        'file': {
+            'title': _('File object'),
+            'type': 'object',
+            'properties': {
+                'filename': {
+                    'type': 'string',
+                    'description': _('Filename'),
+                },
+                'content': {
+                    'type': 'string',
+                    'description': _('Content'),
+                },
+                'content_type': {
+                    'type': 'string',
+                    'description': _('Content type'),
+                },
+            },
+            'required': ['content', 'filename'],
+        },
+        'filename': {
+            'type': 'string',
+            'description': _('Filename (takes precedence over filename in "file" object)'),
+        },
+        'folder_name': {
+            'type': 'string',
+            'description': _('Folder name'),
+        },
+        'root_folder_id': {
+            'type': 'string',
+            'description': _('Root folder identifier'),
+            'pattern': '^[0-9]+$',
+        },
+    },
+    'required': ['file', 'folder_name', 'root_folder_id'],
+}
+
+
+SHARE_FOLDER = {
+    'type': 'object',
+    'title': _('Share Folder'),
+    'properties': {
+        'folder_id': {
+            'type': 'string',
+            'description': _('Folder identifier'),
+            'pattern': '^[0-9]+$',
+        },
+        'emails': {
+            'type': 'array',
+            'description': _('Emails'),
+            'items': {
+                'type': 'string',
+                'format': 'email',
+            },
+        },
+        'days_to_expire': {'type': 'string', 'pattern': '^[0-9]+$'},
+    },
+    'required': ['folder_id', 'emails', 'days_to_expire'],
+    'unflatten': True,
+}
+
+
+DELETE_FOLDER = {
+    'type': 'object',
+    'title': _('Delete Folder'),
+    'properties': {
+        'folder_id': {'type': 'string', 'description': _('Folder identifier'), 'pattern': '^[0-9]+$'}
+    },
+    'required': ['folder_id'],
+}

passerelle/settings.py

@@ -148,6 +148,7 @@ INSTALLED_APPS = (
     'passerelle.apps.esirius',
     'passerelle.apps.family',
     'passerelle.apps.feeds',
+    'passerelle.apps.filr_rest',
     'passerelle.apps.franceconnect_data',
     'passerelle.apps.gdc',
     'passerelle.apps.gesbac',

tests/test_filr_rest.py

@@ -0,0 +1,105 @@
+import base64
+
+import pytest
+import responses
+from django.contrib.contenttypes.models import ContentType
+
+from passerelle.apps.filr_rest.models import Filr
+from passerelle.base.models import AccessRight, ApiUser
+
+
+@pytest.fixture()
+def connector(db):
+    api = ApiUser.objects.create(username='all', keytype='', key='')
+    connector = Filr.objects.create(
+        base_url='http://filr.invalid/',
+        basic_auth_username='foo',
+        basic_auth_password='bar',
+        slug='filr',
+    )
+    obj_type = ContentType.objects.get_for_model(connector)
+    AccessRight.objects.create(
+        codename='can_access', apiuser=api, resource_type=obj_type, resource_pk=connector.pk
+    )
+    return connector
+
+
+def test_upload(app, connector):
+    params = {
+        'file': {
+            'filename': 'bla',
+            'content': base64.b64encode(b'who what').decode(),
+            'content_type': 'text/plain',
+        },
+        'root_folder_id': '1234',
+        'folder_name': 'folder_foo',
+    }
+    with responses.RequestsMock() as rsps:
+        rsps.get(
+            'http://filr.invalid/rest/folders/1234/library_folders',
+            status=200,
+            json={'items': [{'title': 'folder_foo', 'id': 5678}]},
+        )
+        rsps.post(
+            'http://filr.invalid/rest/folders/5678/library_files?file_name=bla',
+            status=200,
+            json={'id': '09c1c3fb530f562401531018f4270000'},
+        )
+        resp = app.post_json('/filr-rest/filr/upload', params=params)
+        json_resp = resp.json
+        assert json_resp['err'] == 0
+        assert json_resp['data'] == {
+            'folder_id': 5678,
+            'file_info': {'id': '09c1c3fb530f562401531018f4270000'},
+        }
+
+
+def test_upload_with_folder_creation(app, connector):
+    params = {
+        'file': {
+            'filename': 'bla',
+            'content': base64.b64encode(b'who what').decode(),
+            'content_type': 'text/plain',
+        },
+        'root_folder_id': '1234',
+        'folder_name': 'folder_foo',
+    }
+    with responses.RequestsMock() as rsps:
+        rsps.get('http://filr.invalid/rest/folders/1234/library_folders', status=200, json={'items': []})
+        rsps.post('http://filr.invalid/rest/folders/1234/library_folders', status=200, json={'id': 82})
+        rsps.post(
+            'http://filr.invalid/rest/folders/82/library_files?file_name=bla',
+            status=200,
+            json={'id': '09c1c3fb530f562401531018f4270000'},
+        )
+        resp = app.post_json('/filr-rest/filr/upload', params=params)
+        json_resp = resp.json
+        assert json_resp['err'] == 0
+        assert json_resp['data'] == {'folder_id': 82, 'file_info': {'id': '09c1c3fb530f562401531018f4270000'}}
+
+
+def test_share_folder(app, connector):
+    params = {
+        'folder_id': '1234',
+        'emails/0': 'foo@invalid',
+        'emails/1': 'bar@invalid',
+        'days_to_expire': '30',
+    }
+    with responses.RequestsMock() as rsps:
+        rsps.post('http://filr.invalid/rest/folders/1234/shares?notify=true', status=200, json={'id': 9})
+        resp = app.post_json('/filr-rest/filr/share-folder', params=params)
+        json_resp = resp.json
+        assert json_resp['err'] == 0
+        assert json_resp['data'] == [{'id': 9}, {'id': 9}]
+
+
+def test_delete_folder(app, connector):
+    params = {
+        'folder_id': '1234',
+    }
+    with responses.RequestsMock() as rsps:
+        rsps.delete('http://filr.invalid/rest/folders/1234', status=204, content_type='application/json')
+        resp = app.post_json('/filr-rest/filr/delete-folder', params=params)
+        json_resp = resp.json
+        assert json_resp['err'] == 0
+        assert json_resp['data'] == ''