diff --git a/passerelle/apps/cmis/migrations/0004_objectwatch.py b/passerelle/apps/cmis/migrations/0004_objectwatch.py new file mode 100644 index 00000000..aec3a054 --- /dev/null +++ b/passerelle/apps/cmis/migrations/0004_objectwatch.py @@ -0,0 +1,40 @@ +# Generated by Django 2.2.26 on 2023-01-17 17:21 + +import django.contrib.postgres.fields.jsonb +import django.db.models.deletion +from django.db import migrations, models + +import passerelle.utils.jsonresponse + + +class Migration(migrations.Migration): + + dependencies = [ + ('cmis', '0003_auto_20181118_0807'), + ] + + operations = [ + migrations.CreateModel( + name='ObjectWatch', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('object_id', models.CharField(db_index=True, max_length=128)), + ('callback_url', models.URLField(db_index=True, max_length=400, verbose_name='Callback URL')), + ( + 'metadata', + django.contrib.postgres.fields.jsonb.JSONField( + encoder=passerelle.utils.jsonresponse.JSONEncoder + ), + ), + ('created', models.DateTimeField(auto_now_add=True)), + ('cancelled', models.DateTimeField(null=True)), + ( + 'resource', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cmis.CmisConnector'), + ), + ], + ), + ] diff --git a/passerelle/apps/cmis/models.py b/passerelle/apps/cmis/models.py index 4373186b..fc2e9bb0 100644 --- a/passerelle/apps/cmis/models.py +++ b/passerelle/apps/cmis/models.py @@ -17,12 +17,14 @@ import base64 import binascii import functools +import json import re from contextlib import contextmanager from io import BytesIO from urllib import error as urllib2 import httplib2 +import requests from cmislib import CmisClient from cmislib.exceptions import ( CmisException, @@ -31,14 +33,16 @@ from cmislib.exceptions import ( PermissionDeniedException, UpdateConflictException, ) +from django.contrib.postgres.fields import JSONField from django.db import models from django.http import HttpResponse +from django.utils import timezone from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ from passerelle.base.models import BaseResource from passerelle.utils.api import endpoint -from passerelle.utils.jsonresponse import APIError +from passerelle.utils.jsonresponse import APIError, JSONEncoder from passerelle.utils.logging import ignore_loggers SPECIAL_CHARS = '!#$%&+-^_`~;[]{}+=~' @@ -100,6 +104,32 @@ UPLOAD_SCHEMA = { } +WATCH_SCHEMA = { + 'type': 'object', + 'title': _('Watch object'), + 'properties': { + 'object_id': { + 'type': 'string', + 'description': _('Object ID of file (can also be a path)'), + }, + 'callback_url': {'type': 'string', 'pattern': '^https?://'}, + }, + 'required': ['object_id', 'callback_url'], +} + +CHECK_OBJECT_SCHEMA = { + 'type': 'object', + 'title': _('Check object'), + 'properties': { + 'object_id': { + 'type': 'string', + 'description': _('Object ID of file (can also be a path)'), + }, + }, + 'required': ['object_id'], +} + + class CmisConnector(BaseResource): cmis_endpoint = models.URLField( max_length=400, verbose_name=_('CMIS Atom endpoint'), help_text=_('URL of the CMIS Atom endpoint') @@ -215,6 +245,107 @@ class CmisConnector(BaseResource): def getmetadata(self, request, object_id): return {'data': self._get_metadata(object_id)} + @endpoint( + description=_('Watch object'), + perm='can_access', + post={ + 'request_body': { + 'schema': { + 'application/json': WATCH_SCHEMA, + } + } + }, + ) + def watch(self, request, post_data): + object_id = post_data['object_id'] + metadata = self._get_metadata(object_id) + callback_url = post_data['callback_url'] + if ObjectWatch.objects.filter( + resource=self, object_id=object_id, callback_url=callback_url, cancelled__isnull=False + ).exists(): + raise APIError('This file is already watched') + object_watch = ObjectWatch.objects.create( + resource=self, object_id=object_id, metadata=metadata, callback_url=callback_url + ) + return { + 'data': { + 'object_id': object_id, + 'watch_id': object_watch.id, + 'metadata': metadata, + 'callback_url': callback_url, + } + } + + @endpoint( + description=_('Check object'), + name='check-object', + perm='can_access', + post={ + 'request_body': { + 'schema': { + 'application/json': CHECK_OBJECT_SCHEMA, + } + } + }, + ) + def check_object(self, request, post_data): + return {'data': self._check_objects(post_data['object_id'])} + + def _check_objects(self, object_id=None): + res = [] + qs = ObjectWatch.objects.filter(cancelled__isnull=True) + if object_id: + qs = qs.filter(object_id=object_id) + for object_watch in qs: + obj_res = {'object_id': object_watch.object_id, 'changed': True, 'callback_err': 0} + res.append(obj_res) + metadata = self._get_metadata(object_watch.object_id) + # simulate a back and forth to database, to get datetimes as strings + metadata = json.loads(JSONEncoder().encode(metadata)) + + # try to compare on lastModificationDate + new_mod_date = metadata.get('cmis', {}).get('lastModificationDate', '') + old_mod_date = object_watch.metadata.get('cmis', {}).get('lastModificationDate', '') + if new_mod_date and old_mod_date: + if new_mod_date == old_mod_date: + obj_res['changed'] = False + continue + # fallback on all the metadata + elif metadata and object_watch.metadata: + if metadata == object_watch.metadata: + obj_res['changed'] = False + continue + + # metadata changed, call the callback url + try: + resp = self.requests.post(object_watch.callback_url, json=metadata) + except (requests.Timeout, requests.RequestException) as e: + obj_res['callback_err'] = 1 + obj_res['callback_err_desc'] = str(e) + else: + try: + resp.raise_for_status() + except requests.RequestException as e: + obj_res['callback_err'] = 1 + obj_res['callback_err_desc'] = str(e) + else: + object_watch.cancelled = timezone.now() + object_watch.save() + + return res + + def daily(self): + self._check_objects() + + +class ObjectWatch(models.Model): + resource = models.ForeignKey(CmisConnector, on_delete=models.CASCADE) + object_id = models.CharField(max_length=128, db_index=True) + callback_url = models.URLField(max_length=400, verbose_name=_('Callback URL'), db_index=True) + metadata = JSONField(encoder=JSONEncoder) + created = models.DateTimeField(auto_now_add=True) + cancelled = models.DateTimeField(null=True) + def wrap_cmis_error(f): @functools.wraps(f) diff --git a/tests/data/cmis/cmis4.out.xml b/tests/data/cmis/cmis4.out.xml new file mode 100644 index 00000000..a1e9247f --- /dev/null +++ b/tests/data/cmis/cmis4.out.xml @@ -0,0 +1,94 @@ + + + + cmisuser + + http://chemistry.apache.org/TDNSbGMzUXRaVzh2ZEdWemREST0= + 2020-02-08T19:17:10Z + test2 + 2020-02-08T19:17:10Z + 2020-02-08T19:17:10Z + + + + + L3Rlc3QtZW8vdGVzdDI= + + + 2020-02-09T19:17:10Z + + + + + + cmis:document + + + false + + + true + + + true + + + true + + + test2 + + + L3Rlc3QtZW8vdGVzdDI= + + + false + + + + + + + + false + + + 6 + + + application/octet-stream + + + test2 + + + + test2 + + + cmis:document + + + cmisuser + + + cmisuser + + + 2020-02-08T19:17:10Z + + + 21N284563 + + + + + + + + + + + + + diff --git a/tests/test_cmis.py b/tests/test_cmis.py index 03420e64..d8e5b880 100644 --- a/tests/test_cmis.py +++ b/tests/test_cmis.py @@ -9,6 +9,7 @@ from urllib import error as urllib2 import httplib2 import py import pytest +import responses from cmislib import CmisClient from cmislib.exceptions import ( CmisException, @@ -21,7 +22,7 @@ from django.contrib.contenttypes.models import ContentType from django.urls import reverse from django.utils.encoding import force_bytes, force_str -from passerelle.apps.cmis.models import CmisConnector +from passerelle.apps.cmis.models import CmisConnector, ObjectWatch from passerelle.base.models import AccessRight, ApiUser, ResourceLog from tests.test_manager import login @@ -680,3 +681,97 @@ def test_get_metadata(mocked_request, app, setup): response = app.get(url, params={'object_id': '/test/file'}) assert response.json['data']['cmis']['contentStreamFileName'] == 'test2' assert response.json['data']['rsj']['idInsertis'] == '21N284563' + + +@mock.patch('httplib2.Http.request') +def test_watch(mocked_request, app, setup): + def cmis_mocked_request(uri, method="GET", body=None, **kwargs): + """simulate the HTTP queries involved""" + response = {'status': '200'} + if method == 'GET' and uri == 'http://example.com/cmisatom': + with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd: + content = fd.read() + elif method == 'GET' and ( + uri.startswith('http://example.com/cmisatom/test/path?path=/test/file') + or uri.startswith( + 'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0' + ) + ): + with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd: + content = fd.read() + + else: + raise Exception('url is not yet mocked: %s' % uri) + return (response, content) + + mocked_request.side_effect = cmis_mocked_request + + resp = app.post_json( + '/cmis/slug-cmis/watch', + params={ + 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0', + 'callback_url': 'https://foo.invalid/trigger', + }, + ) + assert resp.json['err'] == 0 + data = resp.json['data'] + # check that ObjectWatch is created + object_watch = ObjectWatch.objects.get( + resource=setup, + object_id='c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0', + callback_url='https://foo.invalid/trigger', + cancelled=None, + ) + assert data['watch_id'] == object_watch.id + + # check ObjectWatch metadata + metadata_resp = app.get( + '/cmis/slug-cmis/getmetadata', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'} + ) + assert object_watch.metadata == metadata_resp.json['data'] + + # check object, nothing changed + metadata_resp = app.post_json( + '/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'} + ) + assert metadata_resp.json['err'] == 0 + assert metadata_resp.json['data'] == [ + {'callback_err': 0, 'changed': False, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'} + ] + + # new mock to pretend that the file changed + def new_cmis_mocked_request(uri, method="GET", body=None, **kwargs): + """simulate the HTTP queries involved""" + response = {'status': '200'} + if method == 'GET' and uri == 'http://example.com/cmisatom': + with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd: + content = fd.read() + elif method == 'GET' and ( + uri.startswith('http://example.com/cmisatom/test/path?path=/test/file') + or uri.startswith( + 'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0' + ) + ): + with open('%s/tests/data/cmis/cmis4.out.xml' % os.getcwd(), 'rb') as fd: + content = fd.read() + elif method == 'POST' and uri.startswith('https://foo.invalid/trigger'): + content = '' + else: + raise Exception('url is not yet mocked: %s' % uri) + return (response, content) + + mocked_request.side_effect = new_cmis_mocked_request + + with responses.RequestsMock() as rsps: + rsps.post( + 'https://foo.invalid/trigger', + json={}, + ) + # check object + metadata_resp = app.post_json( + '/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'} + ) + assert metadata_resp.json['err'] == 0 + assert metadata_resp.json['data'] == [ + {'callback_err': 0, 'changed': True, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'} + ]