entrouvert
/
passerelle
15
0
Fork 0
Code Issues Pull Requests 14 Releases Activity

cmis: add a file watch mechanism (#73466)
gitea-wip/passerelle/pipeline/pr-main This commit looks good Details
gitea/passerelle/pipeline/pr-main Something is wrong with the build of this commit Details
gitea/passerelle/pipeline/head There was a failure building this commit Details

This commit is contained in:
Emmanuel Cazenave 2023-01-17 11:18:01 +01:00
parent 51d1a1eb0e
commit 9bf91ae7a5
4 changed files with 362 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
passerelle/apps/cmis/migrations/0004_objectwatch.py Normal file
View File

@ -0,0 +1,40 @@
# Generated by Django 2.2.26 on 2023-01-17 17:21
import django.contrib.postgres.fields.jsonb
import django.db.models.deletion
from django.db import migrations, models
import passerelle.utils.jsonresponse
class Migration(migrations.Migration):
dependencies = [
('cmis', '0003_auto_20181118_0807'),
]
operations = [
migrations.CreateModel(
name='ObjectWatch',
fields=[
(
'id',
models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
),
('object_id', models.CharField(db_index=True, max_length=128)),
('callback_url', models.URLField(db_index=True, max_length=400, verbose_name='Callback URL')),
(
'metadata',
django.contrib.postgres.fields.jsonb.JSONField(
encoder=passerelle.utils.jsonresponse.JSONEncoder
),
),
('created', models.DateTimeField(auto_now_add=True)),
('cancelled', models.DateTimeField(null=True)),
(
'resource',
models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cmis.CmisConnector'),
),
],
),
]

133
passerelle/apps/cmis/models.py
View File

@ -17,12 +17,14 @@
import base64
import binascii
import functools
import json
import re
from contextlib import contextmanager
from io import BytesIO
from urllib import error as urllib2
import httplib2
import requests
from cmislib import CmisClient
from cmislib.exceptions import (
CmisException,
@ -31,14 +33,16 @@ from cmislib.exceptions import (
PermissionDeniedException,
UpdateConflictException,
)
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.http import HttpResponse
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import gettext_lazy as _
from passerelle.base.models import BaseResource
from passerelle.utils.api import endpoint
from passerelle.utils.jsonresponse import APIError
from passerelle.utils.jsonresponse import APIError, JSONEncoder
from passerelle.utils.logging import ignore_loggers
SPECIAL_CHARS = '!#$%&+-^_`~;[]{}+=~'
@ -100,6 +104,32 @@ UPLOAD_SCHEMA = {
}
WATCH_SCHEMA = {
'type': 'object',
'title': _('Watch object'),
'properties': {
'object_id': {
'type': 'string',
'description': _('Object ID of file (can also be a path)'),
},
'callback_url': {'type': 'string', 'pattern': '^https?://'},
},
'required': ['object_id', 'callback_url'],
}
CHECK_OBJECT_SCHEMA = {
'type': 'object',
'title': _('Check object'),
'properties': {
'object_id': {
'type': 'string',
'description': _('Object ID of file (can also be a path)'),
},
},
'required': ['object_id'],
}
class CmisConnector(BaseResource):
cmis_endpoint = models.URLField(
max_length=400, verbose_name=_('CMIS Atom endpoint'), help_text=_('URL of the CMIS Atom endpoint')
@ -215,6 +245,107 @@ class CmisConnector(BaseResource):
def getmetadata(self, request, object_id):
return {'data': self._get_metadata(object_id)}
@endpoint(
description=_('Watch object'),
perm='can_access',
post={
'request_body': {
'schema': {
'application/json': WATCH_SCHEMA,
}
}
},
)
def watch(self, request, post_data):
object_id = post_data['object_id']
metadata = self._get_metadata(object_id)
callback_url = post_data['callback_url']
if ObjectWatch.objects.filter(
resource=self, object_id=object_id, callback_url=callback_url, cancelled__isnull=False
).exists():
raise APIError('This file is already watched')
object_watch = ObjectWatch.objects.create(
resource=self, object_id=object_id, metadata=metadata, callback_url=callback_url
)
return {
'data': {
'object_id': object_id,
'watch_id': object_watch.id,
'metadata': metadata,
'callback_url': callback_url,
}
}
@endpoint(
description=_('Check object'),
name='check-object',
perm='can_access',
post={
'request_body': {
'schema': {
'application/json': CHECK_OBJECT_SCHEMA,
}
}
},
)
def check_object(self, request, post_data):
return {'data': self._check_objects(post_data['object_id'])}
def _check_objects(self, object_id=None):
res = []
qs = ObjectWatch.objects.filter(cancelled__isnull=True)
if object_id:
qs = qs.filter(object_id=object_id)
for object_watch in qs:
obj_res = {'object_id': object_watch.object_id, 'changed': True, 'callback_err': 0}
res.append(obj_res)
metadata = self._get_metadata(object_watch.object_id)
# simulate a back and forth to database, to get datetimes as strings
metadata = json.loads(JSONEncoder().encode(metadata))
# try to compare on lastModificationDate
new_mod_date = metadata.get('cmis', {}).get('lastModificationDate', '')
old_mod_date = object_watch.metadata.get('cmis', {}).get('lastModificationDate', '')
if new_mod_date and old_mod_date:
if new_mod_date == old_mod_date:
obj_res['changed'] = False
continue
# fallback on all the metadata
elif metadata and object_watch.metadata:
if metadata == object_watch.metadata:
obj_res['changed'] = False
continue
# metadata changed, call the callback url
try:
resp = self.requests.post(object_watch.callback_url, json=metadata)
except (requests.Timeout, requests.RequestException) as e:
obj_res['callback_err'] = 1
obj_res['callback_err_desc'] = str(e)
else:
try:
resp.raise_for_status()
except requests.RequestException as e:
obj_res['callback_err'] = 1
obj_res['callback_err_desc'] = str(e)
else:
object_watch.cancelled = timezone.now()
object_watch.save()
return res
def daily(self):
self._check_objects()
class ObjectWatch(models.Model):
resource = models.ForeignKey(CmisConnector, on_delete=models.CASCADE)
object_id = models.CharField(max_length=128, db_index=True)
callback_url = models.URLField(max_length=400, verbose_name=_('Callback URL'), db_index=True)
metadata = JSONField(encoder=JSONEncoder)
created = models.DateTimeField(auto_now_add=True)
cancelled = models.DateTimeField(null=True)
def wrap_cmis_error(f):
@functools.wraps(f)

94
tests/data/cmis/cmis4.out.xml Normal file
View File

@ -0,0 +1,94 @@
<?xml version="1.0" encoding="UTF-8"?>
<atom:entry xmlns:atom="http://www.w3.org/2005/Atom" xmlns:cmis="http://docs.oasis-open.org/ns/cmis/core/200908/" xmlns:cmisra="http://docs.oasis-open.org/ns/cmis/restatom/200908/" xmlns:app="http://www.w3.org/2007/app">
<atom:author>
<atom:name>cmisuser</atom:name>
</atom:author>
<atom:id>http://chemistry.apache.org/TDNSbGMzUXRaVzh2ZEdWemREST0=</atom:id>
<atom:published>2020-02-08T19:17:10Z</atom:published>
<atom:title>test2</atom:title>
<app:edited>2020-02-08T19:17:10Z</app:edited>
<atom:updated>2020-02-08T19:17:10Z</atom:updated>
<atom:content src="http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/octet-stream"/>
<cmisra:object>
<cmis:properties>
<cmis:propertyId propertyDefinitionId="cmis:objectId">
<cmis:value>L3Rlc3QtZW8vdGVzdDI=</cmis:value>
</cmis:propertyId>
<cmis:propertyDateTime propertyDefinitionId="cmis:lastModificationDate">
<cmis:value>2020-02-09T19:17:10Z</cmis:value>
</cmis:propertyDateTime>
<cmis:propertyString propertyDefinitionId="cmis:changeToken"/>
<cmis:propertyString propertyDefinitionId="cmis:description"/>
<cmis:propertyId propertyDefinitionId="cmis:secondaryObjectTypeIds"/>
<cmis:propertyId propertyDefinitionId="cmis:baseTypeId">
<cmis:value>cmis:document</cmis:value>
</cmis:propertyId>
<cmis:propertyBoolean propertyDefinitionId="cmis:isImmutable">
<cmis:value>false</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyBoolean propertyDefinitionId="cmis:isLatestVersion">
<cmis:value>true</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyBoolean propertyDefinitionId="cmis:isMajorVersion">
<cmis:value>true</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyBoolean propertyDefinitionId="cmis:isLatestMajorVersion">
<cmis:value>true</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyString propertyDefinitionId="cmis:versionLabel">
<cmis:value>test2</cmis:value>
</cmis:propertyString>
<cmis:propertyId propertyDefinitionId="cmis:versionSeriesId">
<cmis:value>L3Rlc3QtZW8vdGVzdDI=</cmis:value>
</cmis:propertyId>
<cmis:propertyBoolean propertyDefinitionId="cmis:isVersionSeriesCheckedOut">
<cmis:value>false</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyString propertyDefinitionId="cmis:versionSeriesCheckedOutBy"/>
<cmis:propertyString propertyDefinitionId="cmis:versionSeriesCheckedOutId"/>
<cmis:propertyString propertyDefinitionId="cmis:checkinComment">
<cmis:value/>
</cmis:propertyString>
<cmis:propertyBoolean propertyDefinitionId="cmis:isPrivateWorkingCopy">
<cmis:value>false</cmis:value>
</cmis:propertyBoolean>
<cmis:propertyInteger propertyDefinitionId="cmis:contentStreamLength">
<cmis:value>6</cmis:value>
</cmis:propertyInteger>
<cmis:propertyString propertyDefinitionId="cmis:contentStreamMimeType">
<cmis:value>application/octet-stream</cmis:value>
</cmis:propertyString>
<cmis:propertyString propertyDefinitionId="cmis:contentStreamFileName">
<cmis:value>test2</cmis:value>
</cmis:propertyString>
<cmis:propertyId propertyDefinitionId="cmis:contentStreamId"/>
<cmis:propertyString propertyDefinitionId="cmis:name">
<cmis:value>test2</cmis:value>
</cmis:propertyString>
<cmis:propertyId propertyDefinitionId="cmis:objectTypeId">
<cmis:value>cmis:document</cmis:value>
</cmis:propertyId>
<cmis:propertyString propertyDefinitionId="cmis:createdBy">
<cmis:value>cmisuser</cmis:value>
</cmis:propertyString>
<cmis:propertyString propertyDefinitionId="cmis:lastModifiedBy">
<cmis:value>cmisuser</cmis:value>
</cmis:propertyString>
<cmis:propertyDateTime propertyDefinitionId="cmis:creationDate">
<cmis:value>2020-02-08T19:17:10Z</cmis:value>
</cmis:propertyDateTime>
<cmis:propertyString propertyDefinitionId="rsj:idInsertis">
<cmis:value>21N284563</cmis:value>
</cmis:propertyString>
</cmis:properties>
</cmisra:object>
<atom:link rel="service" href="http://example.com/cmisatom/test?repositoryId=test" type="application/atomsvc+xml"/>
<atom:link rel="self" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry" cmisra:id="L3Rlc3QtZW8vdGVzdDI="/>
<atom:link rel="enclosure" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry"/>
<atom:link rel="edit" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry"/>
<atom:link rel="describedby" href="http://example.com/cmisatom/test/type?id=cmis%3Adocument" type="application/atom+xml;type=entry"/>
<atom:link rel="http://docs.oasis-open.org/ns/cmis/link/200908/allowableactions" href="http://example.com/cmisatom/test/allowableactions?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/cmisallowableactions+xml"/>
<atom:link rel="up" href="http://example.com/cmisatom/test/parents?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=feed"/>
<atom:link rel="edit-media" href="http://example.com/cmisatom/test/content?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/octet-stream"/>
<atom:link rel="http://docs.oasis-open.org/ns/cmis/link/200908/acl" href="http://example.com/cmisatom/test/acl?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/cmisacl+xml"/>
</atom:entry>

97
tests/test_cmis.py
View File

@ -9,6 +9,7 @@ from urllib import error as urllib2
import httplib2
import py
import pytest
import responses
from cmislib import CmisClient
from cmislib.exceptions import (
CmisException,
@ -21,7 +22,7 @@ from django.contrib.contenttypes.models import ContentType
from django.urls import reverse
from django.utils.encoding import force_bytes, force_str
from passerelle.apps.cmis.models import CmisConnector
from passerelle.apps.cmis.models import CmisConnector, ObjectWatch
from passerelle.base.models import AccessRight, ApiUser, ResourceLog
from tests.test_manager import login
@ -680,3 +681,97 @@ def test_get_metadata(mocked_request, app, setup):
response = app.get(url, params={'object_id': '/test/file'})
assert response.json['data']['cmis']['contentStreamFileName'] == 'test2'
assert response.json['data']['rsj']['idInsertis'] == '21N284563'
@mock.patch('httplib2.Http.request')
def test_watch(mocked_request, app, setup):
def cmis_mocked_request(uri, method="GET", body=None, **kwargs):
"""simulate the HTTP queries involved"""
response = {'status': '200'}
if method == 'GET' and uri == 'http://example.com/cmisatom':
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'GET' and (
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
or uri.startswith(
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
)
):
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
else:
raise Exception('url is not yet mocked: %s' % uri)
return (response, content)
mocked_request.side_effect = cmis_mocked_request
resp = app.post_json(
'/cmis/slug-cmis/watch',
params={
'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0',
'callback_url': 'https://foo.invalid/trigger',
},
)
assert resp.json['err'] == 0
data = resp.json['data']
# check that ObjectWatch is created
object_watch = ObjectWatch.objects.get(
resource=setup,
object_id='c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0',
callback_url='https://foo.invalid/trigger',
cancelled=None,
)
assert data['watch_id'] == object_watch.id
# check ObjectWatch metadata
metadata_resp = app.get(
'/cmis/slug-cmis/getmetadata', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
)
assert object_watch.metadata == metadata_resp.json['data']
# check object, nothing changed
metadata_resp = app.post_json(
'/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
)
assert metadata_resp.json['err'] == 0
assert metadata_resp.json['data'] == [
{'callback_err': 0, 'changed': False, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
]
# new mock to pretend that the file changed
def new_cmis_mocked_request(uri, method="GET", body=None, **kwargs):
"""simulate the HTTP queries involved"""
response = {'status': '200'}
if method == 'GET' and uri == 'http://example.com/cmisatom':
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'GET' and (
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
or uri.startswith(
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
)
):
with open('%s/tests/data/cmis/cmis4.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'POST' and uri.startswith('https://foo.invalid/trigger'):
content = ''
else:
raise Exception('url is not yet mocked: %s' % uri)
return (response, content)
mocked_request.side_effect = new_cmis_mocked_request
with responses.RequestsMock() as rsps:
rsps.post(
'https://foo.invalid/trigger',
json={},
)
# check object
metadata_resp = app.post_json(
'/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
)
assert metadata_resp.json['err'] == 0
assert metadata_resp.json['data'] == [
{'callback_err': 0, 'changed': True, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
]