cmis: add a file watch mechanism (#73466)
This commit is contained in:
parent
51d1a1eb0e
commit
9bf91ae7a5
|
@ -0,0 +1,40 @@
|
|||
# Generated by Django 2.2.26 on 2023-01-17 17:21
|
||||
|
||||
import django.contrib.postgres.fields.jsonb
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
import passerelle.utils.jsonresponse
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('cmis', '0003_auto_20181118_0807'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='ObjectWatch',
|
||||
fields=[
|
||||
(
|
||||
'id',
|
||||
models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
|
||||
),
|
||||
('object_id', models.CharField(db_index=True, max_length=128)),
|
||||
('callback_url', models.URLField(db_index=True, max_length=400, verbose_name='Callback URL')),
|
||||
(
|
||||
'metadata',
|
||||
django.contrib.postgres.fields.jsonb.JSONField(
|
||||
encoder=passerelle.utils.jsonresponse.JSONEncoder
|
||||
),
|
||||
),
|
||||
('created', models.DateTimeField(auto_now_add=True)),
|
||||
('cancelled', models.DateTimeField(null=True)),
|
||||
(
|
||||
'resource',
|
||||
models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cmis.CmisConnector'),
|
||||
),
|
||||
],
|
||||
),
|
||||
]
|
|
@ -17,12 +17,14 @@
|
|||
import base64
|
||||
import binascii
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from io import BytesIO
|
||||
from urllib import error as urllib2
|
||||
|
||||
import httplib2
|
||||
import requests
|
||||
from cmislib import CmisClient
|
||||
from cmislib.exceptions import (
|
||||
CmisException,
|
||||
|
@ -31,14 +33,16 @@ from cmislib.exceptions import (
|
|||
PermissionDeniedException,
|
||||
UpdateConflictException,
|
||||
)
|
||||
from django.contrib.postgres.fields import JSONField
|
||||
from django.db import models
|
||||
from django.http import HttpResponse
|
||||
from django.utils import timezone
|
||||
from django.utils.functional import cached_property
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from passerelle.base.models import BaseResource
|
||||
from passerelle.utils.api import endpoint
|
||||
from passerelle.utils.jsonresponse import APIError
|
||||
from passerelle.utils.jsonresponse import APIError, JSONEncoder
|
||||
from passerelle.utils.logging import ignore_loggers
|
||||
|
||||
SPECIAL_CHARS = '!#$%&+-^_`~;[]{}+=~'
|
||||
|
@ -100,6 +104,32 @@ UPLOAD_SCHEMA = {
|
|||
}
|
||||
|
||||
|
||||
WATCH_SCHEMA = {
|
||||
'type': 'object',
|
||||
'title': _('Watch object'),
|
||||
'properties': {
|
||||
'object_id': {
|
||||
'type': 'string',
|
||||
'description': _('Object ID of file (can also be a path)'),
|
||||
},
|
||||
'callback_url': {'type': 'string', 'pattern': '^https?://'},
|
||||
},
|
||||
'required': ['object_id', 'callback_url'],
|
||||
}
|
||||
|
||||
CHECK_OBJECT_SCHEMA = {
|
||||
'type': 'object',
|
||||
'title': _('Check object'),
|
||||
'properties': {
|
||||
'object_id': {
|
||||
'type': 'string',
|
||||
'description': _('Object ID of file (can also be a path)'),
|
||||
},
|
||||
},
|
||||
'required': ['object_id'],
|
||||
}
|
||||
|
||||
|
||||
class CmisConnector(BaseResource):
|
||||
cmis_endpoint = models.URLField(
|
||||
max_length=400, verbose_name=_('CMIS Atom endpoint'), help_text=_('URL of the CMIS Atom endpoint')
|
||||
|
@ -215,6 +245,107 @@ class CmisConnector(BaseResource):
|
|||
def getmetadata(self, request, object_id):
|
||||
return {'data': self._get_metadata(object_id)}
|
||||
|
||||
@endpoint(
|
||||
description=_('Watch object'),
|
||||
perm='can_access',
|
||||
post={
|
||||
'request_body': {
|
||||
'schema': {
|
||||
'application/json': WATCH_SCHEMA,
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
def watch(self, request, post_data):
|
||||
object_id = post_data['object_id']
|
||||
metadata = self._get_metadata(object_id)
|
||||
callback_url = post_data['callback_url']
|
||||
if ObjectWatch.objects.filter(
|
||||
resource=self, object_id=object_id, callback_url=callback_url, cancelled__isnull=False
|
||||
).exists():
|
||||
raise APIError('This file is already watched')
|
||||
object_watch = ObjectWatch.objects.create(
|
||||
resource=self, object_id=object_id, metadata=metadata, callback_url=callback_url
|
||||
)
|
||||
return {
|
||||
'data': {
|
||||
'object_id': object_id,
|
||||
'watch_id': object_watch.id,
|
||||
'metadata': metadata,
|
||||
'callback_url': callback_url,
|
||||
}
|
||||
}
|
||||
|
||||
@endpoint(
|
||||
description=_('Check object'),
|
||||
name='check-object',
|
||||
perm='can_access',
|
||||
post={
|
||||
'request_body': {
|
||||
'schema': {
|
||||
'application/json': CHECK_OBJECT_SCHEMA,
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
def check_object(self, request, post_data):
|
||||
return {'data': self._check_objects(post_data['object_id'])}
|
||||
|
||||
def _check_objects(self, object_id=None):
|
||||
res = []
|
||||
qs = ObjectWatch.objects.filter(cancelled__isnull=True)
|
||||
if object_id:
|
||||
qs = qs.filter(object_id=object_id)
|
||||
for object_watch in qs:
|
||||
obj_res = {'object_id': object_watch.object_id, 'changed': True, 'callback_err': 0}
|
||||
res.append(obj_res)
|
||||
metadata = self._get_metadata(object_watch.object_id)
|
||||
# simulate a back and forth to database, to get datetimes as strings
|
||||
metadata = json.loads(JSONEncoder().encode(metadata))
|
||||
|
||||
# try to compare on lastModificationDate
|
||||
new_mod_date = metadata.get('cmis', {}).get('lastModificationDate', '')
|
||||
old_mod_date = object_watch.metadata.get('cmis', {}).get('lastModificationDate', '')
|
||||
if new_mod_date and old_mod_date:
|
||||
if new_mod_date == old_mod_date:
|
||||
obj_res['changed'] = False
|
||||
continue
|
||||
# fallback on all the metadata
|
||||
elif metadata and object_watch.metadata:
|
||||
if metadata == object_watch.metadata:
|
||||
obj_res['changed'] = False
|
||||
continue
|
||||
|
||||
# metadata changed, call the callback url
|
||||
try:
|
||||
resp = self.requests.post(object_watch.callback_url, json=metadata)
|
||||
except (requests.Timeout, requests.RequestException) as e:
|
||||
obj_res['callback_err'] = 1
|
||||
obj_res['callback_err_desc'] = str(e)
|
||||
else:
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
obj_res['callback_err'] = 1
|
||||
obj_res['callback_err_desc'] = str(e)
|
||||
else:
|
||||
object_watch.cancelled = timezone.now()
|
||||
object_watch.save()
|
||||
|
||||
return res
|
||||
|
||||
def daily(self):
|
||||
self._check_objects()
|
||||
|
||||
|
||||
class ObjectWatch(models.Model):
|
||||
resource = models.ForeignKey(CmisConnector, on_delete=models.CASCADE)
|
||||
object_id = models.CharField(max_length=128, db_index=True)
|
||||
callback_url = models.URLField(max_length=400, verbose_name=_('Callback URL'), db_index=True)
|
||||
metadata = JSONField(encoder=JSONEncoder)
|
||||
created = models.DateTimeField(auto_now_add=True)
|
||||
cancelled = models.DateTimeField(null=True)
|
||||
|
||||
|
||||
def wrap_cmis_error(f):
|
||||
@functools.wraps(f)
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<atom:entry xmlns:atom="http://www.w3.org/2005/Atom" xmlns:cmis="http://docs.oasis-open.org/ns/cmis/core/200908/" xmlns:cmisra="http://docs.oasis-open.org/ns/cmis/restatom/200908/" xmlns:app="http://www.w3.org/2007/app">
|
||||
<atom:author>
|
||||
<atom:name>cmisuser</atom:name>
|
||||
</atom:author>
|
||||
<atom:id>http://chemistry.apache.org/TDNSbGMzUXRaVzh2ZEdWemREST0=</atom:id>
|
||||
<atom:published>2020-02-08T19:17:10Z</atom:published>
|
||||
<atom:title>test2</atom:title>
|
||||
<app:edited>2020-02-08T19:17:10Z</app:edited>
|
||||
<atom:updated>2020-02-08T19:17:10Z</atom:updated>
|
||||
<atom:content src="http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/octet-stream"/>
|
||||
<cmisra:object>
|
||||
<cmis:properties>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:objectId">
|
||||
<cmis:value>L3Rlc3QtZW8vdGVzdDI=</cmis:value>
|
||||
</cmis:propertyId>
|
||||
<cmis:propertyDateTime propertyDefinitionId="cmis:lastModificationDate">
|
||||
<cmis:value>2020-02-09T19:17:10Z</cmis:value>
|
||||
</cmis:propertyDateTime>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:changeToken"/>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:description"/>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:secondaryObjectTypeIds"/>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:baseTypeId">
|
||||
<cmis:value>cmis:document</cmis:value>
|
||||
</cmis:propertyId>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isImmutable">
|
||||
<cmis:value>false</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isLatestVersion">
|
||||
<cmis:value>true</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isMajorVersion">
|
||||
<cmis:value>true</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isLatestMajorVersion">
|
||||
<cmis:value>true</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:versionLabel">
|
||||
<cmis:value>test2</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:versionSeriesId">
|
||||
<cmis:value>L3Rlc3QtZW8vdGVzdDI=</cmis:value>
|
||||
</cmis:propertyId>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isVersionSeriesCheckedOut">
|
||||
<cmis:value>false</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:versionSeriesCheckedOutBy"/>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:versionSeriesCheckedOutId"/>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:checkinComment">
|
||||
<cmis:value/>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyBoolean propertyDefinitionId="cmis:isPrivateWorkingCopy">
|
||||
<cmis:value>false</cmis:value>
|
||||
</cmis:propertyBoolean>
|
||||
<cmis:propertyInteger propertyDefinitionId="cmis:contentStreamLength">
|
||||
<cmis:value>6</cmis:value>
|
||||
</cmis:propertyInteger>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:contentStreamMimeType">
|
||||
<cmis:value>application/octet-stream</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:contentStreamFileName">
|
||||
<cmis:value>test2</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:contentStreamId"/>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:name">
|
||||
<cmis:value>test2</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyId propertyDefinitionId="cmis:objectTypeId">
|
||||
<cmis:value>cmis:document</cmis:value>
|
||||
</cmis:propertyId>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:createdBy">
|
||||
<cmis:value>cmisuser</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyString propertyDefinitionId="cmis:lastModifiedBy">
|
||||
<cmis:value>cmisuser</cmis:value>
|
||||
</cmis:propertyString>
|
||||
<cmis:propertyDateTime propertyDefinitionId="cmis:creationDate">
|
||||
<cmis:value>2020-02-08T19:17:10Z</cmis:value>
|
||||
</cmis:propertyDateTime>
|
||||
<cmis:propertyString propertyDefinitionId="rsj:idInsertis">
|
||||
<cmis:value>21N284563</cmis:value>
|
||||
</cmis:propertyString>
|
||||
</cmis:properties>
|
||||
</cmisra:object>
|
||||
<atom:link rel="service" href="http://example.com/cmisatom/test?repositoryId=test" type="application/atomsvc+xml"/>
|
||||
<atom:link rel="self" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry" cmisra:id="L3Rlc3QtZW8vdGVzdDI="/>
|
||||
<atom:link rel="enclosure" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry"/>
|
||||
<atom:link rel="edit" href="http://example.com/cmisatom/test/entry?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=entry"/>
|
||||
<atom:link rel="describedby" href="http://example.com/cmisatom/test/type?id=cmis%3Adocument" type="application/atom+xml;type=entry"/>
|
||||
<atom:link rel="http://docs.oasis-open.org/ns/cmis/link/200908/allowableactions" href="http://example.com/cmisatom/test/allowableactions?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/cmisallowableactions+xml"/>
|
||||
<atom:link rel="up" href="http://example.com/cmisatom/test/parents?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/atom+xml;type=feed"/>
|
||||
<atom:link rel="edit-media" href="http://example.com/cmisatom/test/content?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/octet-stream"/>
|
||||
<atom:link rel="http://docs.oasis-open.org/ns/cmis/link/200908/acl" href="http://example.com/cmisatom/test/acl?id=L3Rlc3QtZW8vdGVzdDI%3D" type="application/cmisacl+xml"/>
|
||||
</atom:entry>
|
|
@ -9,6 +9,7 @@ from urllib import error as urllib2
|
|||
import httplib2
|
||||
import py
|
||||
import pytest
|
||||
import responses
|
||||
from cmislib import CmisClient
|
||||
from cmislib.exceptions import (
|
||||
CmisException,
|
||||
|
@ -21,7 +22,7 @@ from django.contrib.contenttypes.models import ContentType
|
|||
from django.urls import reverse
|
||||
from django.utils.encoding import force_bytes, force_str
|
||||
|
||||
from passerelle.apps.cmis.models import CmisConnector
|
||||
from passerelle.apps.cmis.models import CmisConnector, ObjectWatch
|
||||
from passerelle.base.models import AccessRight, ApiUser, ResourceLog
|
||||
from tests.test_manager import login
|
||||
|
||||
|
@ -680,3 +681,97 @@ def test_get_metadata(mocked_request, app, setup):
|
|||
response = app.get(url, params={'object_id': '/test/file'})
|
||||
assert response.json['data']['cmis']['contentStreamFileName'] == 'test2'
|
||||
assert response.json['data']['rsj']['idInsertis'] == '21N284563'
|
||||
|
||||
|
||||
@mock.patch('httplib2.Http.request')
|
||||
def test_watch(mocked_request, app, setup):
|
||||
def cmis_mocked_request(uri, method="GET", body=None, **kwargs):
|
||||
"""simulate the HTTP queries involved"""
|
||||
response = {'status': '200'}
|
||||
if method == 'GET' and uri == 'http://example.com/cmisatom':
|
||||
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
|
||||
content = fd.read()
|
||||
elif method == 'GET' and (
|
||||
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
|
||||
or uri.startswith(
|
||||
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
|
||||
)
|
||||
):
|
||||
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
|
||||
content = fd.read()
|
||||
|
||||
else:
|
||||
raise Exception('url is not yet mocked: %s' % uri)
|
||||
return (response, content)
|
||||
|
||||
mocked_request.side_effect = cmis_mocked_request
|
||||
|
||||
resp = app.post_json(
|
||||
'/cmis/slug-cmis/watch',
|
||||
params={
|
||||
'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0',
|
||||
'callback_url': 'https://foo.invalid/trigger',
|
||||
},
|
||||
)
|
||||
assert resp.json['err'] == 0
|
||||
data = resp.json['data']
|
||||
# check that ObjectWatch is created
|
||||
object_watch = ObjectWatch.objects.get(
|
||||
resource=setup,
|
||||
object_id='c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0',
|
||||
callback_url='https://foo.invalid/trigger',
|
||||
cancelled=None,
|
||||
)
|
||||
assert data['watch_id'] == object_watch.id
|
||||
|
||||
# check ObjectWatch metadata
|
||||
metadata_resp = app.get(
|
||||
'/cmis/slug-cmis/getmetadata', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
|
||||
)
|
||||
assert object_watch.metadata == metadata_resp.json['data']
|
||||
|
||||
# check object, nothing changed
|
||||
metadata_resp = app.post_json(
|
||||
'/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
|
||||
)
|
||||
assert metadata_resp.json['err'] == 0
|
||||
assert metadata_resp.json['data'] == [
|
||||
{'callback_err': 0, 'changed': False, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
|
||||
]
|
||||
|
||||
# new mock to pretend that the file changed
|
||||
def new_cmis_mocked_request(uri, method="GET", body=None, **kwargs):
|
||||
"""simulate the HTTP queries involved"""
|
||||
response = {'status': '200'}
|
||||
if method == 'GET' and uri == 'http://example.com/cmisatom':
|
||||
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
|
||||
content = fd.read()
|
||||
elif method == 'GET' and (
|
||||
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
|
||||
or uri.startswith(
|
||||
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
|
||||
)
|
||||
):
|
||||
with open('%s/tests/data/cmis/cmis4.out.xml' % os.getcwd(), 'rb') as fd:
|
||||
content = fd.read()
|
||||
elif method == 'POST' and uri.startswith('https://foo.invalid/trigger'):
|
||||
content = ''
|
||||
else:
|
||||
raise Exception('url is not yet mocked: %s' % uri)
|
||||
return (response, content)
|
||||
|
||||
mocked_request.side_effect = new_cmis_mocked_request
|
||||
|
||||
with responses.RequestsMock() as rsps:
|
||||
rsps.post(
|
||||
'https://foo.invalid/trigger',
|
||||
json={},
|
||||
)
|
||||
# check object
|
||||
metadata_resp = app.post_json(
|
||||
'/cmis/slug-cmis/check-object', params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
|
||||
)
|
||||
assert metadata_resp.json['err'] == 0
|
||||
assert metadata_resp.json['data'] == [
|
||||
{'callback_err': 0, 'changed': True, 'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'}
|
||||
]
|
||||
|
|
Loading…
Reference in New Issue