entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 12 Releases Wiki Activity

export_import: invalid bundle (#88132)

This commit is contained in:
Lauréline Guérin 2024-03-21 15:02:02 +01:00
parent 1a4be6ec3e
commit 50cd07545c
No known key found for this signature in database
GPG Key ID: 1FAB9B9B4F93D473
6 changed files with 275 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
combo/apps/export_import/api_views.py
View File

@ -32,6 +32,7 @@ from rest_framework.response import Response
from combo.apps.export_import.models import Application, ApplicationAsyncJob, ApplicationElement
from combo.apps.wcs.utils import WCSError
from combo.data.models import Page, PageSnapshot
from combo.utils.api import APIErrorBadRequest
from combo.utils.misc import is_portal_agent
klasses = {klass.application_component_type: klass for klass in [Page]}
@ -204,89 +205,95 @@ class BundleCheck(GenericAPIView):
def put(self, request, *args, **kwargs):
tar_io = io.BytesIO(request.read())
page_type = 'portal-agent-pages' if is_portal_agent() else 'pages'
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
application_slug = manifest.get('slug')
application_version = manifest.get('version_number')
if not application_slug or not application_version:
return Response({'data': {}})
differences = []
unknown_elements = []
no_history_elements = []
legacy_elements = []
content_type = ContentType.objects.get_for_model(Page)
for element in manifest.get('elements'):
if element.get('type') != page_type:
continue
try:
with tarfile.open(fileobj=tar_io) as tar:
try:
page = Page.objects.get(uuid=element['slug'])
except Page.DoesNotExist:
unknown_elements.append(
{
'type': element['type'],
'slug': element['slug'],
}
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
except KeyError:
raise APIErrorBadRequest(_('Invalid tar file, missing manifest'))
application_slug = manifest.get('slug')
application_version = manifest.get('version_number')
if not application_slug or not application_version:
return Response({'data': {}})
differences = []
unknown_elements = []
no_history_elements = []
legacy_elements = []
content_type = ContentType.objects.get_for_model(Page)
for element in manifest.get('elements'):
if element.get('type') != page_type:
continue
try:
page = Page.objects.get(uuid=element['slug'])
except Page.DoesNotExist:
unknown_elements.append(
{
'type': element['type'],
'slug': element['slug'],
}
)
continue
elements_qs = ApplicationElement.objects.filter(
application__slug=application_slug,
content_type=content_type,
object_id=page.pk,
)
continue
elements_qs = ApplicationElement.objects.filter(
application__slug=application_slug,
content_type=content_type,
object_id=page.pk,
)
if not elements_qs.exists():
# object exists, but not linked to the application
legacy_elements.append(
{
'type': element['type'],
'slug': element['slug'],
# information needed here, Relation objects may not exist yet in hobo
'text': page.title,
'url': request.build_absolute_uri(
reverse(
'api-export-import-component-redirect',
kwargs={
'uuid': str(page.uuid),
'component_type': page.application_component_type,
},
)
),
}
)
continue
snapshot_for_app = (
PageSnapshot.objects.filter(
page=page,
application_slug=application_slug,
application_version=application_version,
)
.order_by('timestamp')
.last()
)
if not snapshot_for_app:
# no snapshot for this bundle
no_history_elements.append(
{
'type': element['type'],
'slug': element['slug'],
}
)
continue
last_snapshot = PageSnapshot.objects.filter(page=page).latest('timestamp')
if snapshot_for_app.pk != last_snapshot.pk:
differences.append(
{
'type': element['type'],
'slug': element['slug'],
'url': '%s%s?version1=%s&version2=%s'
% (
request.build_absolute_uri('/')[:-1],
reverse('combo-manager-page-history-compare', args=[page.pk]),
snapshot_for_app.pk,
last_snapshot.pk,
),
}
if not elements_qs.exists():
# object exists, but not linked to the application
legacy_elements.append(
{
'type': element['type'],
'slug': element['slug'],
# information needed here, Relation objects may not exist yet in hobo
'text': page.title,
'url': request.build_absolute_uri(
reverse(
'api-export-import-component-redirect',
kwargs={
'uuid': str(page.uuid),
'component_type': page.application_component_type,
},
)
),
}
)
continue
snapshot_for_app = (
PageSnapshot.objects.filter(
page=page,
application_slug=application_slug,
application_version=application_version,
)
.order_by('timestamp')
.last()
)
if not snapshot_for_app:
# no snapshot for this bundle
no_history_elements.append(
{
'type': element['type'],
'slug': element['slug'],
}
)
continue
last_snapshot = PageSnapshot.objects.filter(page=page).latest('timestamp')
if snapshot_for_app.pk != last_snapshot.pk:
differences.append(
{
'type': element['type'],
'slug': element['slug'],
'url': '%s%s?version1=%s&version2=%s'
% (
request.build_absolute_uri('/')[:-1],
reverse('combo-manager-page-history-compare', args=[page.pk]),
snapshot_for_app.pk,
last_snapshot.pk,
),
}
)
except tarfile.TarError:
raise APIErrorBadRequest(_('Invalid tar file'))
return Response(
{
@ -309,9 +316,15 @@ class BundleImport(GenericAPIView):
def put(self, request, *args, **kwargs):
tar_io = io.BytesIO(request.read())
with tarfile.open(fileobj=tar_io) as tar:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
application_slug = manifest.get('slug')
try:
with tarfile.open(fileobj=tar_io) as tar:
try:
manifest = json.loads(tar.extractfile('manifest.json').read().decode())
except KeyError:
raise APIErrorBadRequest(_('Invalid tar file, missing manifest'))
application_slug = manifest.get('slug')
except tarfile.TarError:
raise APIErrorBadRequest(_('Invalid tar file'))
job = ApplicationAsyncJob(
action=self.action,
)

31
combo/apps/export_import/models.py
View File

@ -33,6 +33,10 @@ from django.utils.translation import gettext_lazy as _
from combo.utils.misc import is_portal_agent
class BundleKeyError(Exception):
pass
class Application(models.Model):
name = models.CharField(max_length=100)
slug = models.SlugField(max_length=100, unique=True)
@ -59,10 +63,10 @@ class Application(models.Model):
slug=manifest.get('slug'), defaults={'editable': editable}
)
application.name = manifest.get('application')
application.description = manifest.get('description')
application.documentation_url = manifest.get('documentation_url')
application.description = manifest.get('description') or ''
application.documentation_url = manifest.get('documentation_url') or ''
application.version_number = manifest.get('version_number') or 'unknown'
application.version_notes = manifest.get('version_notes')
application.version_notes = manifest.get('version_notes') or ''
if not editable:
application.editable = editable
application.visible = manifest.get('visible', True)
@ -171,8 +175,6 @@ class ApplicationAsyncJob(models.Model):
last_update_timestamp = models.DateTimeField(auto_now=True)
completion_timestamp = models.DateTimeField(default=None, null=True)
raise_exception = True
def run(self, spool=False):
if 'uwsgi' in sys.modules and spool:
from combo.utils.spooler import run_async_job
@ -183,11 +185,12 @@ class ApplicationAsyncJob(models.Model):
self.save()
try:
getattr(self, self.action)()
except BundleKeyError as e:
self.status = 'failed'
self.exception = str(e)
except Exception:
self.status = 'failed'
self.exception = traceback.format_exc()
if self.raise_exception:
raise
finally:
if self.status == 'running':
self.status = 'completed'
@ -212,9 +215,17 @@ class ApplicationAsyncJob(models.Model):
for element in manifest.get('elements'):
if element.get('type') != page_type:
continue
pages.append(
json.loads(tar.extractfile(f'{page_type}/{element["slug"]}').read().decode()).get('data')
)
try:
pages.append(
json.loads(tar.extractfile(f'{page_type}/{element["slug"]}').read().decode()).get(
'data'
)
)
except KeyError:
raise BundleKeyError(
'Invalid tar file, missing component %s/%s.' % (page_type, element['slug'])
)
# init cache of application elements, from manifest
self.application_elements = set()
# install pages

2
combo/settings.py
View File

@ -391,6 +391,8 @@ CHART_FILTERS_CELL_ENABLED = True
# default country code for phonenumbers' user phone parsing
DEFAULT_COUNTRY_CODE = '33'
REST_FRAMEWORK = {'EXCEPTION_HANDLER': 'combo.utils.api.exception_handler'}
def debug_show_toolbar(request):
from debug_toolbar.middleware import show_toolbar as dt_show_toolbar # pylint: disable=import-error

60
combo/utils/api.py Normal file
View File

@ -0,0 +1,60 @@
# combo - content management system
# Copyright (C) 2017-2024 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.utils.translation import gettext_lazy as _
from rest_framework.response import Response as DRFResponse
from rest_framework.views import exception_handler as DRF_exception_handler
class Response(DRFResponse):
def __init__(self, data=None, *args, **kwargs):
# add reason for compatibility (https://dev.entrouvert.org/issues/24025)
if data is not None and 'err_class' in data:
data['reason'] = data['err_class']
super().__init__(data=data, *args, **kwargs)
class APIError(Exception):
http_status = 200
def __init__(self, message, *args, err=1, err_class=None, errors=None):
self.err_desc = _(message) % args
self.err = err
self.err_class = err_class or message % args
self.errors = errors
super().__init__(self.err_desc)
def to_response(self):
data = {
'err': self.err,
'err_class': self.err_class,
'err_desc': self.err_desc,
}
if self.errors:
data['errors'] = self.errors
return Response(data, status=self.http_status)
class APIErrorBadRequest(APIError):
http_status = 400
def exception_handler(exc, context):
if isinstance(exc, APIError):
return exc.to_response()
return DRF_exception_handler(exc, context)

3
tests/settings.py
View File

@ -125,7 +125,8 @@ REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': [
'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
]
],
'EXCEPTION_HANDLER': 'combo.utils.api.exception_handler',
}
DEFAULT_COUNTRY_CODE = '33'

94
tests/test_applification_api.py
View File

@ -347,6 +347,45 @@ def test_bundle_import(app, john_doe):
assert last_snapshot.application_slug == 'test'
assert last_snapshot.application_version == '42.1'
# bad file format
resp = app.put('/api/export-import/bundle-import/', b'garbage', status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file'
# missing manifest
tar_io = io.BytesIO()
with tarfile.open(mode='w', fileobj=tar_io) as tar:
foo_fd = io.BytesIO(json.dumps({'foo': 'bar'}, indent=2).encode())
tarinfo = tarfile.TarInfo('foo.json')
tarinfo.size = len(foo_fd.getvalue())
tar.addfile(tarinfo, fileobj=foo_fd)
resp = app.put('/api/export-import/bundle-import/', tar_io.getvalue(), status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file, missing manifest'
# missing component
tar_io = io.BytesIO()
with tarfile.open(mode='w', fileobj=tar_io) as tar:
manifest_json = {
'application': 'Test',
'slug': 'test',
'elements': [{'type': 'pages', 'slug': str(uuid.uuid4()), 'name': 'foo'}],
}
manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode())
tarinfo = tarfile.TarInfo('manifest.json')
tarinfo.size = len(manifest_fd.getvalue())
tar.addfile(tarinfo, fileobj=manifest_fd)
resp = app.put('/api/export-import/bundle-import/', tar_io.getvalue())
job_url = resp.json['url']
resp = app.get(job_url)
assert resp.json['data']['status'] == 'failed'
job = ApplicationAsyncJob.objects.get(uuid=job_url.split('/')[-3])
assert job.status == 'failed'
assert (
job.exception
== 'Invalid tar file, missing component pages/%s.' % manifest_json['elements'][0]['slug']
)
def test_bundle_import_pages_position(app, john_doe):
app.authorization = ('Basic', (john_doe.username, john_doe.username))
@ -952,6 +991,45 @@ def test_bundle_declare(app, john_doe):
assert application.visible is True
assert ApplicationElement.objects.count() == 0
# bad file format
resp = app.put('/api/export-import/bundle-declare/', b'garbage', status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file'
# missing manifest
tar_io = io.BytesIO()
with tarfile.open(mode='w', fileobj=tar_io) as tar:
foo_fd = io.BytesIO(json.dumps({'foo': 'bar'}, indent=2).encode())
tarinfo = tarfile.TarInfo('foo.json')
tarinfo.size = len(foo_fd.getvalue())
tar.addfile(tarinfo, fileobj=foo_fd)
resp = app.put('/api/export-import/bundle-declare/', tar_io.getvalue(), status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file, missing manifest'
# missing component
tar_io = io.BytesIO()
with tarfile.open(mode='w', fileobj=tar_io) as tar:
manifest_json = {
'application': 'Test',
'slug': 'test',
'elements': [{'type': 'pages', 'slug': str(uuid.uuid4()), 'name': 'foo'}],
}
manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode())
tarinfo = tarfile.TarInfo('manifest.json')
tarinfo.size = len(manifest_fd.getvalue())
tar.addfile(tarinfo, fileobj=manifest_fd)
resp = app.put('/api/export-import/bundle-declare/', tar_io.getvalue())
job_url = resp.json['url']
resp = app.get(job_url)
assert resp.json['data']['status'] == 'failed'
job = ApplicationAsyncJob.objects.get(uuid=job_url.split('/')[-3])
assert job.status == 'failed'
assert (
job.exception
== 'Invalid tar file, missing component pages/%s.' % manifest_json['elements'][0]['slug']
)
def test_bundle_unlink(app, john_doe, bundle):
app.authorization = ('Basic', (john_doe.username, john_doe.username))
@ -1153,6 +1231,22 @@ def test_bundle_check(app, john_doe):
}
}
# bad file format
resp = app.put('/api/export-import/bundle-check/', b'garbage', status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file'
# missing manifest
tar_io = io.BytesIO()
with tarfile.open(mode='w', fileobj=tar_io) as tar:
foo_fd = io.BytesIO(json.dumps({'foo': 'bar'}, indent=2).encode())
tarinfo = tarfile.TarInfo('foo.json')
tarinfo.size = len(foo_fd.getvalue())
tar.addfile(tarinfo, fileobj=foo_fd)
resp = app.put('/api/export-import/bundle-check/', tar_io.getvalue(), status=400)
assert resp.json['err']
assert resp.json['err_desc'] == 'Invalid tar file, missing manifest'
@mock.patch('requests.Session.send', side_effect=mocked_requests_send)
def test_page_dependencies_card_models(mock_send, app, john_doe):