From 165ffc8527770e4adae09b27a14ee03aef73630d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= Date: Fri, 6 Jan 2023 16:14:33 +0100 Subject: [PATCH] data: import/export based on uuids (#67710) --- combo/apps/search/models.py | 3 +-- combo/apps/wcs/models.py | 3 +-- combo/data/models.py | 34 +++++++++++++--------------------- tests/test_import_export.py | 5 +++-- tests/test_manager.py | 17 ++++++++++------- tests/test_search.py | 14 ++++++-------- tests/wcs/test_card.py | 4 +++- 7 files changed, 37 insertions(+), 43 deletions(-) diff --git a/combo/apps/search/models.py b/combo/apps/search/models.py index 37e8f7af..9d147f2e 100644 --- a/combo/apps/search/models.py +++ b/combo/apps/search/models.py @@ -168,8 +168,7 @@ class SearchCell(CellBase): return cell_data for options in cell_data['fields']['_search_services']['options'].values(): if options.get('target_page'): - page_slug = options['target_page'].strip('/').split('/')[-1] - options['target_page'] = Page.get_page_ids_by_slugs().get(page_slug) or '' + options['target_page'] = Page.get_page_ids_by_uuids().get(options['target_page']) return cell_data def modify_global_context(self, context, request): diff --git a/combo/apps/wcs/models.py b/combo/apps/wcs/models.py index 308fb9ed..0a0682c4 100644 --- a/combo/apps/wcs/models.py +++ b/combo/apps/wcs/models.py @@ -947,8 +947,7 @@ class WcsCardCell(CardMixin, CellBase): custom_schema = cell_data['fields']['custom_schema'] for cell in custom_schema.get('cells') or []: if cell.get('page'): - page_slug = cell['page'].strip('/').split('/')[-1] - cell['page'] = Page.get_page_ids_by_slugs().get(page_slug) or '' + cell['page'] = Page.get_page_ids_by_uuids().get(cell['page']) or '' return cell_data diff --git a/combo/data/models.py b/combo/data/models.py index 89091a9b..b51458f1 100644 --- a/combo/data/models.py +++ b/combo/data/models.py @@ -180,9 +180,8 @@ class PageManager(models.Manager): self.snapshots = kwargs.pop('snapshots', False) super().__init__(*args, **kwargs) - def get_by_natural_key(self, path): - parts = [x for x in path.strip('/').split('/') if x] or ['index'] - return self.get(slug=parts[-1]) + def get_by_natural_key(self, uuid): + return self.get(uuid=uuid) def get_queryset(self): queryset = super().get_queryset() @@ -264,12 +263,12 @@ class Page(models.Model): return str(self.title) def natural_key(self): - return (self.get_online_url(follow_redirection=False).strip('/'),) + return (str(self.uuid),) @classmethod @utils.cache_during_request - def get_page_ids_by_slugs(cls): - return {page.slug: page.pk for page in cls.objects.only('pk', 'slug')} + def get_page_ids_by_uuids(cls): + return {str(page.uuid): page.pk for page in cls.objects.only('pk', 'uuid')} def picture_extension(self): if not self.picture: @@ -587,12 +586,12 @@ class Page(models.Model): qs_kwargs = {} if snapshot: qs_kwargs = {'snapshot': snapshot} # don't take uuid from snapshot: it has to be unique ! - elif json_page['fields'].get('parent'): - qs_kwargs = {'parent__slug': json_page['fields']['parent'][0].split('/')[-1] or 'index'} - page, created = Page.objects.get_or_create(slug=json_page['fields']['slug'], **qs_kwargs) + else: + qs_kwargs = {'uuid': json_page['fields']['uuid']} + page, created = Page.objects.get_or_create(**qs_kwargs) json_page['pk'] = page.id - parent_slug = json_page['fields'].get('parent') or [] - if parent_slug and not Page.objects.filter(slug=parent_slug[0].split('/')[-1] or 'index').exists(): + parent_uuid = json_page['fields'].get('parent') or [] + if parent_uuid and not Page.objects.filter(uuid=parent_uuid[0]).exists(): # parent not found, remove it and exclude page from navigation json_page['fields'].pop('parent') json_page['fields']['exclude_from_navigation'] = True @@ -657,13 +656,7 @@ class Page(models.Model): for json_page in json_site: # pre-create pages - parent = None - if json_page['fields'].get('parent'): - parent = json_page['fields']['parent'][0].split('/')[-1] or 'index' - - page, created = Page.objects.get_or_create( - slug=json_page['fields']['slug'], parent__slug=parent - ) + page, created = Page.objects.get_or_create(uuid=json_page['fields']['uuid']) to_load.append((page, created, json_page)) # delete cells of already existing pages @@ -787,7 +780,7 @@ class PageSnapshot(models.Model): # keep current page order json_page['fields']['order'] = self.page.order # and current parent - json_page['fields']['parent'] = [self.page.parent.slug] if self.page.parent else None + json_page['fields']['parent'] = self.page.parent.natural_key() if self.page.parent else None # and current exclude_from_navigation value json_page['fields']['exclude_from_navigation'] = self.page.exclude_from_navigation # restore snapshot @@ -1725,8 +1718,7 @@ class LinkCell(CellBase): @classmethod def prepare_serialized_data(cls, cell_data): if cell_data['fields'].get('link_page'): - link_page_slug = cell_data['fields']['link_page'][0].strip('/').split('/')[-1] - if link_page_slug not in Page.get_page_ids_by_slugs(): + if cell_data['fields']['link_page'][0] not in Page.get_page_ids_by_uuids(): del cell_data['fields']['link_page'] return cell_data diff --git a/tests/test_import_export.py b/tests/test_import_export.py index 7d6b8271..0f08acf3 100644 --- a/tests/test_import_export.py +++ b/tests/test_import_export.py @@ -5,6 +5,7 @@ import shutil import sys import tarfile import tempfile +import uuid from io import BytesIO, StringIO import pytest @@ -97,7 +98,7 @@ def test_import_export(app, some_data): def test_import_export_with_parent(app, some_data): output = get_output_of_command('export_site') payload = json.loads(output) - payload['pages'][1]['fields']['parent'] = ['one'] + payload['pages'][1]['fields']['parent'] = [str(Page.objects.get(slug='one').uuid)] Page.objects.all().delete() import_site(data=payload) @@ -111,7 +112,7 @@ def test_import_export_with_unknown_parent(app, some_data): output = get_output_of_command('export_site') payload = json.loads(output) payload['pages'][0]['fields']['exclude_from_navigation'] = False - payload['pages'][0]['fields']['parent'] = ['unknown-parent'] + payload['pages'][0]['fields']['parent'] = [str(uuid.uuid4())] Page.objects.all().delete() import_site(data=payload) diff --git a/tests/test_manager.py b/tests/test_manager.py index 98d3be53..91a9fc46 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -5,6 +5,7 @@ import os import re import shutil import urllib.parse +import uuid from io import BytesIO from unittest import mock @@ -1005,7 +1006,7 @@ def test_export_page_with_redirection(app, admin_user): resp = app.get('/manage/pages/%s/' % page1.pk) resp = resp.click('Export') assert resp.json['pages'][0]['cells'][0]['fields']['link_page'] == [ - 'two' + str(page2.uuid) ] # and not http://www.example.net @@ -1045,7 +1046,7 @@ def test_site_export_import_json(app, admin_user): resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json') with CaptureQueriesContext(connection) as ctx: resp = resp.form.submit() - assert len(ctx.captured_queries) in [304, 305] + assert len(ctx.captured_queries) in [295, 296] assert Page.objects.count() == 4 assert PageSnapshot.objects.all().count() == 4 @@ -1056,7 +1057,7 @@ def test_site_export_import_json(app, admin_user): resp.form['site_file'] = Upload('site-export.json', site_export, 'application/json') with CaptureQueriesContext(connection) as ctx: resp = resp.form.submit() - assert len(ctx.captured_queries) == 274 + assert len(ctx.captured_queries) == 276 assert set(Page.objects.get(slug='one').related_cells['cell_types']) == {'data_textcell', 'data_linkcell'} assert Page.objects.count() == 4 assert LinkCell.objects.count() == 2 @@ -1176,7 +1177,7 @@ def test_site_export_import_unknown_parent(app, admin_user): resp = resp.form.submit() payload = json.loads(force_str(resp.body)) payload['pages'][0]['fields']['exclude_from_navigation'] = False - payload['pages'][0]['fields']['parent'] = ['unknown-parent'] + payload['pages'][0]['fields']['parent'] = [str(uuid.uuid4())] resp = app.get('/manage/') resp = resp.click('Import Site') @@ -1197,13 +1198,15 @@ def test_site_export_import_unknown_page(app, admin_user): resp = app.get('/manage/site-export') resp = resp.form.submit() payload = json.loads(force_str(resp.body)) - payload['pages'][0]['cells'][0]['fields']['root_page'] = ['unknown-parent'] + payload['pages'][0]['cells'][0]['fields']['root_page'] = [str(uuid.uuid4())] Page.objects.all().delete() resp = app.get('/manage/site-import') resp.form['site_file'] = Upload('site-export.json', force_bytes(json.dumps(payload)), 'application/json') resp = resp.form.submit() - assert resp.context['form'].errors['site_file'] == ['Unknown page "unknown-parent".'] + assert resp.context['form'].errors['site_file'] == [ + 'Unknown page "%s".' % payload['pages'][0]['cells'][0]['fields']['root_page'][0] + ] assert Page.objects.count() == 0 resp = app.get('/manage/site-import') @@ -2519,7 +2522,7 @@ def test_page_versionning(app, admin_user): with CaptureQueriesContext(connection) as ctx: resp2 = resp.click('view', index=1) - assert len(ctx.captured_queries) == 70 + assert len(ctx.captured_queries) == 72 assert Page.snapshots.latest('pk').related_cells == {'cell_types': ['data_textcell']} snapshot_page = Page.snapshots.latest('pk') assert snapshot_page.uuid != page.uuid diff --git a/tests/test_search.py b/tests/test_search.py index 80847f56..d6f911a1 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -2,6 +2,7 @@ import datetime import json import os import sys +import uuid from io import StringIO from unittest import mock @@ -1689,12 +1690,9 @@ def test_test_export_import_search_cell_with_target_page(): site_export = get_output_of_command('export_site') site_data = json.loads(site_export) assert len(site_data['pages']) == 3 - assert ( - site_data['pages'][-1]['cells'][0]['fields']['_search_services']['options'][ - 'cards:c21f969b:card-bar' - ]['target_page'] - == 'root/card' - ) + assert site_data['pages'][-1]['cells'][0]['fields']['_search_services']['options'][ + 'cards:c21f969b:card-bar' + ]['target_page'] == str(card_page.uuid) import_site(data={}, clean=True) assert Page.objects.all().count() == 0 @@ -1717,8 +1715,8 @@ def test_test_export_import_search_cell_with_target_page(): site_data['pages'][-1]['cells'][0]['fields']['_search_services']['options']['cards:c21f969b:card-bar'][ 'target_page' - ] = 'unknown' + ] = str(uuid.uuid4()) import_site(data=site_data, clean=True) new_card_page = Page.objects.get(slug='card') new_cell = SearchCell.objects.get() - assert new_cell._search_services['options']['cards:c21f969b:card-bar']['target_page'] == '' + assert new_cell._search_services['options']['cards:c21f969b:card-bar']['target_page'] is None diff --git a/tests/wcs/test_card.py b/tests/wcs/test_card.py index 87966307..10de7a0a 100644 --- a/tests/wcs/test_card.py +++ b/tests/wcs/test_card.py @@ -3387,7 +3387,9 @@ def test_export_import_card_cell_with_page_link(): site_export = get_output_of_command('export_site') site_data = json.loads(site_export) assert len(site_data['pages']) == 3 - assert site_data['pages'][-1]['cells'][0]['fields']['custom_schema']['cells'][0]['page'] == 'root/card' + assert site_data['pages'][-1]['cells'][0]['fields']['custom_schema']['cells'][0]['page'] == str( + card_page.uuid + ) import_site(data={}, clean=True) assert Page.objects.all().count() == 0