combo/tests/test_import_export.py

608 lines
22 KiB
Python

import base64
import json
import os
import shutil
import sys
import tarfile
import tempfile
import uuid
from io import BytesIO, StringIO
import pytest
from django.contrib.auth.models import Group
from django.core.files import File
from django.core.files.storage import default_storage
from django.core.management import call_command
from django.core.management.base import CommandError
from django.utils.encoding import force_bytes, force_str
from combo.apps.assets.models import Asset
from combo.apps.assets.utils import clean_assets_files
from combo.apps.gallery.models import GalleryCell, Image
from combo.apps.lingo.models import PaymentBackend, Regie
from combo.apps.maps.models import Map, MapLayer, MapLayerOptions
from combo.apps.pwa.models import PwaNavigationEntry, PwaSettings
from combo.data.models import LinkCell, Page, SiteSettings, TextCell
from combo.data.utils import ImportSiteError, MissingGroups, export_site, import_site
pytestmark = pytest.mark.django_db
@pytest.fixture
def some_data():
Page.objects.create(title='One', slug='one')
Page.objects.create(title='Two', slug='two')
page = Page.objects.create(title='Three', slug='three')
cell = TextCell(page=page, order=0, text='hello world', placeholder='content')
cell.save()
@pytest.fixture
def some_map_layers():
MapLayer(label='Foo', slug='foo', geojson_url='http://example.net/foo/').save()
MapLayer(label='Bar', slug='bar', geojson_url='http://example.net/bar/').save()
@pytest.fixture
def some_assets():
Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
Asset(key='favicon', asset=File(BytesIO(b'test2'), 'test2.png')).save()
def get_output_of_command(command, *args, **kwargs):
old_stdout = sys.stdout
output = sys.stdout = StringIO()
call_command(command, format_json=True, *args, **kwargs)
sys.stdout = old_stdout
return output.getvalue()
def test_import_export(app, some_data):
output = get_output_of_command('export_site')
assert len(json.loads(output)['pages']) == 3
import_site(data={}, clean=True)
assert Page.objects.all().count() == 0
assert TextCell.objects.all().count() == 0
empty_output = get_output_of_command('export_site')
assert len(json.loads(empty_output)['pages']) == 0
Page(title='test', slug='test').save()
old_stdin = sys.stdin
sys.stdin = StringIO(json.dumps({}))
assert Page.objects.count() == 1
try:
call_command('import_site', '-', clean=True)
finally:
sys.stdin = old_stdin
assert Page.objects.count() == 0
with tempfile.NamedTemporaryFile() as f:
f.write(force_bytes(output))
f.flush()
call_command('import_site', f.name)
assert Page.objects.count() == 3
assert TextCell.objects.all().count() == 1
import_site(data={}, if_empty=True)
assert Page.objects.count() == 3
assert TextCell.objects.all().count() == 1
import_site(data={}, clean=True)
tempdir = tempfile.mkdtemp('chrono-test')
empty_output = get_output_of_command('export_site', output=os.path.join(tempdir, 't.json'))
assert os.path.exists(os.path.join(tempdir, 't.json'))
shutil.rmtree(tempdir)
def test_import_export_with_parent(app, some_data):
output = get_output_of_command('export_site')
payload = json.loads(output)
payload['pages'][1]['fields']['parent'] = [str(Page.objects.get(slug='one').uuid)]
Page.objects.all().delete()
import_site(data=payload)
assert Page.objects.count() == 3
two = Page.objects.get(slug='two')
assert two.parent.slug == 'one'
def test_import_export_with_unknown_parent(app, some_data):
output = get_output_of_command('export_site')
payload = json.loads(output)
payload['pages'][0]['fields']['exclude_from_navigation'] = False
payload['pages'][0]['fields']['parent'] = [str(uuid.uuid4())]
Page.objects.all().delete()
import_site(data=payload)
assert Page.objects.count() == 3
for page in Page.objects.all():
assert page.parent is None
one = Page.objects.get(slug='one')
assert one.exclude_from_navigation is True
def test_import_export_with_nested_parents():
one = Page.objects.create(title='One', slug='one')
two = Page.objects.create(title='Two', slug='two', parent=one)
Page.objects.create(title='Three', slug='three', parent=two)
output = get_output_of_command('export_site')
Page.objects.all().delete()
import_site(data=json.loads(output))
assert Page.objects.count() == 3
two = Page.objects.get(slug='two')
assert two.parent.slug == 'one'
two = Page.objects.get(slug='three')
assert two.parent.slug == 'two'
def test_import_export_map_layers(app, some_map_layers):
output = get_output_of_command('export_site')
assert len(json.loads(output)['map-layers']) == 2
import_site(data={}, clean=True)
assert MapLayer.objects.all().count() == 0
empty_output = get_output_of_command('export_site')
assert len(json.loads(empty_output)['map-layers']) == 0
MapLayer(label='Baz', slug='baz', geojson_url='http://example.net/baz/').save()
old_stdin = sys.stdin
sys.stdin = StringIO(json.dumps({}))
assert MapLayer.objects.count() == 1
try:
call_command('import_site', '-', clean=True)
finally:
sys.stdin = old_stdin
assert MapLayer.objects.count() == 0
with tempfile.NamedTemporaryFile() as f:
f.write(force_bytes(output))
f.flush()
call_command('import_site', f.name)
assert MapLayer.objects.count() == 2
import_site(data={}, if_empty=True)
assert MapLayer.objects.count() == 2
def test_import_export_map_cells(app, some_data, some_map_layers):
page = Page.objects.get(slug='one')
cell = Map(page=page, order=0, placeholder='content')
cell.save()
MapLayerOptions.objects.create(map_cell=cell, map_layer=MapLayer.objects.get(slug='foo'))
site_export = get_output_of_command('export_site')
import_site(data={}, clean=True)
assert Map.objects.count() == 0
assert MapLayer.objects.count() == 0
site_data = json.loads(site_export)
import_site(data=site_data, clean=True)
assert Map.objects.count() == 1
assert MapLayer.objects.filter(slug='foo').exists() is True
assert Map.objects.all()[0].layers.all()[0].slug == 'foo'
# test old export format
import_site(data={}, clean=True)
assert Map.objects.count() == 0
assert MapLayer.objects.count() == 0
del site_data['pages'][0]['cells'][0]['layers']
site_data['pages'][0]['cells'][0]['fields']['layers'] = [['foo']]
import_site(data=site_data, clean=True)
assert Map.objects.count() == 1
assert MapLayer.objects.filter(slug='foo').exists() is True
assert Map.objects.all()[0].layers.all()[0].slug == 'foo'
# test import with missing MapLayer
del site_data['map-layers']
MapLayer.objects.all().delete()
with pytest.raises(ImportSiteError, match='Unknown map layer "foo"'):
import_site(data=site_data, clean=True)
def test_group_restrictions_import_export(app, some_data):
group = Group(name='A Group')
group.save()
page = Page.objects.get(slug='one')
page.groups.set([group])
page.save()
cell = TextCell.objects.get(order=0)
cell.groups.set([group])
cell.save()
output = get_output_of_command('export_site')
assert len(json.loads(output)['pages']) == 3
import_site(data={}, clean=True)
assert Page.objects.all().count() == 0
assert TextCell.objects.all().count() == 0
Group.objects.all().delete()
with pytest.raises(MissingGroups) as excinfo:
import_site(json.loads(output), clean=True)
assert excinfo.value.names == ['A Group']
with pytest.raises(CommandError, match='Missing groups: A Group'):
with tempfile.NamedTemporaryFile() as f:
f.write(force_bytes(output))
f.flush()
call_command('import_site', f.name, clean=True)
assert Page.objects.count() == 0
group = Group(name='A Group')
group.save()
import_site(json.loads(output), clean=True)
assert Page.objects.all().count() == 3
assert TextCell.objects.all().count() == 1
page = Page.objects.get(slug='one')
assert [x.name for x in page.groups.all()] == ['A Group']
cell = TextCell.objects.get(order=0)
assert [x.name for x in cell.groups.all()] == ['A Group']
def test_import_export_assets(app, some_assets):
output = get_output_of_command('export_site')
assert len(json.loads(output)['assets']) == 2
import_site(data={}, clean=True)
assert Asset.objects.all().count() == 0
empty_output = get_output_of_command('export_site')
assert len(json.loads(empty_output)['assets']) == 0
Asset(key='footer', asset=File(StringIO('test3'), 'test3.png')).save()
old_stdin = sys.stdin
sys.stdin = StringIO(json.dumps({}))
assert Asset.objects.count() == 1
try:
call_command('import_site', '-', clean=True)
finally:
sys.stdin = old_stdin
assert Asset.objects.count() == 0
with tempfile.NamedTemporaryFile() as f:
f.write(force_bytes(output))
f.flush()
call_command('import_site', f.name)
assert Asset.objects.count() == 2
import_site(data={}, if_empty=True)
assert Asset.objects.count() == 2
def test_import_export_pwa_settings(app):
output = get_output_of_command('export_site')
pwa_settings = PwaSettings.singleton()
pwa_settings.offline_text = 'Hello world'
pwa_settings.offline_retry_button = False
pwa_settings.save()
# check exort with no application icon
output = get_output_of_command('export_site')
pwa_settings.application_icon = File(BytesIO(b'te\30st'), 'test.png')
pwa_settings.save()
output = get_output_of_command('export_site')
import_site(data={}, clean=True)
assert PwaSettings.objects.all().count() == 0
import_site(data=json.loads(output))
assert PwaSettings.singleton().offline_retry_button is False
assert PwaSettings.singleton().offline_text == 'Hello world'
# check identical file was not touched
assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test.png'
assert PwaSettings.objects.get().application_icon.read() == b'te\30st'
# check with a change in icon file content
data = json.loads(output)
data['pwa']['settings']['icon:base64'] = force_str(base64.encodebytes(b'TEST'))
import_site(data=data)
assert PwaSettings.objects.get().application_icon.read() == b'TEST'
# check with a change in icon file name
data = json.loads(output)
data['pwa']['settings']['icon:base64'] = force_str(base64.encodebytes(b'TEST2'))
data['pwa']['settings']['application_icon'] = 'pwa/test2.png'
import_site(data=data)
assert os.path.basename(PwaSettings.objects.get().application_icon.file.name) == 'test2.png'
assert PwaSettings.objects.get().application_icon.read() == b'TEST2'
def test_import_export_pwa_navigation(app, some_data):
page = Page.objects.get(slug='one')
entry1 = PwaNavigationEntry(label='a', url='/', order=0)
entry2 = PwaNavigationEntry(link_page=page, order=1, icon=File(BytesIO(b'te\30st'), 'test.png'))
entry1.save()
entry2.save()
output = get_output_of_command('export_site')
import_site(data={}, clean=True)
assert PwaNavigationEntry.objects.all().count() == 0
import_site(data=json.loads(output))
assert PwaNavigationEntry.objects.all().count() == 2
# check identical file was not touched
assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test.png'
assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'te\30st'
# check a second import doesn't create additional entries
import_site(data=json.loads(output))
assert PwaNavigationEntry.objects.all().count() == 2
# check with a change in icon file content
data = json.loads(output)
data['pwa']['navigation'][1]['icon:base64'] = force_str(base64.encodebytes(b'TEST'))
import_site(data=data)
assert PwaNavigationEntry.objects.all().count() == 2
assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'TEST'
# check with a change in icon file name
data = json.loads(output)
data['pwa']['navigation'][1]['fields']['icon'] = 'pwa/test2.png'
data['pwa']['navigation'][1]['icon:base64'] = force_str(base64.encodebytes(b'TEST2'))
import_site(data=data)
assert PwaNavigationEntry.objects.all().count() == 2
assert os.path.basename(PwaNavigationEntry.objects.get(order=1).icon.file.name) == 'test2.png'
assert PwaNavigationEntry.objects.get(order=1).icon.read() == b'TEST2'
def test_import_export_gallery_images(app, some_data):
page = Page.objects.get(slug='one')
gallery = GalleryCell(page=page, order=2, placeholder='images')
gallery.save()
image1 = Image(gallery=gallery, image='path/foo.jpg', title='foo', order=1)
image2 = Image(gallery=gallery, image='path/bar.jpg', title='bar', order=2)
image1.save()
image2.save()
output = get_output_of_command('export_site')
import_site(data={}, clean=True)
assert Image.objects.all().count() == 0
import_site(data=json.loads(output))
assert Image.objects.all().count() == 2
image1 = Image.objects.get(title='foo')
assert image1.image == 'path/foo.jpg'
assert image1.gallery.placeholder == 'images'
def test_import_export_extra_fields(app, some_data):
site_export = export_site()
for page in site_export['pages']:
if page['fields']['slug'] == 'one':
page['fields']['extra_field_not_in_model'] = True
elif page['fields']['slug'] == 'three':
page['cells'][0]['fields']['extra_field_not_in_model'] = True
import_site(site_export)
assert Page.objects.count() == 3
assert TextCell.objects.count() == 1
def test_import_export_tar(tmpdir, some_assets):
filename = os.path.join(str(tmpdir), 'file.tar')
# build import having some_assets fixtures assets: banner and favicon
call_command('export_site', '--output', filename)
def populate_site():
Page.objects.all().delete()
Asset.objects.all().delete()
clean_assets_files()
Page.objects.create(title='One', slug='one')
Asset(key='banner', asset=File(BytesIO(b'original content'), 'test.png')).save()
Asset(key='logo', asset=File(BytesIO(b'logo'), 'logo.png')).save()
populate_site()
call_command('import_site', filename) # default behaviour
assert Page.objects.count() == 1
assert Asset.objects.count() == 3
assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
with open('%s/assets/test.png' % default_storage.path('')) as fd:
assert fd.read() == 'original content'
populate_site()
call_command('import_site', filename, '--overwrite')
assert Page.objects.count() == 1
assert Asset.objects.count() == 3
assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
with open('%s/assets/test.png' % default_storage.path('')) as fd:
assert fd.read() == 'test'
populate_site()
call_command('import_site', filename, '--if-empty')
assert Page.objects.count() == 1
assert Asset.objects.count() == 2
assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
with open('%s/assets/test.png' % default_storage.path('')) as fd:
assert fd.read() == 'original content'
assert Asset.objects.get(key='logo').asset.name == 'assets/logo.png'
assert os.path.isfile('%s/assets/logo.png' % default_storage.path(''))
populate_site()
call_command('import_site', filename, '--clean')
assert Page.objects.count() == 0
assert Asset.objects.count() == 2
assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
with open('%s/assets/test.png' % default_storage.path('')) as fd:
assert fd.read() == 'test'
assert not Asset.objects.filter(key='logo')
assert not os.path.isfile('%s/assets/logo.png' % default_storage.path(''))
# error cases
with pytest.raises(CommandError, match=r'No such file or directory'):
call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir)
with pytest.raises(CommandError, match='TAR format require output filename parameter'):
call_command('export_site', '--output', '-')
with pytest.raises(CommandError, match=r'No such file or directory'):
call_command('import_site', '%s/noway/foo.tar' % tmpdir)
with tarfile.open(filename, 'w'):
# empty tar file
pass
with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'):
call_command('import_site', filename)
def test_import_export_payment(app):
backend = PaymentBackend.objects.create(label='Test', slug='test', service_options={'test': True})
Regie.objects.create(label='Test regie', slug='test-regie', payment_backend=backend)
Regie.objects.create(label='Test regie 2', slug='test-regie-2', payment_backend=backend)
output = get_output_of_command('export_site')
payload = json.loads(output)
assert len(payload['payment']['backends']) == 1
assert len(payload['payment']['regies']) == 2
import_site(payload)
assert PaymentBackend.objects.count() == 1
assert Regie.objects.count() == 2
PaymentBackend.objects.all().delete()
Regie.objects.all().delete()
import_site(payload)
backend = PaymentBackend.objects.get(slug='test')
assert backend.label == 'Test'
assert backend.service_options == {'test': True}
assert Regie.objects.count() == 2
regie = Regie.objects.first()
assert regie.payment_backend == backend
import_site(data={}, clean=True)
assert PaymentBackend.objects.count() == 0
assert Regie.objects.count() == 0
empty_output = get_output_of_command('export_site')
assert len(json.loads(empty_output)['payment']['backends']) == 0
assert len(json.loads(empty_output)['payment']['regies']) == 0
def test_import_export_settings(app):
site_settings = SiteSettings.get_singleton()
site_settings.initial_login_page_path = '/test/'
site_settings.welcome_page_path = '/hop/'
site_settings.save()
output = get_output_of_command('export_site')
payload = json.loads(output)
assert len(payload['site_settings']) == 2
site_settings.initial_login_page_path = ''
site_settings.welcome_page_path = ''
site_settings.save()
import_site(payload)
site_settings.refresh_from_db()
assert site_settings.initial_login_page_path == '/test/'
assert site_settings.welcome_page_path == '/hop/'
site_settings.initial_login_page_path = ''
site_settings.welcome_page_path = ''
site_settings.save()
output = get_output_of_command('export_site')
payload = json.loads(output)
assert len(payload['site_settings']) == 2
import_site(payload)
site_settings.refresh_from_db()
assert site_settings.initial_login_page_path == ''
assert site_settings.welcome_page_path == ''
def test_import_export_linkcell_to_missing_page(app, admin_user):
page1 = Page.objects.create(title='One', slug='one')
page2 = Page.objects.create(title='Two', slug='two')
LinkCell.objects.create(page=page1, link_page=page2, placeholder='content', order=0)
output = get_output_of_command('export_site')
payload = json.loads(output)
del payload['pages'][1]
import_site(data=payload, clean=True)
cell = LinkCell.objects.get()
assert cell.link_page is None
assert cell.get_validity_info().invalid_reason_code == 'data_url_not_defined'
def test_import_export_linkcell_to_cell_page(app, admin_user):
root_page = Page.objects.create(title='Home', slug='index')
page = Page.objects.create(title='One', slug='one', parent=root_page)
LinkCell.objects.create(page=page, link_page=page, placeholder='content', order=0)
output = get_output_of_command('export_site')
payload = json.loads(output)
import_site(data=payload)
cell = LinkCell.objects.get()
assert Page.objects.count() == 2
root_page = Page.objects.get(slug='index')
page = Page.objects.get(slug='one')
assert cell.link_page == page
assert page.parent == root_page
def test_import_export_linkcell_to_url(app, admin_user):
page1 = Page.objects.create(title='One', slug='one')
LinkCell.objects.create(page=page1, url='https://example.com', placeholder='content', order=0)
output = get_output_of_command('export_site')
payload = json.loads(output)
import_site(data=payload, clean=True)
cell = LinkCell.objects.get()
assert cell.url == 'https://example.com'
def test_import_export_duplicated_slugs():
first_page = Page.objects.create(title='Title', slug='title', description='1')
output = get_output_of_command('export_site')
# create real situation where a subpage has the same slug as a top-level page
second_page = Page.objects.create(title='Test', slug='test')
third_page = Page.objects.create(title='Title', slug='title', description='test', parent=second_page)
first_page.description = '2'
first_page.save()
import_site(data=json.loads(output))
assert Page.objects.count() == 3
# top-level page has been updated
first_page.refresh_from_db()
assert first_page.description == '1'
# same slug subpage was left untouched
third_page.refresh_from_db()
assert third_page.description == 'test'
output = get_output_of_command('export_site')
third_page.description = ''
third_page.save()
import_site(data=json.loads(output))
assert Page.objects.count() == 3
# top level page was left untouched
first_page.refresh_from_db()
assert first_page.description == '1'
# same slug subpage has been updated
third_page.refresh_from_db()
assert third_page.description == 'test'
Page.objects.all().delete()
import_site(data=json.loads(output))
assert Page.objects.count() == 3
assert Page.objects.filter(parent__isnull=True).count() == 2