entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 10 Releases Wiki Activity

commands: add tar format for site export/import (#39425)

This commit is contained in:
Nicolas Roche 2020-07-13 17:45:49 +02:00
parent f0e0de43e5
commit 0f856f1334
5 changed files with 156 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
combo/data/management/commands/export_site.py
View File

@ -17,9 +17,10 @@
import json
import sys
from django.core.management.base import BaseCommand
from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext_lazy as _
from combo.data.utils import export_site
from combo.data.utils import export_site, export_site_tar
class Command(BaseCommand):
help = 'Export the site'
@ -28,10 +29,24 @@ class Command(BaseCommand):
parser.add_argument(
'--output', metavar='FILE', default=None,
help='name of a file to write output to')
parser.add_argument(
'--format-json', action='store_true', default=False,
help='use JSON format with no asset files')
def handle(self, *args, **options):
if options['output'] and options['output'] != '-':
output = open(options['output'], 'w')
if options['format_json']:
if options['output'] and options['output'] != '-':
output = open(options['output'], 'w')
else:
output = sys.stdout
json.dump(export_site(), output, indent=2)
else:
output = sys.stdout
json.dump(export_site(), output, indent=2)
if options['output'] and options['output'] != '-':
try:
output = open(options['output'], 'wb')
except IOError as e:
raise CommandError(e)
export_site_tar(output)
output.close()
else:
raise CommandError(_('TAR format require output filename parameter'))

35
combo/data/management/commands/import_site.py
View File

@ -16,11 +16,11 @@
import json
import sys
import tarfile
from django.core.management.base import BaseCommand, CommandError
from django.utils.encoding import force_text
from combo.data.utils import import_site, MissingGroups
from combo.data.utils import import_site, import_site_tar, ImportSiteError
class Command(BaseCommand):
help = 'Import an exported site'
@ -34,15 +34,36 @@ class Command(BaseCommand):
parser.add_argument(
'--if-empty', action='store_true', default=False,
help='Import only if site is empty')
parser.add_argument(
'--overwrite', action='store_true', default=False,
help='Overwrite asset files')
def handle(self, filename, *args, **options):
if filename == '-':
format = 'json'
fd = sys.stdin
else:
fd = open(filename)
try:
fd = open(filename, 'rb')
except IOError as e:
raise CommandError(e)
try:
tarfile.open(mode='r', fileobj=fd)
except tarfile.TarError as e:
format = 'json'
fd = open(filename, 'r')
else:
format = 'tar'
fd = open(filename, 'rb')
try:
import_site(json.load(fd),
if_empty=options['if_empty'],
clean=options['clean'])
except MissingGroups as e:
if format == 'json':
import_site(json.load(fd),
if_empty=options['if_empty'],
clean=options['clean'])
else:
import_site_tar(fd,
if_empty=options['if_empty'],
clean=options['clean'],
overwrite=options['overwrite'])
except ImportSiteError as e:
raise CommandError(e)

41
combo/data/utils.py
View File

@ -14,6 +14,9 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import tarfile
from django.contrib.auth.models import Group
from django.db import transaction
from django.utils import six
@ -21,13 +24,20 @@ from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext_lazy as _
from combo.apps.assets.models import Asset
from combo.apps.assets.utils import add_tar_content
from combo.apps.assets.utils import clean_assets_files
from combo.apps.assets.utils import untar_assets_files
from combo.apps.assets.utils import tar_assets_files
from combo.apps.maps.models import MapLayer
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry
from .models import Page
class ImportSiteError(Exception):
pass
@python_2_unicode_compatible
class MissingGroups(Exception):
class MissingGroups(ImportSiteError):
def __init__(self, names):
self.names = names
@ -85,3 +95,32 @@ def import_site(data, if_empty=False, clean=False, request=None):
if data.get('pwa'):
PwaSettings.load_serialized_settings(data['pwa'].get('settings'))
PwaNavigationEntry.load_serialized_objects(data['pwa'].get('navigation'))
def export_site_tar(fd):
tar = tarfile.open(mode='w', fileobj=fd)
data = export_site()
del data['assets']
add_tar_content(tar, '_site.json', json.dumps(data, indent=2))
tar_assets_files(tar)
tar.close()
def import_site_tar(fd, if_empty=False, clean=False, overwrite=False, request=None):
tar = tarfile.open(mode='r', fileobj=fd)
try:
tarinfo = tar.getmember('_site.json')
except KeyError:
raise ImportSiteError(_('TAR file should provide _site.json file'))
if if_empty and (Page.objects.count() or MapLayer.objects.count()):
return
if clean:
clean_assets_files()
json_site = tar.extractfile(tarinfo).read()
data = json.loads(json_site.decode('utf-8'))
data.update(untar_assets_files(tar, overwrite=overwrite))
import_site(data, if_empty=if_empty, clean=clean, request=request)
tar.close()

66
tests/test_import_export.py
View File

@ -4,17 +4,20 @@ import json
import os
import shutil
import sys
import tarfile
import tempfile
import pytest
from django.contrib.auth.models import Group
from django.core.files import File
from django.core.files.storage import default_storage
from django.core.management import call_command
from django.core.management.base import CommandError
from django.utils.encoding import force_bytes, force_text
from django.utils.six import BytesIO, StringIO
from combo.apps.assets.models import Asset
from combo.apps.assets.utils import clean_assets_files
from combo.apps.gallery.models import Image, GalleryCell
from combo.apps.maps.models import MapLayer, Map, MapLayerOptions
from combo.apps.pwa.models import PwaSettings, PwaNavigationEntry
@ -46,7 +49,7 @@ def some_assets():
def get_output_of_command(command, *args, **kwargs):
old_stdout = sys.stdout
output = sys.stdout = StringIO()
call_command(command, *args, **kwargs)
call_command(command, format_json=True, *args, **kwargs)
sys.stdout = old_stdout
return output.getvalue()
@ -357,3 +360,64 @@ def test_import_export_extra_fields(app, some_data):
import_site(site_export)
assert Page.objects.count() == 3
assert TextCell.objects.count() == 1
def test_import_export_tar(tmpdir, some_assets):
filename = os.path.join(str(tmpdir), 'file.tar')
# build import having some_assets fixtures assets: banner and favicon
call_command('export_site', '--output', filename)
def populate_site():
Page.objects.all().delete()
Asset.objects.all().delete()
clean_assets_files()
Page.objects.create(title='One', slug='one')
Asset(key='banner', asset=File(BytesIO(b'original content'), 'test.png')).save()
Asset(key='logo', asset=File(BytesIO(b'logo'), 'logo.png')).save()
populate_site()
call_command('import_site', filename) # default behaviour
assert Page.objects.count() == 1
assert Asset.objects.count() == 3
Asset.objects.get(key='banner').asset.name == 'assets/test.png'
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'original content'
populate_site()
call_command('import_site', filename, '--overwrite')
assert Page.objects.count() == 1
assert Asset.objects.count() == 3
Asset.objects.get(key='banner').asset.name == 'assets/test.png'
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'test'
populate_site()
call_command('import_site', filename, '--if-empty')
assert Page.objects.count() == 1
assert Asset.objects.count() == 2
Asset.objects.get(key='banner').asset.name == 'assets/test3.png'
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'original content'
Asset.objects.get(key='logo').asset.name == 'assets/logo.png'
assert os.path.isfile('%s/assets/logo.png' % default_storage.path(''))
populate_site()
call_command('import_site', filename, '--clean')
assert Page.objects.count() == 0
assert Asset.objects.count() == 2
Asset.objects.get(key='banner').asset.name == 'assets/test.png'
assert open('%s/assets/test.png' % default_storage.path('')).read() == 'test'
assert not Asset.objects.filter(key='logo')
assert not os.path.isfile('%s/assets/logo.png' % default_storage.path(''))
# error cases
with pytest.raises(CommandError, match=r'No such file or directory'):
call_command('export_site', '--output', '%s/noway/foo.tar' % tmpdir)
with pytest.raises(CommandError, match='TAR format require output filename parameter'):
call_command('export_site', '--output', '-')
with pytest.raises(CommandError, match=r'No such file or directory'):
call_command('import_site', '%s/noway/foo.tar' % tmpdir)
tarfile.open(filename, 'w').close() # empty tar file
with pytest.raises(CommandError, match=r'TAR file should provide _site.json file'):
call_command('import_site', filename)

4
tests/test_pages.py
View File

@ -341,13 +341,13 @@ def test_import_export_management_commands():
os.unlink(export_filename)
cmd = ExportSiteCommand()
cmd.handle(output=export_filename)
cmd.handle(output=export_filename, format_json=True)
assert os.path.exists(export_filename)
stdout = sys.stdout
try:
sys.stdout = StringIO()
cmd.handle(output='-')
cmd.handle(output='-', format_json=True)
assert sys.stdout.getvalue() == open(export_filename).read()
finally:
sys.stdout = stdout