assets: don't export or clean non asset files (#86870)

combo/apps/assets/utils.py

@@ -23,10 +23,28 @@ from django.core.files.storage import default_storage
 
 from .models import Asset
 
+ASSET_DIRS = [
+    'assets',
+    'page-pictures',
+    'uploads',
+]
+
+
+def is_asset_dir(basedir):
+    # exclude dirs like cache or applications, which contain non asset files
+    media_prefix = default_storage.path('')
+    asset_basedirs = [os.path.join(media_prefix, ad) for ad in ASSET_DIRS]
+    for adb in asset_basedirs:
+        if basedir.startswith(adb):
+            return True
+    return False
+
 
 def clean_assets_files():
     media_prefix = default_storage.path('')
     for basedir, dummy, filenames in os.walk(media_prefix):
+        if not is_asset_dir(basedir):
+            continue
         for filename in filenames:
             os.remove('%s/%s' % (basedir, filename))
 
@@ -59,6 +77,8 @@ def untar_assets_files(tar, overwrite=False):
 def tar_assets_files(tar):
     media_prefix = default_storage.path('')
     for basedir, dummy, filenames in os.walk(media_prefix):
+        if not is_asset_dir(basedir):
+            continue
         for filename in filenames:
             tar.add(os.path.join(basedir, filename), os.path.join(basedir, filename)[len(media_prefix) :])
     export = {'assets': Asset.export_all_for_json()}

tests/test_assets.py

@@ -22,6 +22,19 @@ from combo.apps.assets.utils import (
 pytestmark = pytest.mark.django_db
 
 
+def _clean_media():
+    media_prefix = default_storage.path('')
+    for basedir, dummy, filenames in os.walk(media_prefix):
+        for filename in filenames:
+            os.remove('%s/%s' % (basedir, filename))
+
+
+@pytest.fixture(autouse=True)
+def clean_media():
+    yield
+    _clean_media()
+
+
 @pytest.fixture
 def some_assets():
     Asset(key='banner', asset=File(BytesIO(b'test'), 'test.png')).save()
@@ -80,13 +93,22 @@ def test_asset_set_api(app, john_doe):
             status=400,
         )
         assert resp.json.get('err') == 1
-    clean_assets_files()
 
 
 def test_clean_assets_files(some_assets):
-    assert count_asset_files() == 2
+    path = default_storage.path('')
+    os.makedirs(os.path.join(path, 'page-pictures'), exist_ok=True)
+    with open('%s/page-pictures/pictures.png' % path, 'w') as fd:
+        fd.write('foo')
+    os.makedirs(os.path.join(path, 'applications'), exist_ok=True)  # application icons
+    with open('%s/applications/appli.png' % path, 'w') as fd:
+        fd.write('foo')
+    os.makedirs(os.path.join(path, 'cache', '07', 'db'), exist_ok=True)  # cache for thumbnails
+    with open('%s/cache/07/db/hash.png' % path, 'w') as fd:
+        fd.write('foo')
+    assert count_asset_files() == 5
     clean_assets_files()
-    assert count_asset_files() == 0
+    assert count_asset_files() == 2  # application icons and thumbnail cache not removed
 
 
 def test_add_tar_content(tmpdir):
@@ -125,34 +147,47 @@ def test_tar_untar_assets(some_assets):
         assert fd.read() == 'test'
     with open('%s/assets/test2.png' % path) as fd:
         assert fd.read() == 'foo'
-    clean_assets_files()
 
 
 def test_import_export_assets(some_assets, tmpdir):
+    path = default_storage.path('')
+    os.makedirs(os.path.join(path, 'page-pictures'), exist_ok=True)
+    with open('%s/page-pictures/pictures.png' % path, 'w') as fd:
+        fd.write('foo')
+    os.makedirs(os.path.join(path, 'applications'), exist_ok=True)  # application icons
+    with open('%s/applications/appli.png' % path, 'w') as fd:
+        fd.write('foo')
+    os.makedirs(os.path.join(path, 'cache', '07', 'db'), exist_ok=True)  # cache for thumbnails
+    with open('%s/cache/07/db/hash.png' % path, 'w') as fd:
+        fd.write('foo')
     filename = os.path.join(str(tmpdir), 'file.tar')
     assert Asset.objects.count() == 2
-    assert count_asset_files() == 2
+    assert count_asset_files() == 5
     with open(filename, 'wb') as fd:
         export_assets(fd)
 
-    path = default_storage.path('')
     os.remove('%s/assets/test.png' % path)
     with open('%s/assets/test2.png' % path, 'w') as fd:
         fd.write('foo')
-    assert count_asset_files() == 1
+    assert count_asset_files() == 4
     Asset.objects.all().delete()
     assert Asset.objects.count() == 0
 
     with open(filename, 'rb') as fd:
         import_assets(fd, overwrite=True)
-    assert count_asset_files() == 2
+    assert count_asset_files() == 5
     with open('%s/assets/test.png' % path) as fd:
         assert fd.read() == 'test'
     with open('%s/assets/test2.png' % path) as fd:
         assert fd.read() == 'test2'
     clean_assets_files()
+    assert count_asset_files() == 2
+
+    _clean_media()
     assert count_asset_files() == 0
-    clean_assets_files()
+    with open(filename, 'rb') as fd:
+        import_assets(fd, overwrite=True)
+    assert count_asset_files() == 3  # only assets and page-picture in tar file and imported
 
 
 def test_assets_export_size_view(app, some_assets):

tests/test_import_export.py

@@ -413,13 +413,15 @@ def test_import_export_tar(tmpdir, some_assets):
         Page.objects.all().delete()
         Asset.objects.all().delete()
         clean_assets_files()
-        Page.objects.create(title='One', slug='one')
+        Page(title='One', slug='one', picture=File(BytesIO(b'picture'), 'picture.png')).save()
         Asset(key='banner', asset=File(BytesIO(b'original content'), 'test.png')).save()
         Asset(key='logo', asset=File(BytesIO(b'logo'), 'logo.png')).save()
 
     populate_site()
     call_command('import_site', filename)  # default behaviour
     assert Page.objects.count() == 1
+    assert Page.objects.first().picture.name == 'page-pictures/picture.png'
+    assert os.path.isfile('%s/page-pictures/picture.png' % default_storage.path(''))
     assert Asset.objects.count() == 3
     assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
     with open('%s/assets/test.png' % default_storage.path('')) as fd:
@@ -428,6 +430,8 @@ def test_import_export_tar(tmpdir, some_assets):
     populate_site()
     call_command('import_site', filename, '--overwrite')
     assert Page.objects.count() == 1
+    assert Page.objects.first().picture.name == 'page-pictures/picture.png'
+    assert os.path.isfile('%s/page-pictures/picture.png' % default_storage.path(''))
     assert Asset.objects.count() == 3
     assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
     with open('%s/assets/test.png' % default_storage.path('')) as fd:
@@ -436,6 +440,8 @@ def test_import_export_tar(tmpdir, some_assets):
     populate_site()
     call_command('import_site', filename, '--if-empty')
     assert Page.objects.count() == 1
+    assert Page.objects.first().picture.name == 'page-pictures/picture.png'
+    assert os.path.isfile('%s/page-pictures/picture.png' % default_storage.path(''))
     assert Asset.objects.count() == 2
     assert Asset.objects.get(key='banner').asset.name == 'assets/test.png'
     with open('%s/assets/test.png' % default_storage.path('')) as fd:
@@ -452,6 +458,7 @@ def test_import_export_tar(tmpdir, some_assets):
         assert fd.read() == 'test'
     assert not Asset.objects.filter(key='logo')
     assert not os.path.isfile('%s/assets/logo.png' % default_storage.path(''))
+    assert not os.path.isfile('%s/page-pictures/picture.png' % default_storage.path(''))
 
     # error cases
     with pytest.raises(CommandError, match=r'No such file or directory'):