misc: ignore files used in blocks when looking for unused files (#46431)

tests/test_formdef.py

@@ -14,6 +14,7 @@ from django.utils.encoding import force_bytes
 from django.utils.six import BytesIO
 from quixote import cleanup
 from wcs import fields
+from wcs.blocks import BlockDef
 from wcs.formdef import FormDef, get_formdefs_of_all_kinds
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.form import PicklableUpload
@@ -264,11 +265,21 @@ def test_page_field_migration(pub):
 def test_unused_file_removal_job(pub):
     from wcs.formdef import clean_unused_files
 
+    BlockDef.wipe()
     FormDef.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.FileField(id='234', required=True, label='Test2', type='file'),
+    ]
+    block.store()
+
     formdef = FormDef()
     formdef.name = 'removal job'
     formdef.fields = [
         fields.FileField(id='5', label='file', varname='filefield'),
+        fields.BlockField(id='6', label='test', type='block:foobar', max_items=3),
     ]
     formdef.store()
     formdef.data_class().wipe()
@@ -350,6 +361,28 @@ def test_unused_file_removal_job(pub):
         clean_unused_files(pub)
         assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 0
 
+        # file in block field
+        formdata = formdef.data_class()()
+        formdata.just_created()
+        formdata.data = {
+            '6': {
+                'data': [
+                    {'234': PicklableUpload('test.txt', 'text/plain')},
+                    {'234': PicklableUpload('test2.txt', 'text/plain')},
+                ],
+                'schema': {'234': 'file'},
+            },
+        }
+        formdata.data['6']['data'][0]['234'].receive([b'hello world'])
+        formdata.data['6']['data'][1]['234'].receive([b'hello world block'])
+        formdata.store()
+        assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 2
+        clean_unused_files(pub)
+        assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 2
+        formdata.remove_self()
+        clean_unused_files(pub)
+        assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 0
+
         # non local storage: nothing happens
         formdata = formdef.data_class()()
         formdata.just_created()
@@ -402,8 +435,8 @@ def test_unused_file_removal_job(pub):
         assert len(os.listdir(os.path.join(pub.app_dir, 'uploads'))) == 0
 
         if behaviour == 'move':
-            # 2 files ("hello world" + "hello world 2")
-            assert len(os.listdir(os.path.join(pub.app_dir, 'unused-files/uploads/'))) == 2
+            # 3 files ("hello world" + "hello world 2" + "hello world block")
+            assert len(os.listdir(os.path.join(pub.app_dir, 'unused-files/uploads/'))) == 3
             # 1 attachment
             assert len(glob.glob(os.path.join(pub.app_dir, 'unused-files/attachments/*/*'))) == 1
 
@@ -431,6 +464,7 @@ def test_get_formdefs_of_all_kinds(pub):
     from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
     from wcs.workflows import WorkflowBackofficeFieldsFormDef, WorkflowVariablesFieldsFormDef
 
+    BlockDef.wipe()
     FormDef.wipe()
     Workflow.wipe()
     formdefs = get_formdefs_of_all_kinds()

wcs/formdef.py

@@ -1625,6 +1625,11 @@ def clean_unused_files(publisher):
                         (formdata.workflow_data or {}).values()):
                     if is_upload(field_data):
                         yield field_data.get_fs_filename()
+                    elif isinstance(field_data, dict) and isinstance(field_data.get('data'), list):
+                        for subfield_rowdata in field_data.get('data'):
+                            for field_data in subfield_rowdata.values():
+                                if is_upload(field_data):
+                                    yield field_data.get_fs_filename()
                 for part in formdata.iter_evolution_parts():
                     if is_attachment(part):
                         yield part.filename