misc: display after job progression (#46672)

tests/api/test_all.py

@@ -246,6 +246,6 @@ def test_afterjobs_status(pub):
             'status': 'registered',
             'creation_time': job.creation_time,
             'completion_time': None,
-            'completion_status': None,
+            'completion_status': '',
         },
     }

tests/backoffice_pages/test_export.py

@@ -221,7 +221,7 @@ def test_backoffice_export_long_listings(pub, threshold):
 
     # check afterjob ajax call
     status_resp = app.get('/afterjobs/' + job_id)
-    assert status_resp.text == 'completed|completed'
+    assert status_resp.text == 'completed|completed 2/2 (100%)'
 
     # check error handling
     app.get('/afterjobs/whatever', status=404)

wcs/admin/settings.py

@@ -1037,6 +1037,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                             if f in ('.indexes', '.max_id'):
                                 continue
                             z.write(os.path.join(path, f), os.path.join(d, f))
+                            job.increment_count()
                     if 'datasources' in self.dirs:
                         for ds in NamedDataSource.select():
                             if ds.external == 'agenda':
@@ -1047,6 +1048,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('datasources', str(ds.id)),
                                 ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'formdefs' in self.dirs:
                         for formdef in FormDef.select():
                             node = formdef.export_to_xml(include_id=True)
@@ -1055,6 +1057,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('formdefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'carddefs' in self.dirs:
                         for formdef in CardDef.select():
                             node = formdef.export_to_xml(include_id=True)
@@ -1063,6 +1066,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('carddefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'workflows' in self.dirs:
                         for workflow in Workflow.select():
                             node = workflow.export_to_xml(include_id=True)
@@ -1071,6 +1075,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('workflows_xml', str(workflow.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'blockdefs' in self.dirs:
                         for blockdef in BlockDef.select():
                             node = blockdef.export_to_xml(include_id=True)
@@ -1079,6 +1084,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('blockdefs_xml', str(blockdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'roles' in self.dirs:
                         for role in get_publisher().role_class.select():
                             node = role.export_to_xml(include_id=True)
@@ -1087,16 +1093,21 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('roles_xml', str(role.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
 
                     if self.settings:
                         z.write(os.path.join(self.app_dir, 'config.pck'), 'config.pck')
+                        job.increment_count()
                         for f in os.listdir(self.app_dir):
                             if f.startswith('idp-') and os.path.splitext(f)[-1] in ('.pem', '.xml'):
                                 z.write(os.path.join(self.app_dir, f), f)
+                                job.increment_count()
                         if os.path.exists(os.path.join(self.app_dir, 'config')):
                             for f in os.listdir(os.path.join(self.app_dir, 'config')):
                                 z.write(os.path.join(self.app_dir, 'config', f), os.path.join('config', f))
+                                job.increment_count()
 
+                job.total_count = job.current_count
                 job.file_content = c.getvalue()
                 job.store()
 

wcs/api.py

@@ -1172,7 +1172,7 @@ class AfterJobDirectory(Directory):
                     'label': self.afterjob.label,
                     'creation_time': self.afterjob.creation_time,
                     'completion_time': self.afterjob.completion_time,
-                    'completion_status': self.afterjob.completion_status,
+                    'completion_status': self.afterjob.get_completion_status(),
                 },
             },
             cls=misc.JSONEncoder,

wcs/backoffice/data_management.py

@@ -363,6 +363,9 @@ class ImportFromCsvAfterJob(AfterJob):
     def execute(self):
         self.carddef = self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])
         carddata_class = self.carddef.data_class()
+        self.total_count = len(self.kwargs['data_lines'])
+        self.store()
+
         for item in self.kwargs['data_lines']:
             data_instance = carddata_class()
             user_value = item.pop('_user', None)
@@ -371,6 +374,7 @@ class ImportFromCsvAfterJob(AfterJob):
             data_instance.just_created()
             data_instance.store()
             data_instance.perform_workflow(event='csv-import-created')
+            self.increment_count()
 
     def done_action_url(self):
         carddef = self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])

wcs/backoffice/management.py

@@ -3894,7 +3894,7 @@ class MassActionAfterJob(AfterJob):
             return
 
         formdatas = formdef.data_class().get_ids(item_ids, order_by='receipt_time')
-        self.completion_status = f'0/{len(formdatas)}'
+        self.total_count = len(formdatas)
         self.store()
 
         publisher = get_publisher()
@@ -3910,7 +3910,7 @@ class MassActionAfterJob(AfterJob):
                 {
                     'oldest_form': oldest_lazy_form,
                     'mass_action_index': i,
-                    'mass_action_length': len(formdatas),
+                    'mass_action_length': self.total_count,
                 }
             )
             if getattr(action['action'], 'status_action', False):
@@ -3921,8 +3921,7 @@ class MassActionAfterJob(AfterJob):
             else:
                 # global action
                 formdata.perform_global_action(action['action'].id, user)
-            self.completion_status = f'{i + 1}/{len(formdatas)}'
-            self.store()
+            self.increment_count()
 
     def done_action_url(self):
         return self.kwargs['return_url']
@@ -3970,6 +3969,8 @@ class CsvExportAfterJob(AfterJob):
         items, total_count = FormDefUI(formdef).get_listing_items(
             fields, selected_filter, user=user, query=query, criterias=criterias, order_by=order_by
         )
+        self.total_count = total_count
+        self.store()
 
         return self.create_export(formdef, fields, items, total_count)
 
@@ -3982,6 +3983,7 @@ class CsvExportAfterJob(AfterJob):
 
         for filled in items:
             csv_output.writerow(tuple(x['value'] for x in self.get_spreadsheet_line(fields, filled)))
+            self.increment_count()
 
         self.file_content = output.getvalue()
         self.content_type = 'text/csv'
@@ -4025,6 +4027,7 @@ class OdsExportAfterJob(CsvExportAfterJob):
                     data_field=item['field'],
                     native_value=item['native_value'],
                 )
+            self.increment_count()
 
         output = io.BytesIO()
         workbook.save(output)

wcs/qommon/afterjobs.py

@@ -35,9 +35,10 @@ class AfterJobStatusDirectory(Directory):
 
         response = get_response()
         response.set_content_type('text/plain')
-        if not job.completion_status:
+        completion_status = job.get_completion_status()
+        if not completion_status:
             return job.status + '|' + str(_(job.status))
-        return job.status + '|' + str(_(job.status)) + ' ' + job.completion_status
+        return job.status + '|' + str(_(job.status)) + ' ' + str(completion_status)
 
 
 class AfterJob(StorableObject):
@@ -48,7 +49,8 @@ class AfterJob(StorableObject):
     status = None
     creation_time = None
     completion_time = None
-    completion_status = None
+    current_count = None
+    total_count = None
 
     execute = None
 
@@ -73,6 +75,22 @@ class AfterJob(StorableObject):
     def done_action_attributes(self):
         return self.done_button_attributes_arg
 
+    def increment_count(self):
+        self.current_count = (self.current_count or 0) + 1
+        self.store()
+
+    def get_completion_status(self):
+        current_count = self.current_count or 0
+
+        if not current_count:
+            return ''
+
+        if not self.total_count:
+            return _(f'{current_count} (unknown total)')
+
+        percent = int(current_count * 100 / self.total_count)
+        return _(f'{current_count}/{self.total_count} ({percent}%)')
+
     def run(self, spool=False):
         if self.completion_time:
             return