workflow_tests: apply global action timeout trigger on skip time (#88404)

debian/control

@@ -28,6 +28,7 @@ Depends: graphviz,
          python3-lasso,
          python3-lxml,
          python3-pil,
+         python3-psutil,
          python3-psycopg2,
          python3-pyproj,
          python3-quixote,

setup.py

@@ -204,6 +204,7 @@ setup(
         'setproctitle',
         'phonenumbers',
         'emoji',
+        'psutil',
         'freezegun',
     ],
     package_dir={'wcs': 'wcs'},

tests/admin_pages/test_workflow.py

@@ -8,7 +8,7 @@ import responses
 from pyquery import PyQuery
 from webtest import Upload
 
-from wcs import fields
+from wcs import fields, workflow_tests
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import WorkflowCategory
@@ -17,6 +17,7 @@ from wcs.mail_templates import MailTemplate
 from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.errors import ConnectionError
 from wcs.qommon.http_request import HTTPRequest
+from wcs.testdef import TestDef, TestResult
 from wcs.wf.create_formdata import CreateFormdataWorkflowStatusItem, Mapping
 from wcs.wf.form import WorkflowFormFieldsFormDef
 from wcs.workflows import (
@@ -633,7 +634,10 @@ def test_workflows_delete_status_reassign(pub, name):
             resp = resp.follow()
             assert formdefs[-1].data_class().get(formdata2.id).status == 'wf-%s' % wf_bar.id
 
-        assert AfterJob.count() == 3  # status change + rebuild_security + tests
+        if name in ('forms', 'cards'):
+            assert AfterJob.count() == 3  # status change + rebuild_security + form or card tests
+        else:
+            assert AfterJob.count() == 4  # status change + rebuild_security + card tests + form tests
         resp = resp.click('Back')
         assert resp.request.path == f'/backoffice/workflows/{workflow.id}/'
 
@@ -4357,3 +4361,58 @@ def test_workflows_function_and_role_with_same_name(pub):
         (str(role1.id), False, 'Foo'),
         (str(role2.id), False, 'Foobar [role]'),  # same name as function -> role suffix
     ]
+
+
+def test_workflow_test_results(pub):
+    create_superuser(pub)
+    TestDef.wipe()
+    TestResult.wipe()
+
+    Workflow.wipe()
+    workflow = Workflow(name='Workflow One')
+    workflow.add_status(name='New status')
+    workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.workflow_id = workflow.id
+    formdef.name = 'test title'
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test'
+    resp = resp.form.submit('submit').follow()
+    assert TestResult.count() == 0
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+
+    testdef = TestDef.create_from_formdata(formdef, formdata)
+    testdef.store()
+
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test 2'
+    resp = resp.form.submit('submit').follow()
+    assert TestResult.count() == 0
+
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertStatus(id='1'),
+    ]
+    testdef.store()
+
+    resp = app.get('/backoffice/workflows/1/edit')
+    resp.form['name'] = 'test 3'
+    resp = resp.form.submit('submit').follow()
+
+    assert TestResult.count() == 1
+    result = TestResult.select()[0]
+    assert result.reason == 'Change in workflow'
+
+    resp = resp.click('add status')
+    resp.forms[0]['name'] = 'new status'
+    resp = resp.forms[0].submit()
+
+    assert TestResult.count() == 2
+    result = TestResult.select(order_by='id')[1]
+    assert result.reason == 'Workflow: New status "new status"'

tests/admin_pages/test_workflow_tests.py

@@ -170,22 +170,38 @@ def test_workflow_tests_edit_actions(pub):
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
-    assert resp.text.count(escape('Click on "Accept"')) == 1
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Duplicate').follow()
-    assert resp.text.count(escape('Click on "Accept"')) == 2
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Edit', index=0)
     resp.form['button_name'] = 'Reject'
     resp = resp.form.submit().follow()
 
-    assert resp.text.count(escape('Click on "Accept"')) == 1
-    assert resp.text.count(escape('Click on "Reject"')) == 1
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
+
+    resp = resp.click('Duplicate', index=0).follow()
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     resp = resp.click('Delete', index=0)
     resp = resp.form.submit().follow()
-    assert resp.text.count(escape('Click on "Accept"')) == 1
-    assert resp.text.count(escape('Click on "Reject"')) == 0
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Reject" by backoffice user',
+        'Click on "Accept" by backoffice user',
+    ]
 
     # simulate invalid action
     testdef = TestDef.get(testdef.id)
@@ -193,7 +209,9 @@ def test_workflow_tests_edit_actions(pub):
     testdef.store()
 
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
-    assert 'There are no workflow test actions yet.' in resp.text
+    assert [x.text for x in resp.pyquery('ul li.workflow-test-action span.type')] == [
+        'Click on "Accept" by backoffice user',
+    ]
 
 
 def test_workflow_tests_action_button_click(pub):
@@ -379,6 +397,20 @@ def test_workflow_tests_action_assert_email(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     assert escape('Email to "a@entrouvert.com" (+2)') in resp.text
 
+    assert_email.addresses = []
+    assert_email.subject_strings = ['Hello your form has been submitted']
+    assert_email.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert escape('Subject must contain "Hello your form has been su(…)"') in resp.text
+
+    assert_email.subject_strings = []
+    assert_email.body_strings = ['Hello your form has been submitted']
+    assert_email.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert escape('Body must contain "Hello your form has been su(…)"') in resp.text
+
 
 def test_workflow_tests_action_assert_sms(pub):
     create_superuser(pub)
@@ -408,14 +440,14 @@ def test_workflow_tests_action_assert_sms(pub):
 
     resp = resp.click('Edit')
     resp.form['phone_numbers$element0'] = '0123456789'
-    resp.form['body'] = 'Hello'
+    resp.form['body'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'SMS to 0123456789' in resp.text
 
     assert_sms = TestDef.get(testdef.id).workflow_tests.actions[0]
     assert assert_sms.phone_numbers == ['0123456789']
-    assert assert_sms.body == 'Hello'
+    assert assert_sms.body == 'Hello your form has been submitted'
 
     assert_sms.phone_numbers = ['0123456789', '0123456781', '0123456782']
     assert_sms.parent.store()
@@ -423,6 +455,12 @@ def test_workflow_tests_action_assert_sms(pub):
     resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
     assert escape('SMS to 0123456789 (+2)') in resp.text
 
+    assert_sms.phone_numbers = []
+    assert_sms.parent.store()
+
+    resp = app.get('/backoffice/forms/1/tests/%s/workflow/' % testdef.id)
+    assert 'Hello your form has been su(…)' in resp.text
+
 
 def test_workflow_tests_action_assert_anonymise(pub):
     create_superuser(pub)
@@ -497,10 +535,11 @@ def test_workflow_tests_action_assert_history_message(pub):
     assert 'not configured' in resp.text
 
     resp = resp.click('Edit')
-    resp.form['message'] = 'Hello'
+    resp.form['message'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
+    assert 'Hello your form has been su(…)' in resp.text
 
 
 def test_workflow_tests_action_assert_alert(pub):
@@ -525,10 +564,11 @@ def test_workflow_tests_action_assert_alert(pub):
     assert 'not configured' in resp.text
 
     resp = resp.click('Edit')
-    resp.form['message'] = 'Hello'
+    resp.form['message'] = 'Hello your form has been submitted'
     resp = resp.form.submit().follow()
 
     assert 'not configured' not in resp.text
+    assert 'Hello your form has been su(…)' in resp.text
 
 
 def test_workflow_tests_action_assert_criticality(pub):

tests/form_pages/test_all.py

@@ -3273,6 +3273,48 @@ def test_workflow_message_with_template_error(pub):
     assert logged_error.summary == "Error in template of workflow message ('int' object is not iterable)"
 
 
+def test_workflow_condition_on_message_age_in_hours(pub, freezer):
+    create_user(pub)
+
+    formdef = create_formdef()
+    formdef.fields = []
+    formdef.store()
+
+    workflow = Workflow(name='test')
+    st1 = workflow.add_status('Status1', 'st1')
+
+    display1 = st1.add_action('displaymsg')
+    display1.message = 'message-to-all'
+    display1.to = []
+
+    workflow.store()
+
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub), username='foo', password='foo')
+    page = app.get('/test/')
+    page = page.forms[0].submit('submit')  # form page
+    page = page.forms[0].submit('submit')  # confirmation page
+    page = page.follow()
+    assert 'message-to-all' in page.text
+
+    formdata = formdef.data_class().select()[0]
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' in page.text
+
+    display1.condition = {'type': 'django', 'value': 'form_receipt_datetime|age_in_hours >= 1'}
+    workflow.store()
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' not in page.text
+
+    freezer.tick(60 * 60)
+    page = app.get(formdata.get_url())
+    assert 'message-to-all' in page.text
+
+
 def test_session_cookie_flags(pub):
     create_formdef()
     app = get_app(pub)

tests/form_pages/test_draft.py

@@ -857,3 +857,25 @@ def test_draft_store_page_id_when_no_page_and_no_confirmation(pub):
     assert formdef.data_class().count() == 1
     formdata = formdef.data_class().select()[0]
     assert formdata.status == 'wf-new'
+
+
+def test_draft_error_then_autosave(pub):
+    formdef = create_formdef()
+    formdef.enable_tracking_codes = True
+    formdef.fields = [
+        fields.PageField(id='0', label='1st page'),
+        fields.StringField(id='1', label='string 1'),
+        fields.PageField(id='2', label='2nd page'),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = get_app(pub)
+    resp = app.get('/test/')
+    resp = resp.form.submit('submit')  # error
+    assert formdef.data_class().count() == 1  # server roundtrip -> draft
+
+    resp.form['f1'] = 'test'
+    app.post('/test/autosave', params=resp.form.submit_fields())
+    assert formdef.data_class().count() == 1  # make sure same draft got reused
+    assert formdef.data_class().select()[0].data['1'] == 'test'

tests/test_carddef.py

@@ -624,6 +624,7 @@ def test_data_source_custom_view_digest(pub):
         'custom-view:view': '{{ form_var_foo }} Foo Bar',
     }
     carddef.store()
+    pub.reset_caches()
     # rebuild digests
     carddata.store()
     carddata2.store()
@@ -761,6 +762,7 @@ def test_get_data_source_custom_view_order_by(pub):
     ]
     carddef.digest_templates['custom-view:view'] = '{{ form_var_bar }}'
     carddef.store()
+    pub.reset_caches()
     for carddata in carddef.data_class().select():
         carddata.store()  # rebuild digests
     assert [i['text'] for i in CardDef.get_data_source_items('carddef:foo:view')] == [
@@ -1386,6 +1388,7 @@ def test_card_update_related(pub):
         ItemsField(id='1', label='Test', data_source={'type': 'carddef:foo'}),
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {'1': ['1', '2']}
@@ -1419,6 +1422,7 @@ def test_card_update_related(pub):
         BlockField(id='2', label='Test2', block_slug=blockdef.slug),  # left empty
     ]
     formdef.store()
+    pub.reset_caches()
 
     formdata = formdef.data_class()()
     formdata.data = {

tests/test_formdata.py

@@ -1942,14 +1942,14 @@ def test_lazy_formdata_queryset_filter(pub, variable_test_data):
     assert tmpl.render(context) == 'None'
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
-    assert logged_error.summary == '|pending used on invalid queryset (\'\')'
+    assert logged_error.summary == '|pending used on something else than a queryset (\'\')'
 
     pub.loggederror_class.wipe()
     tmpl = Template('{{""|filter_value:"foo"}}')
     assert tmpl.render(context) == 'None'
     assert pub.loggederror_class.count() == 1
     logged_error = pub.loggederror_class.select()[0]
-    assert logged_error.summary == '|filter_value used on invalid queryset (\'\')'
+    assert logged_error.summary == '|filter_value used on something else than a queryset (\'\')'
 
 
 def test_lazy_formdata_queryset_filter_non_unique_varname(pub, variable_test_data):
@@ -5776,6 +5776,7 @@ def test_reverse_links(pub):
 
     # test reverse relation
     carddef1.store()  # build & store reverse_relations
+    pub.reset_caches()
     pub.substitutions.reset()
     pub.substitutions.feed(pub)
     pub.substitutions.feed(carddef1)
@@ -5790,6 +5791,7 @@ def test_reverse_links(pub):
     # test with natural id
     carddef1.id_template = 'X{{ form_var_name1 }}Y'
     carddef1.store()
+    pub.reset_caches()
     carddata1.store()
     assert carddata1.id_display == 'Xfoo1Y'
     carddata2.data['1'] = carddata1.get_natural_key()

tests/test_publisher.py

@@ -552,6 +552,39 @@ def test_cron_command_job_exception(settings):
     clean_temporary_pub()
 
 
+def test_cron_command_job_log(settings):
+    pub = create_temporary_pub()
+
+    def job1(pub, job=None):
+        job.log('hello')
+        job.log_debug('debug')
+
+    @classmethod
+    def register_test_cronjobs(cls):
+        cls.register_cronjob(CronJob(job1, name='job1', days=[10]))
+
+    get_publisher().set_tenant_by_hostname('example.net')
+    sql.mark_cron_status('done')
+
+    with mock.patch('wcs.publisher.WcsPublisher.register_cronjobs', register_test_cronjobs):
+        get_publisher_class().cronjobs = []
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net') == ['start', "running jobs: ['job1']", 'hello']
+
+        pub.load_site_options()
+        pub.site_options.set('options', 'cron-log-level', 'debug')
+        with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+            pub.site_options.write(fd)
+
+        clear_log_files()
+        call_command('cron', job_name='job1', domain='example.net')
+        assert get_logs('example.net')[:3] == ['start', "running jobs: ['job1']", 'hello']
+        assert re.match(r'\(mem: .*\) debug', get_logs('example.net')[3])
+
+    clean_temporary_pub()
+
+
 def test_clean_afterjobs():
     pub = create_temporary_pub()
 

tests/test_storage.py

@@ -31,6 +31,15 @@ class Foobar(StorableObject):
     unique_value = None
 
 
+class Foobar2(StorableObject):
+    _names = 'tests%s' % random.randint(0, 100000)
+    _indexes = ['unique_value']
+    _hashed_indexes = ['value']
+
+    value = None
+    unique_value = None
+
+
 def test_store():
     test = Foobar()
     test.value = 'value'
@@ -307,3 +316,30 @@ def test_umask():
     cache_umask()
     test.store()
     assert (os.stat(test.get_object_filename()).st_mode % 0o1000) == 0o664
+
+
+def test_publisher_cache():
+    pub.reset_caches()
+
+    Foobar.wipe()
+    Foobar2.wipe()
+
+    test = Foobar()
+    test.value = 'value'
+    test.unique_value = 'unique-value'
+    test.store()
+
+    test2 = Foobar2()
+    test2.value = 'value'
+    test2.unique_value = 'unique-value'
+    test2.store()
+
+    test = Foobar.cached_get('1')
+    assert test.value == 'value'
+    assert Foobar.cached_get('1') is test  # same object
+
+    assert Foobar.get_on_index('unique-value', 'unique_value') is not test
+    assert Foobar.get_on_index('unique-value', 'unique_value', use_cache=True) is test
+
+    assert Foobar2.cached_get('1') is not test
+    assert Foobar2.cached_get('1') is Foobar2.get_on_index('unique-value', 'unique_value', use_cache=True)

tests/test_workflow_tests.py

@@ -739,6 +739,12 @@ def test_workflow_tests_sms(pub):
 
     testdef = TestDef.create_from_formdata(formdef, formdata)
     testdef.agent_id = user.id
+    testdef.workflow_tests.actions = [
+        workflow_tests.AssertSMS(),
+    ]
+
+    testdef.run(formdef)
+
     testdef.workflow_tests.actions = [
         workflow_tests.AssertSMS(phone_numbers=['0123456789'], body='Hello'),
     ]
@@ -1211,6 +1217,9 @@ def test_workflow_tests_webservice_status_jump(pub):
 
 
 def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
+    pub.cfg['sms'] = {'sender': 'xxx', 'passerelle_url': 'http://passerelle.invalid/'}
+    pub.write_cfg()
+
     role = pub.role_class(name='test role')
     role.store()
     user = create_user(pub, is_admin=True)
@@ -1303,6 +1312,7 @@ def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
     assert formdata.status == 'wf-end-status'
 
     testdef = TestDef.create_from_formdata(formdef, formdata, add_workflow_tests=True)
+    testdef.agent_id = user.id
     testdef.run(formdef)
 
     actions = testdef.workflow_tests.actions
@@ -1337,3 +1347,70 @@ def test_workflow_tests_create_from_formdata(pub, http_requests, freezer):
 
     assert actions[-1].key == 'assert-status'
     assert actions[-1].status_name == 'End status'
+
+
+def test_workflow_tests_create_from_formdata_multiple_buttons(pub, http_requests):
+    role = pub.role_class(name='test role')
+    role.store()
+    user = create_user(pub, is_admin=True)
+    user.roles = [role.id]
+    user.store()
+
+    workflow = Workflow(name='Workflow One')
+    new_status = workflow.add_status('New status', 'new-status')
+    middle_status = workflow.add_status('Middle status', 'middle-status')
+    end_status = workflow.add_status('End status', 'end-status')
+
+    choice = new_status.add_action('choice')
+    choice.label = 'Go to middle status'
+    choice.status = middle_status.id
+    choice.by = [role.id]
+
+    choice = middle_status.add_action('choice')
+    choice.label = 'Go to end status'
+    choice.status = end_status.id
+    choice.by = [role.id]
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'test title'
+    formdef.workflow_id = workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.user_id = user.id
+    formdata.just_created()
+    formdata.store()
+
+    formdata.perform_workflow()
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get(formdata.get_url())
+    resp = resp.form.submit('button1').follow()
+    resp = resp.form.submit('button1').follow()
+    formdata.refresh_from_storage()
+    assert formdata.status == 'wf-end-status'
+
+    testdef = TestDef.create_from_formdata(formdef, formdata, add_workflow_tests=True)
+    testdef.agent_id = user.id
+    testdef.run(formdef)
+
+    actions = testdef.workflow_tests.actions
+    assert len(actions) == 5
+
+    assert actions[0].key == 'assert-status'
+    assert actions[0].status_name == 'New status'
+
+    assert actions[1].key == 'button-click'
+    assert actions[1].button_name == 'Go to middle status'
+
+    assert actions[2].key == 'assert-status'
+    assert actions[2].status_name == 'Middle status'
+
+    assert actions[3].key == 'button-click'
+    assert actions[3].button_name == 'Go to end status'
+
+    assert actions[4].key == 'assert-status'
+    assert actions[4].status_name == 'End status'

tests/utilities.py

@@ -146,6 +146,7 @@ def create_temporary_pub(pickle_mode=False, lazy_mode=False):
             sql.Audit.wipe()
             sql_mark_current_test()
         pub.write_cfg()
+        pub.reset_caches()
         return pub
 
     os.symlink(os.path.join(os.path.dirname(__file__), 'templates'), os.path.join(pub.app_dir, 'templates'))

tests/workflow/test_jump.py

@@ -1,7 +1,9 @@
 import datetime
+import os
 from unittest import mock
 
 import pytest
+from django.core.management import call_command
 from pyquery import PyQuery
 from quixote import cleanup
 
@@ -11,6 +13,7 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
 from wcs.workflows import Workflow, perform_items
 
+from ..test_publisher import get_logs
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import admin_user  # noqa pylint: disable=unused-import
 
@@ -629,3 +632,44 @@ def test_jump_self_timeout(pub):
     formdata.store()
     formdata.record_workflow_event('backoffice-created')
     _apply_timeouts(pub)
+
+
+def test_timeout_cron_debug_log(pub):
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='timeout')
+    st1 = workflow.add_status('Status1', 'st1')
+    workflow.add_status('Status2', 'st2')
+
+    jump = st1.add_action('jump', id='_jump')
+    jump.by = ['_submitter', '_receiver']
+    jump.timeout = 30 * 60  # 30 minutes
+    jump.status = 'st2'
+
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.workflow_id = workflow.id
+    assert formdef.get_workflow().id == workflow.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    rewind(formdata, seconds=40 * 60)
+    formdata.store()
+    formdata_id = formdata.id
+
+    pub.load_site_options()
+    pub.site_options.set('options', 'cron-log-level', 'debug')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    call_command('cron', job_name='evaluate_jumps', domain='example.net', force_job=True)
+
+    assert formdef.data_class().get(formdata_id).status == 'wf-st2'
+    assert get_logs('example.net')[:2] == ['start', "running jobs: ['evaluate_jumps']"]
+    assert 'applying timeouts on baz' in get_logs('example.net')[2]
+    assert 'event: timeout-jump' in get_logs('example.net')[3]

wcs/admin/tests.py

@@ -651,12 +651,13 @@ class TestResultsDirectory(Directory):
 
 
 class TestsAfterJob(AfterJob):
-    def __init__(self, objectdef, reason, snapshot=None, **kwargs):
+    def __init__(self, objectdef, reason, snapshot=None, triggered_by='', **kwargs):
         super().__init__(
             objectdef_class=objectdef.__class__,
             objectdef_id=objectdef.id,
             reason=reason,
             snapshot_id=snapshot.id if snapshot else None,
+            triggered_by=triggered_by,
             **kwargs,
         )
 
@@ -667,7 +668,7 @@ class TestsAfterJob(AfterJob):
             return
         reason = self.kwargs['reason']
 
-        result = self.run_tests(objectdef, reason)
+        result = self.run_tests(objectdef, reason, self.kwargs.get('triggered_by', ''))
 
         if result and self.kwargs['snapshot_id'] is not None:
             snapshot = get_publisher().snapshot_class.get(self.kwargs['snapshot_id'])
@@ -675,11 +676,14 @@ class TestsAfterJob(AfterJob):
             snapshot.store()
 
     @staticmethod
-    def run_tests(objectdef, reason):
+    def run_tests(objectdef, reason, triggered_by=''):
         testdefs = TestDef.select_for_objectdef(objectdef)
         if not testdefs:
             return
 
+        if triggered_by == 'workflow-change' and not any(x.workflow_tests.actions for x in testdefs):
+            return
+
         for test in testdefs:
             try:
                 test.run(objectdef)

wcs/admin/workflow_tests.py

@@ -95,7 +95,8 @@ class WorkflowTestActionPage(Directory):
     def duplicate(self):
         new_action = copy.deepcopy(self.action)
         new_action.id = self.testdef.workflow_tests.get_new_action_id()
-        self.testdef.workflow_tests.actions.append(new_action)
+        action_position = self.testdef.workflow_tests.actions.index(self.action)
+        self.testdef.workflow_tests.actions.insert(action_position + 1, new_action)
         self.testdef.store()
         return redirect('..')
 

wcs/carddata.py

@@ -156,7 +156,7 @@ class UpdateRelationsAfterJob(AfterJob):
         update_related_seen = get_publisher()._update_related_seen
 
         try:
-            carddef = CardDef.get(self.kwargs['carddef_id'])
+            carddef = CardDef.cached_get(self.kwargs['carddef_id'])
             carddata = carddef.data_class().get(self.kwargs['carddata_id'])
         except KeyError:
             # card got removed (probably the afterjob met some unexpected delay), ignore.
@@ -169,7 +169,7 @@ class UpdateRelationsAfterJob(AfterJob):
             obj_type, obj_slug = obj_ref.split(':')
             obj_class = klass.get(obj_type)
             try:
-                objdef = obj_class.get_by_slug(obj_slug)
+                objdef = obj_class.get_by_slug(obj_slug, use_cache=True)
             except KeyError:
                 continue
             criterias = []

wcs/carddef.py

@@ -204,7 +204,7 @@ class CardDef(FormDef):
         assert data_source_id.startswith('carddef:')
         parts = data_source_id.split(':')
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         criterias = [StrictNotEqual('status', 'draft'), Null('anonymised')]
@@ -299,7 +299,7 @@ class CardDef(FormDef):
         if len(parts) != 3:
             return []
         try:
-            carddef = cls.get_by_urlname(parts[1])
+            carddef = cls.get_by_urlname(parts[1], use_cache=True)
         except KeyError:
             return []
         custom_view = cls.get_data_source_custom_view(data_source_id, carddef=carddef)

wcs/formdata.py

@@ -1940,7 +1940,7 @@ class FormData(StorableObject):
                     elif obj_type == 'carddef':
                         obj_class = CardDef
                     try:
-                        _objectdef = obj_class.get_by_urlname(slug)
+                        _objectdef = obj_class.get_by_urlname(slug, use_cache=True)
                     except KeyError:
                         yield (
                             _('Linked object def by id %(object_id)s') % {'object_id': slug},

wcs/formdef.py

@@ -788,9 +788,13 @@ class FormDef(StorableObject):
         self.workflow_options.update(variables)
 
     @classmethod
-    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False):
+    def get_by_urlname(cls, url_name, ignore_migration=False, ignore_errors=False, use_cache=False):
         return cls.get_on_index(
-            url_name, 'url_name', ignore_migration=ignore_migration, ignore_errors=ignore_errors
+            url_name,
+            'url_name',
+            ignore_migration=ignore_migration,
+            ignore_errors=ignore_errors,
+            use_cache=use_cache,
         )
 
     get_by_slug = get_by_urlname

wcs/forms/root.py

@@ -1372,6 +1372,9 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
                 # a new ConditionsVars will get added to the substitution
                 # variables.
                 form_data = copy.copy(session.get_by_magictoken(magictoken, {}))
+                if form_data:
+                    # keep new copy in session
+                    session.add_magictoken(magictoken, form_data)
                 try:
                     with get_publisher().substitutions.temporary_feed(transient_formdata, force_mode='lazy'):
                         data = self.formdef.get_data(form, raise_on_error=True)

wcs/locale/fr/LC_MESSAGES/django.po

@@ -4,8 +4,8 @@ msgid ""
 msgstr ""
 "Project-Id-Version: wcs 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-04-01 18:14+0200\n"
-"PO-Revision-Date: 2024-04-01 18:14+0200\n"
+"POT-Creation-Date: 2024-04-02 12:58+0200\n"
+"PO-Revision-Date: 2024-04-02 11:55+0200\n"
 "Last-Translator: Thomas Noël <tnoel@entrouvert.com>\n"
 "Language-Team: french\n"
 "Language: fr\n"
@@ -1724,14 +1724,6 @@ msgstr "Valeur par défaut : %s"
 msgid "Change in workflow variables"
 msgstr "Changement dans les variables de workflow"
 
-#: admin/forms.py
-msgid "Workflow Options"
-msgstr "Options du workflow"
-
-#: admin/forms.py
-msgid "Change in workflow options"
-msgstr "Changement dans les options de workflow"
-
 #: admin/forms.py
 msgid "Import Form"
 msgstr "Importer un formulaire"
@@ -3211,19 +3203,6 @@ msgstr "Nouveau statut « %s »"
 msgid "New Status"
 msgstr "Nouveau statut"
 
-#: admin/workflows.py
-msgid "or you can use this field to directly replace a workflow parameter:"
-msgstr ""
-"ou vous pouvez utiliser ce champ pour remplacer un paramètre du workflow :"
-
-#: admin/workflows.py
-msgid "This takes priority over a variable name"
-msgstr "Ce choix est prioritaire par rapport au nom de variable"
-
-#: admin/workflows.py qommon/substitution.py
-msgid "Variable"
-msgstr "Variable"
-
 #: admin/workflows.py
 msgid "Default Value"
 msgstr "Valeur par défaut"
@@ -9085,6 +9064,10 @@ msgstr "Impossibilité de communiquer avec le fournisseur d’identités."
 msgid "Authentication error"
 msgstr "Erreur d’authentification"
 
+#: qommon/substitution.py
+msgid "Variable"
+msgstr "Variable"
+
 #: qommon/template.py
 msgid "the homepage"
 msgstr "la page d’accueil"
@@ -9264,8 +9247,8 @@ msgstr "{%% temporary_action_button %%} nécessite un paramètre « label »"
 
 #: qommon/templatetags/qommon.py
 #, python-format
-msgid "|%s used on invalid queryset (%r)"
-msgstr "|%s utilisé sur une requête invalide (%r)"
+msgid "|%s used on something else than a queryset (%r)"
+msgstr "|%s utilisé sur autre chose qu’une requête (%r)"
 
 #: qommon/templatetags/qommon.py
 #, python-format
@@ -9995,6 +9978,17 @@ msgstr " %%"
 msgid "There are no mail templates defined."
 msgstr "Il n’y a pas de modèle de courriel défini."
 
+#: templates/wcs/backoffice/includes/sql-fields-integrity.html
+msgid "There are integrity errors in the database column types."
+msgstr ""
+"Il y a des erreurs d’intégrité dans les types de colonne de la base de "
+"données."
+
+#: templates/wcs/backoffice/includes/sql-fields-integrity.html
+#, python-format
+msgid "expected: %(expected)s, got: %(got)s."
+msgstr "attendu : %(expected)s, récupéré : %(got)s."
+
 #: templates/wcs/backoffice/includes/test-result-fragment.html
 #: wf/display_message.py wf/register_comment.py
 msgid "Success"
@@ -12398,6 +12392,16 @@ msgstr "Vérifier l’envoi d’un courriel"
 msgid "Email to \"%s\""
 msgstr "Courriel vers « %s »"
 
+#: workflow_tests.py
+#, python-format
+msgid "Subject must contain \"%s\""
+msgstr "Le sujet doit contenir « %s »"
+
+#: workflow_tests.py
+#, python-format
+msgid "Body must contain \"%s\""
+msgstr "Le corps doit contenir « %s »"
+
 #: workflow_tests.py
 msgid "No email was sent."
 msgstr "Aucun courriel envoyé."
@@ -12843,6 +12847,15 @@ msgid "Reindexing cards and forms after workflow change"
 msgstr ""
 "Ré-indexation des demandes et des fiches après modification du workflow"
 
+#: workflows.py
+#, python-format
+msgid "Workflow: %s"
+msgstr "Workflow : %s"
+
+#: workflows.py
+msgid "Change in workflow"
+msgstr "Modification dans le workflow"
+
 #: workflows.py
 msgid "Previously Marked Status"
 msgstr "Statut précédemment marqué"

wcs/qommon/cron.py

@@ -20,6 +20,7 @@ import sys
 import time
 from contextlib import contextmanager
 
+import psutil
 from django.conf import settings
 from django.utils.timezone import localtime
 from quixote import get_publisher
@@ -64,8 +65,8 @@ class CronJob:
                     'long job: %s (took %s minutes, %d CPU minutes)' % (self.name, minutes, process_minutes)
                 )
 
-    @staticmethod
-    def log(message, in_tenant=True):
+    @classmethod
+    def log(cls, message, in_tenant=True):
         now = localtime()
         if in_tenant:
             base_dir = get_publisher().tenant.directory
@@ -76,6 +77,12 @@ class CronJob:
         with open(os.path.join(log_dir, 'cron.log-%s' % now.strftime('%Y%m%d')), 'a+') as fd:
             fd.write('%s [%s] %s\n' % (now.isoformat(), os.getpid(), message))
 
+    def log_debug(self, message, in_tenant=True):
+        if get_publisher().get_site_option('cron-log-level') != 'debug':
+            return
+        memory = psutil.Process().memory_info().rss / (1024 * 1024)
+        self.log(f'(mem: {memory:.1f}M) {message}', in_tenant=in_tenant)
+
     def is_time(self, timetuple):
         minutes = self.minutes
         if minutes:
@@ -121,6 +128,7 @@ def cron_worker(publisher, since, job_name=None):
     if jobs:
         CronJob.log('running jobs: %r' % sorted([x.name or x for x in jobs]))
     for job in jobs:
+        publisher.current_cron_job = job
         publisher.install_lang()
         publisher.setup_timezone()
         publisher.reset_formdata_state()

wcs/qommon/storage.py

@@ -528,7 +528,7 @@ class StorableObject:
         return cls.sort_results(objects, order_by)
 
     @classmethod
-    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False):
+    def get_on_index(cls, id, index, ignore_errors=False, ignore_migration=False, use_cache=False):
         if not cls._indexes:
             raise KeyError()
         objects_dir = cls.get_objects_dir()
@@ -536,6 +536,12 @@ class StorableObject:
         if not os.path.exists(index_dir):
             cls.rebuild_indexes()
         filename = os.path.join(index_dir, str(fix_key(id)))
+        if use_cache:
+            try:
+                object_id = os.readlink(filename).split('/')[-1]
+            except FileNotFoundError:
+                raise KeyError(id)
+            return cls.cached_get(object_id, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
         return cls.get_filename(filename, ignore_errors=ignore_errors, ignore_migration=ignore_migration)
 
     @classmethod

wcs/qommon/templatetags/qommon.py

@@ -53,7 +53,7 @@ from django.template import defaultfilters
 from django.utils import dateparse
 from django.utils.encoding import force_bytes, force_str
 from django.utils.safestring import mark_safe
-from django.utils.timezone import is_naive, make_aware
+from django.utils.timezone import is_naive, localtime, make_aware, make_naive
 
 from wcs.qommon import _, calendar, evalutils, upload_storage
 from wcs.qommon.admin.texts import TextsDirectory
@@ -355,7 +355,7 @@ def age_in_hours(value, now=None):
         if not now:
             return ''
     else:
-        now = datetime.datetime.now()
+        now = make_naive(localtime())
     return int((now - value).total_seconds() / 3600)
 
 
@@ -750,7 +750,9 @@ def decorate_queryset_filter(func, name, attr):
     @functools.wraps(func)
     def f(queryset, *args, **kwargs):
         if not hasattr(queryset, attr):
-            get_publisher().record_error(_('|%s used on invalid queryset (%r)') % (name, queryset))
+            get_publisher().record_error(
+                _('|%s used on something else than a queryset (%r)') % (name, queryset)
+            )
             return None
         return func(queryset, *args, **kwargs)
 

wcs/snapshots.py

@@ -223,7 +223,7 @@ class Snapshot:
             # else: keep serialization and ignore patch
             obj.store()
 
-            if get_response():
+            if get_response() and obj.object_type in ('formdef', 'carddef'):
                 from wcs.admin.tests import TestsAfterJob
 
                 get_response().add_after_job(

wcs/variables.py

@@ -2041,7 +2041,7 @@ class CardsSource:
 
     def __getattr__(self, attr):
         try:
-            return LazyFormDef(CardDef.get_by_urlname(attr))
+            return LazyFormDef(CardDef.get_by_urlname(attr, use_cache=True))
         except KeyError:
             raise CardDefDoesNotExist(attr)
 
@@ -2056,6 +2056,6 @@ class FormsSource:
 
     def __getattr__(self, attr):
         try:
-            return LazyFormDef(FormDef.get_by_urlname(attr))
+            return LazyFormDef(FormDef.get_by_urlname(attr, use_cache=True))
         except KeyError:
             raise FormDefDoesNotExist(attr)

wcs/wf/create_formdata.py

@@ -214,9 +214,7 @@ class LinkedFormdataEvolutionPart(EvolutionPart):
 
     @property
     def formdef(self):
-        if not hasattr(self, '_formdef'):
-            self._formdef = self.formdef_class.get(self.formdef_id, ignore_errors=True)
-        return self._formdef
+        return self.formdef_class.cached_get(self.formdef_id, ignore_errors=True, ignore_migration=True)
 
     @property
     def formdata(self):

wcs/wf/jump.py

@@ -413,6 +413,11 @@ def _apply_timeouts(publisher, **kwargs):
                 ]
                 formdatas = formdata_class.select_iterator(criterias, ignore_errors=True, itersize=200)
 
+                if job:
+                    job.log_debug(
+                        f'applying timeouts on {formdef.url_name} (id:{formdef.id}), status_id: {status_id}'
+                    )
+
                 for formdata in formdatas:
                     for jump_action in wfs_status[str(formdef.workflow_id)][formdata.status]:
                         get_publisher().reset_formdata_state()

wcs/workflow_tests.py

@@ -22,7 +22,7 @@ from django.utils.timezone import localtime
 from quixote import get_publisher, get_session
 
 from wcs import wf
-from wcs.qommon import _
+from wcs.qommon import _, misc
 from wcs.qommon.form import (
     EmailWidget,
     IntWidget,
@@ -147,6 +147,7 @@ class WorkflowTests(XmlStorableObject):
 
     def add_action(self, action_class):
         action = action_class(id=self.get_new_action_id())
+        action.parent = self
         self.actions.append(action)
         return action
 
@@ -185,7 +186,7 @@ class WorkflowTests(XmlStorableObject):
 
         if workflow_traces:
             action = self.add_action(AssertStatus)
-            action.set_attributes_from_trace(formdata.formdef, workflow_traces[-1])
+            action.status_name = formdata.get_status().name
 
     def export_actions_to_xml(self, element, attribute_name, **kwargs):
         for action in self.actions:
@@ -289,7 +290,7 @@ class ButtonClick(WorkflowTestAction):
                 button_name = [
                     x.label
                     for x in self.get_all_choice_actions(formdef)
-                    if x.id == trace.event_args['action_item_id']
+                    if x.id == trace.event_args['action_item_id'] and 'wf-%s' % x.parent.id == trace.status_id
                 ][0]
             except IndexError:
                 return
@@ -463,13 +464,17 @@ class AssertEmail(WorkflowTestAction):
 
     @property
     def details_label(self):
-        if not self.addresses:
-            return ''
+        label = ''
 
-        label = _('Email to "%s"') % self.addresses[0]
+        if self.addresses:
+            label = _('Email to "%s"') % self.addresses[0]
 
-        if len(self.addresses) > 1:
-            label = '%s (+%s)' % (label, len(self.addresses) - 1)
+            if len(self.addresses) > 1:
+                label = '%s (+%s)' % (label, len(self.addresses) - 1)
+        elif self.subject_strings:
+            label = _('Subject must contain "%s"') % misc.ellipsize(self.subject_strings[0])
+        elif self.body_strings:
+            label = _('Body must contain "%s"') % misc.ellipsize(self.body_strings[0])
 
         return label
 
@@ -772,13 +777,15 @@ class AssertSMS(WorkflowTestAction):
 
     @property
     def details_label(self):
-        if not self.phone_numbers:
-            return ''
+        label = ''
 
-        label = _('SMS to %s') % self.phone_numbers[0]
+        if self.phone_numbers:
+            label = _('SMS to %s') % self.phone_numbers[0]
 
-        if len(self.phone_numbers) > 1:
-            label = '%s (+%s)' % (label, len(self.phone_numbers) - 1)
+            if len(self.phone_numbers) > 1:
+                label = '%s (+%s)' % (label, len(self.phone_numbers) - 1)
+        elif self.body:
+            label = misc.ellipsize(self.body)
 
         return label
 
@@ -793,7 +800,7 @@ class AssertSMS(WorkflowTestAction):
                 details = [_('SMS phone numbers: %s') % ', '.join(sms['phone_numbers'])]
                 raise WorkflowTestError(_('SMS was not sent to %s.') % recipient, details=details)
 
-        if self.body != sms['body']:
+        if self.body and self.body != sms['body']:
             details = [_('SMS body: "%s"') % sms['body']]
             raise WorkflowTestError(_('SMS body mismatch.'), details=details)
 
@@ -862,7 +869,6 @@ class AssertRedirect(WorkflowTestAction):
 
 class AssertHistoryMessage(WorkflowTestAction):
     label = _('Assert history message is displayed')
-    details_label = ''
 
     key = 'assert-history-message'
     message = None
@@ -871,6 +877,10 @@ class AssertHistoryMessage(WorkflowTestAction):
         ('message', 'str'),
     ]
 
+    @property
+    def details_label(self):
+        return misc.ellipsize(self.message)
+
     def perform(self, formdata):
         try:
             message = formdata.history_messages.pop(0)
@@ -896,7 +906,6 @@ class AssertHistoryMessage(WorkflowTestAction):
 
 class AssertAlert(WorkflowTestAction):
     label = _('Assert alert is displayed')
-    details_label = ''
 
     key = 'assert-alert'
     message = None
@@ -905,6 +914,10 @@ class AssertAlert(WorkflowTestAction):
         ('message', 'str'),
     ]
 
+    @property
+    def details_label(self):
+        return misc.ellipsize(self.message)
+
     def perform(self, formdata):
         messages = formdata.get_workflow_messages()
 

wcs/workflow_traces.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+from quixote import get_publisher
 from quixote.html import TemplateIO, htmltext
 
 from wcs import sql
@@ -21,6 +22,15 @@ from wcs.qommon import _, misc
 
 
 class WorkflowTrace(sql.WorkflowTrace):
+    def store(self, *args, **kwargs):
+        super().store(*args, **kwargs)
+        job = getattr(get_publisher(), 'current_cron_job', None)
+        if job:
+            job.log_debug(
+                f'stored trace ({self.id}), {self.formdef_type}/{self.formdef_id}-{self.formdata_id}, '
+                f'event: {self.event or "-"}, action: {self.action_item_key or "-"}'
+            )
+
     @classmethod
     def select_for_formdata(cls, formdata):
         return cls.select(
@@ -94,13 +104,11 @@ class WorkflowTrace(sql.WorkflowTrace):
         from wcs.carddef import CardDef
         from wcs.formdef import FormDef
 
-        if not hasattr(self, '_formdef'):
-            formdef_class = FormDef
-            if 'carddata' in self.event:
-                formdef_class = CardDef
+        formdef_class = FormDef
+        if 'carddata' in self.event:
+            formdef_class = CardDef
 
-            self._formdef = formdef_class.get(self.event_args.get('external_formdef_id'), ignore_errors=True)
-        return self._formdef
+        return formdef_class.cached_get(self.event_args.get('external_formdef_id'), ignore_errors=True)
 
     @property
     def formdata(self):

wcs/workflows.py

@@ -943,6 +943,16 @@ class Workflow(StorableObject):
         if not migration_update:
             if get_response():
                 get_response().add_after_job(_('Reindexing cards and forms after workflow change'), update)
+                from wcs.admin.tests import TestsAfterJob
+
+                for formdef in itertools.chain(self.formdefs(), self.carddefs()):
+                    get_response().add_after_job(
+                        TestsAfterJob(
+                            formdef,
+                            reason=_('Workflow: %s') % comment if comment else _('Change in workflow'),
+                            triggered_by='workflow-change',
+                        )
+                    )
             else:
                 update()