From fb50c89dc39452eeaec6be207d59e42eaa07c71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Wed, 20 Sep 2023 17:28:32 +0200 Subject: [PATCH] misc: unify scan for all kinds of dependencies, including datasources (#81414) --- tests/api/test_export_import.py | 14 ++++++++++-- wcs/comment_templates.py | 6 ++---- wcs/fields/base.py | 17 +++------------ wcs/fields/comment.py | 6 ++---- wcs/fields/computed.py | 4 ++-- wcs/fields/page.py | 6 ++---- wcs/fields/title.py | 6 ++---- wcs/formdef.py | 8 ++----- wcs/mail_templates.py | 6 ++---- wcs/qommon/misc.py | 38 ++++++++++----------------------- wcs/workflows.py | 17 +++------------ 11 files changed, 43 insertions(+), 85 deletions(-) diff --git a/tests/api/test_export_import.py b/tests/api/test_export_import.py index d696976b3..27db0161d 100644 --- a/tests/api/test_export_import.py +++ b/tests/api/test_export_import.py @@ -215,6 +215,9 @@ def test_export_import_dependencies(pub): data_source = NamedDataSource(name='foobar') data_source.store() + data_source2 = NamedDataSource(name='foobaz') + data_source2.store() + display_form = status.add_action('form', id='_x') display_form.formdef = WorkflowFormFieldsFormDef(item=display_form) display_form.formdef.fields.append( @@ -249,7 +252,10 @@ def test_export_import_dependencies(pub): StringField( label='Test', data_source={'type': 'foobar'}, - prefill={'type': 'string', 'value': '{{ forms|objects:"test-bis" }}'}, + prefill={ + 'type': 'string', + 'value': '{{ data_source.foobaz.plop }} {{ forms|objects:"test-bis" }}', + }, ) ) display_form.formdef.fields.append( @@ -347,6 +353,7 @@ def test_export_import_dependencies(pub): resp = get_app(pub).get(sign_uri(resp.json['data'][0]['urls']['dependencies'])) assert {(x['id'], x['type']) for x in resp.json['data']} == { ('foobar', 'data-sources'), + ('foobaz', 'data-sources'), ('test', 'wscalls'), ('test_bis', 'wscalls'), ('test_ter', 'wscalls'), @@ -380,6 +387,7 @@ def test_export_import_dependencies(pub): resp = get_app(pub).get(sign_uri(resp.json['data'][0]['urls']['dependencies'])) assert {(x['id'], x['type']) for x in resp.json['data']} == { ('foobar', 'data-sources'), + ('foobaz', 'data-sources'), ('test_bis', 'wscalls'), ('test_ter', 'wscalls'), ('test-bis', 'cards'), @@ -445,7 +453,9 @@ def test_export_import_dependencies(pub): data_source.store() resp = get_app(pub).get(sign_uri('/api/export-import/workflows/')) resp = get_app(pub).get(sign_uri(resp.json['data'][0]['urls']['dependencies'])) - data_sources_entry = [x for x in resp.json['data'] if x['type'] == 'data-sources'][0] + data_sources_entry = [ + x for x in resp.json['data'] if x['type'] == 'data-sources' and x['id'] == 'foobar' + ][0] resp = get_app(pub).get(sign_uri(data_sources_entry['urls']['dependencies'])) assert {(x['id'], x['type']) for x in resp.json['data']} == {('cat', 'data-sources-categories')} diff --git a/wcs/comment_templates.py b/wcs/comment_templates.py index c13c9b884..6a18651de 100644 --- a/wcs/comment_templates.py +++ b/wcs/comment_templates.py @@ -21,7 +21,7 @@ from quixote import get_publisher from wcs.categories import CommentTemplateCategory from wcs.qommon import _, get_logger from wcs.qommon.form import OptGroup -from wcs.qommon.misc import check_carddefs, check_formdefs, check_wscalls +from wcs.qommon.misc import get_dependencies_from_template from wcs.qommon.xml_storage import XmlStorableObject @@ -130,9 +130,7 @@ class CommentTemplate(XmlStorableObject): def get_dependencies(self): yield self.category for string in self.get_computed_strings(): - yield from check_wscalls(string) - yield from check_carddefs(string) - yield from check_formdefs(string) + yield from get_dependencies_from_template(string) def get_computed_strings(self): yield self.comment diff --git a/wcs/fields/base.py b/wcs/fields/base.py index df9e72a42..6098326c7 100644 --- a/wcs/fields/base.py +++ b/wcs/fields/base.py @@ -39,14 +39,7 @@ from wcs.qommon.form import ( TextWidget, VarnameWidget, ) -from wcs.qommon.misc import ( - check_carddefs, - check_formdefs, - check_wscalls, - date_format, - ellipsize, - xml_node_text, -) +from wcs.qommon.misc import date_format, ellipsize, get_dependencies_from_template, xml_node_text from wcs.qommon.template import Template, TemplateError @@ -665,16 +658,12 @@ class Field: prefill = self.prefill if prefill: if prefill.get('type') == 'string': - yield from check_wscalls(prefill.get('value')) - yield from check_carddefs(prefill.get('value')) - yield from check_formdefs(prefill.get('value')) + yield from get_dependencies_from_template(prefill.get('value')) if getattr(self, 'condition', None): condition = self.condition if condition: if condition.get('type') == 'django': - yield from check_wscalls(condition.get('value')) - yield from check_carddefs(condition.get('value')) - yield from check_formdefs(condition.get('value')) + yield from get_dependencies_from_template(condition.get('value')) def get_parameters_view(self): r = TemplateIO(html=True) diff --git a/wcs/fields/comment.py b/wcs/fields/comment.py index 3c775ef0c..d177b6531 100644 --- a/wcs/fields/comment.py +++ b/wcs/fields/comment.py @@ -30,7 +30,7 @@ from wcs.qommon.form import ( TextWidget, WysiwygTextWidget, ) -from wcs.qommon.misc import check_carddefs, check_formdefs, check_wscalls +from wcs.qommon.misc import get_dependencies_from_template from .base import Field, register_field_class @@ -126,9 +126,7 @@ class CommentField(Field): def get_dependencies(self): yield from super().get_dependencies() - yield from check_wscalls(self.label) - yield from check_carddefs(self.label) - yield from check_formdefs(self.label) + yield from get_dependencies_from_template(self.label) register_field_class(CommentField) diff --git a/wcs/fields/computed.py b/wcs/fields/computed.py index 75570c9d8..a8deb7fec 100644 --- a/wcs/fields/computed.py +++ b/wcs/fields/computed.py @@ -17,7 +17,7 @@ from wcs import data_sources from wcs.qommon import _, misc from wcs.qommon.form import CheckboxWidget, ComputedExpressionWidget, StringWidget, TextWidget, VarnameWidget -from wcs.qommon.misc import check_wscalls +from wcs.qommon.misc import get_dependencies_from_template from .base import Field, register_field_class @@ -91,7 +91,7 @@ class ComputedField(Field): def get_dependencies(self): yield from super().get_dependencies() - yield from check_wscalls(self.value_template) + yield from get_dependencies_from_template(self.value_template) register_field_class(ComputedField) diff --git a/wcs/fields/page.py b/wcs/fields/page.py index 086fd00ad..e53426b4c 100644 --- a/wcs/fields/page.py +++ b/wcs/fields/page.py @@ -24,7 +24,7 @@ from quixote.html import TemplateIO, htmltext from wcs.conditions import Condition from wcs.qommon import _ from wcs.qommon.form import CompositeWidget, ConditionWidget, StringWidget, VarnameWidget, WidgetListAsTable -from wcs.qommon.misc import check_carddefs, check_formdefs, check_wscalls, xml_node_text +from wcs.qommon.misc import get_dependencies_from_template, xml_node_text from .base import Field, register_field_class @@ -242,9 +242,7 @@ class PageField(Field): for post_condition in post_conditions: condition = post_condition.get('condition') or {} if condition.get('type') == 'django': - yield from check_wscalls(condition.get('value')) - yield from check_carddefs(condition.get('value')) - yield from check_formdefs(condition.get('value')) + yield from get_dependencies_from_template(condition.get('value')) def i18n_scan(self, base_location): location = '%s%s/' % (base_location, self.id) diff --git a/wcs/fields/title.py b/wcs/fields/title.py index 67d84e117..70d7acfdb 100644 --- a/wcs/fields/title.py +++ b/wcs/fields/title.py @@ -20,7 +20,7 @@ from quixote.html import htmltext from wcs.qommon import _ from wcs.qommon.form import CheckboxesWidget, ConditionWidget, HtmlWidget, StringWidget -from wcs.qommon.misc import check_carddefs, check_formdefs, check_wscalls +from wcs.qommon.misc import get_dependencies_from_template from .base import Field, register_field_class @@ -89,9 +89,7 @@ class TitleField(Field): def get_dependencies(self): yield from super().get_dependencies() - yield from check_wscalls(self.label) - yield from check_carddefs(self.label) - yield from check_formdefs(self.label) + yield from get_dependencies_from_template(self.label) register_field_class(TitleField) diff --git a/wcs/formdef.py b/wcs/formdef.py index a28868e9e..542ad4d48 100644 --- a/wcs/formdef.py +++ b/wcs/formdef.py @@ -45,10 +45,8 @@ from .qommon.errors import UnknownReferencedErrorMixin from .qommon.form import Form, HtmlWidget from .qommon.misc import ( JSONEncoder, - check_carddefs, - check_formdefs, - check_wscalls, get_as_datetime, + get_dependencies_from_template, is_attachment, is_upload, simplify, @@ -651,9 +649,7 @@ class FormDef(StorableObject): self.lateral_template, self.submission_lateral_template, ]: - yield from check_wscalls(template) - yield from check_carddefs(template) - yield from check_formdefs(template) + yield from get_dependencies_from_template(template) @property def keywords_list(self): diff --git a/wcs/mail_templates.py b/wcs/mail_templates.py index 81b1e5c70..61fcf9f7b 100644 --- a/wcs/mail_templates.py +++ b/wcs/mail_templates.py @@ -21,7 +21,7 @@ from quixote import get_publisher from wcs.categories import MailTemplateCategory from wcs.qommon import _, get_logger from wcs.qommon.form import OptGroup -from wcs.qommon.misc import check_carddefs, check_formdefs, check_wscalls +from wcs.qommon.misc import get_dependencies_from_template from wcs.qommon.xml_storage import XmlStorableObject @@ -132,9 +132,7 @@ class MailTemplate(XmlStorableObject): def get_dependencies(self): yield self.category for string in self.get_computed_strings(): - yield from check_wscalls(string) - yield from check_carddefs(string) - yield from check_formdefs(string) + yield from get_dependencies_from_template(string) def get_computed_strings(self): yield self.subject diff --git a/wcs/qommon/misc.py b/wcs/qommon/misc.py index ad0fb5569..6a8e5353f 100644 --- a/wcs/qommon/misc.py +++ b/wcs/qommon/misc.py @@ -1308,39 +1308,23 @@ def strip_some_tags(value, allowed_tags): return mark_safe(value) -def check_wscalls(string): +def get_dependencies_from_template(string): + from wcs.carddef import CardDef + from wcs.data_sources import NamedDataSource + from wcs.formdef import FormDef from wcs.wscalls import NamedWsCall - if not isinstance(string, str): + if not isinstance(string, str) or string.startswith('='): return - if string.startswith('='): # python expression - return - for match in re.findall(r'webservice\.\w+', string): - ws_slug = match[11:] - yield NamedWsCall.get_by_slug(ws_slug) + for ws_slug in re.findall(r'webservice\.([\w_]+)', string): + yield NamedWsCall.get_by_slug(ws_slug, ignore_errors=True) -def check_carddefs(string): - from wcs.carddef import CardDef + for ds_slug in re.findall(r'data_source\.([\w_]+)', string): + yield NamedDataSource.get_by_slug(ds_slug, ignore_errors=True) - if not isinstance(string, str): - return - if string.startswith('='): # python expression - return - for match in re.findall(r'cards\|objects:"[\w-]+"', string): - carddef_slug = match[15:] - carddef_slug = carddef_slug[:-1] + for carddef_slug in re.findall(r'cards\|objects:"([\w_-]+)"', string): yield CardDef.get_by_slug(carddef_slug, ignore_errors=True) - -def check_formdefs(string): - from wcs.formdef import FormDef - - if not isinstance(string, str): - return - if string.startswith('='): # python expression - return - for match in re.findall(r'forms\|objects:"[\w-]+"', string): - formdef_slug = match[15:] - formdef_slug = formdef_slug[:-1] + for formdef_slug in re.findall(r'forms\|objects:"([\w_-]+)"', string): yield FormDef.get_by_slug(formdef_slug, ignore_errors=True) diff --git a/wcs/workflows.py b/wcs/workflows.py index f323271eb..a79d3a951 100644 --- a/wcs/workflows.py +++ b/wcs/workflows.py @@ -59,14 +59,7 @@ from .qommon.form import ( WidgetListOfRoles, ) from .qommon.humantime import seconds2humanduration -from .qommon.misc import ( - check_carddefs, - check_formdefs, - check_wscalls, - file_digest, - get_as_datetime, - xml_node_text, -) +from .qommon.misc import file_digest, get_as_datetime, get_dependencies_from_template, xml_node_text from .qommon.template import Template, TemplateError from .qommon.upload_storage import PicklableUpload, get_storage_object from .roles import get_user_roles, logged_users_role @@ -2717,16 +2710,12 @@ class WorkflowStatusItem(XmlSerialisable): yield from get_role_dependencies(getattr(self, 'by', None)) yield from get_role_dependencies(getattr(self, 'to', None)) for string in self.get_computed_strings(): - yield from check_wscalls(string) - yield from check_carddefs(string) - yield from check_formdefs(string) + yield from get_dependencies_from_template(string) if getattr(self, 'condition', None): condition = self.condition if condition: if condition.get('type') == 'django': - yield from check_wscalls(condition.get('value')) - yield from check_carddefs(condition.get('value')) - yield from check_formdefs(condition.get('value')) + yield from get_dependencies_from_template(condition.get('value')) @noop_mark def perform(self, formdata):