entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 31 Releases Activity

formdata: new digests field (#45633)

This commit is contained in:
Lauréline Guérin 2021-06-24 15:25:46 +02:00
parent 32826ca8a2
commit bb47de23af
No known key found for this signature in database
GPG Key ID: 1FAB9B9B4F93D473
14 changed files with 150 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/admin_pages/test_card.py
View File

@ -266,7 +266,7 @@ def test_card_digest_template(pub):
# afterjobs are actually run synchronously during tests; we don't have
# to wait to check the digest has been updated:
assert carddef.data_class().get(carddata.id).digest == 'XbarY'
assert carddef.data_class().get(carddata.id).digests['default'] == 'XbarY'
carddef.digest_templates = {'default': '{{ form_var_foo }}'}
carddef.store()

2
tests/admin_pages/test_form.py
View File

@ -905,7 +905,7 @@ def test_form_templates(pub):
assert 'Existing forms will be updated in the background.' in resp.text
# afterjobs are actually run synchronously during tests; we don't have
# to wait to check the digest has been updated:
assert formdef.data_class().get(formdata.id).digest == 'XhelloY'
assert formdef.data_class().get(formdata.id).digests['default'] == 'XhelloY'
resp = app.get('/backoffice/forms/1/options/templates')
resp.form['lateral_template'] = 'X{{form_var_test}}Y'

10
tests/api/test_carddef.py
View File

@ -147,19 +147,19 @@ def test_cards(pub, local_user):
assert len(resp.json['data']) == 1
assert resp.json['data'][0]['display_id'] == formdata.get_display_id()
assert resp.json['data'][0]['display_name'] == formdata.get_display_name()
assert resp.json['data'][0]['digest'] == formdata.digest
assert resp.json['data'][0]['text'] == formdata.digest
assert resp.json['data'][0]['digest'] == formdata.digests['default']
assert resp.json['data'][0]['text'] == formdata.digests['default']
resp = get_app(pub).get(
sign_uri('/api/cards/test/list?NameID=%s&full=on' % local_user.name_identifiers[0])
)
assert resp.json['data'][0]['fields']['foo'] == 'blah'
assert resp.json['data'][0]['digest'] == formdata.digest
assert resp.json['data'][0]['text'] == formdata.digest
assert resp.json['data'][0]['digest'] == formdata.digests['default']
assert resp.json['data'][0]['text'] == formdata.digests['default']
# get single carddata (as signed request without any user specified, so
# no check for permissions)
resp = get_app(pub).get(sign_uri('/api/cards/test/%s/' % formdata.id))
assert resp.json['text'] == formdata.digest
assert resp.json['text'] == formdata.digests['default']
# get schema
resp = get_app(pub).get(sign_uri('/api/cards/test/@schema'), status=200)

2
tests/api/test_user.py
View File

@ -290,7 +290,7 @@ def test_user_forms(pub, local_user):
# check digest is part of contents
formdef.digest_templates = {'default': 'XYZ'}
formdef.data_class().get(formdata.id).store()
assert formdef.data_class().get(formdata.id).digest == 'XYZ'
assert formdef.data_class().get(formdata.id).digests['default'] == 'XYZ'
resp = get_app(pub).get(sign_uri('/api/user/forms', user=local_user))
assert resp.json['data'][0]['form_digest'] == 'XYZ'

4
tests/backoffice_pages/test_all.py
View File

@ -3100,7 +3100,7 @@ def test_global_listing(pub):
)
formdata.formdef.digest_templates = {'default': 'digest of number <{{form_number}}>'}
formdata.store()
assert formdata.get(formdata.id).digest
assert formdata.get(formdata.id).digests['default']
resp = app.get('/backoffice/management/listing')
assert formdata.get_url(backoffice=True) in resp.text
assert 'digest of number &lt;%s&gt;' % formdata.id_display in resp.text
@ -3572,7 +3572,7 @@ def test_backoffice_sidebar_user_context(pub):
user_pending_form_url = re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0]
partial_resp = app.get(user_pending_form_url)
assert number31.get_url(backoffice=True) not in partial_resp.text
assert number31.digest in partial_resp.text
assert number31.digests['default'] in partial_resp.text
assert '<span class="formname">%s</span>' % number31.formdef.name in partial_resp.text
# another item with status = new

12
tests/test_formdata.py
View File

@ -2308,35 +2308,35 @@ def test_form_digest_date(pub):
formdata = formdef.data_class()()
formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop 2015-05-12 plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015-05-12 plop'
pub.cfg['language'] = {'language': 'fr'}
pub.write_cfg()
formdata = formdef.data_class()()
formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop 12/05/2015 plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 12/05/2015 plop'
formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
formdef.store()
formdata = formdef.data_class()()
formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop 2015 plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'
formdef.digest_templates = {'default': 'plop {{ form_var_date_raw|date:"Y" }} plop'}
formdef.store()
formdata = formdef.data_class()()
formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop 2015 plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'
formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
formdef.store()
formdata = formdef.data_class()()
formdata.data = {'0': None}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop plop'
# check there's no crash when an invaliad variable is given
formdef.digest_templates = {'default': 'plop {{ blah|date:"Y" }} plop'}
@ -2344,7 +2344,7 @@ def test_form_digest_date(pub):
formdata = formdef.data_class()()
formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
formdata.store()
assert formdef.data_class().get(formdata.id).digest == 'plop plop'
assert formdef.data_class().get(formdata.id).digests['default'] == 'plop plop'
def test_lazy_formdata_decimal_filter(pub):

2
wcs/backoffice/data_management.py
View File

@ -311,7 +311,7 @@ class CardFillPage(FormFillPage):
popup_response_data = json.dumps(
{
'value': str(filled.id),
'obj': str(filled.digest),
'obj': str(filled.default_digest),
}
)
return template.QommonTemplateResponse(

28
wcs/backoffice/management.py
View File

@ -1024,8 +1024,8 @@ class ManagementDirectory(Directory):
if include_submission_channel:
r += htmltext('<td>%s</td>') % formdata.get_submission_channel_label()
r += htmltext('<td>%s') % formdata.formdef.name
if formdata.digest:
r += htmltext(' <small>%s</small>') % formdata.digest
if formdata.default_digest:
r += htmltext(' <small>%s</small>') % formdata.default_digest
r += htmltext('</td>')
r += htmltext('<td><a href="%s">%s</a></td>') % (
formdata.get_url(backoffice=True),
@ -2355,17 +2355,19 @@ class FormPage(Directory):
if get_publisher().is_using_postgresql():
self.formdef.data_class().load_all_evolutions(items)
if get_request().form.get('full') == 'on':
output = [
filled.get_json_export_dict(include_files=False, anonymise=anonymise, user=user)
for filled in items
]
output = []
for filled in items:
data = filled.get_json_export_dict(include_files=False, anonymise=anonymise, user=user)
data.pop('digests')
data['digest'] = filled.default_digest
output.append(data)
else:
output = [
{
'id': filled.id,
'display_id': filled.get_display_id(),
'display_name': filled.get_display_name(),
'digest': filled.digest,
'digest': filled.default_digest,
'text': filled.get_display_label(),
'url': filled.get_url(),
'receipt_time': datetime.datetime(*filled.receipt_time[:6]),
@ -2476,8 +2478,8 @@ class FormPage(Directory):
formdata.id,
)
summary = force_text(formdata.get_display_name(), charset)
if formdata.digest:
summary += ' - %s' % force_text(formdata.digest, charset)
if formdata.default_digest:
summary += ' - %s' % force_text(formdata.default_digest, charset)
vevent.add('summary').value = summary
vevent.add('dtstart').value = dtstart
if dtend:
@ -2488,8 +2490,8 @@ class FormPage(Directory):
form_name = force_text(formdef.name, charset)
status_name = force_text(formdata.get_status_label(), charset)
description = '%s | %s | %s\n' % (form_name, formdata.get_display_id(), status_name)
if formdata.digest:
description += '%s\n' % force_text(formdata.digest, charset)
if formdata.default_digest:
description += '%s\n' % force_text(formdata.default_digest, charset)
description += backoffice_url
# TODO: improve performance by loading all users in one
# single query before the loop
@ -3193,8 +3195,8 @@ class FormBackOfficeStatusPage(FormStatusPage):
submit_date,
status_label,
)
if formdata.digest:
r += htmltext('<small>%s</small>') % formdata.digest
if formdata.default_digest:
r += htmltext('<small>%s</small>') % formdata.default_digest
r += htmltext('</li>')
r += htmltext('</ul>')
r += htmltext('</div>')

4
wcs/carddata.py
View File

@ -35,7 +35,7 @@ class CardData(FormData):
def get_data_source_structured_item(self):
item = {
'id': self.id,
'text': self.digest,
'text': self.default_digest,
}
for field in self.formdef.get_all_fields():
if not field.varname:
@ -46,7 +46,7 @@ class CardData(FormData):
return item
def get_display_label(self):
return self.digest or self.get_display_name()
return self.default_digest or self.get_display_name()
def get_author_qualification(self):
return None

6
wcs/carddef.py
View File

@ -24,7 +24,7 @@ from wcs.categories import CardDefCategory
from wcs.formdef import FormDef, get_formdefs_of_all_kinds
from .qommon import _, force_text, misc
from .qommon.storage import Equal, ILike, NotEqual
from .qommon.storage import ElementEqual, ElementILike, Equal, NotEqual
from .qommon.template import Template
if not hasattr(types, 'ClassType'):
@ -215,7 +215,7 @@ class CardDef(FormDef):
criteria.value = WorkflowStatusItem.compute(criteria.value)
if query:
criterias.append(ILike('digest', query))
criterias.append(ElementILike('digests', 'default', query))
if get_by_id:
try:
if int(get_by_id) >= 2 ** 31:
@ -228,7 +228,7 @@ class CardDef(FormDef):
else:
criterias.append(Equal('id', get_by_id))
if get_by_text:
criterias.append(Equal('digest', get_by_text))
criterias.append(ElementEqual('digests', 'default', get_by_text))
items = [
x.get_data_source_structured_item()

39
wcs/formdata.py
View File

@ -246,7 +246,7 @@ class FormData(StorableObject):
submission_context = None
submission_channel = None
criticality_level = 0
digest = None
digests = None
prefilling_data = None
workflow_data = None
@ -291,6 +291,14 @@ class FormData(StorableObject):
self.submission_agent_id = str(self.submission_context.get('agent_id'))
changed = True
if 'digest' in self.__dict__:
# migration from a simple digest to digests
if not self.digests:
self.digests = {}
self.digests['default'] = self.__dict__['digest']
del self.__dict__['digest']
changed = True
if changed:
self.store()
@ -424,7 +432,9 @@ class FormData(StorableObject):
self.evolution = [evo]
def set_auto_fields(self, *args, **kwargs):
fields = {'digest': self.formdef.default_digest_template}
fields = {}
for key, value in (self.formdef.digest_templates or {}).items():
fields['template:%s' % key] = value
if not self.id_display:
# only set id_display once as it may have been set automatically
# by interpreting a webservice response.
@ -451,13 +461,22 @@ class FormData(StorableObject):
if any(fields.values()):
context = self.get_substitution_variables()
context['formdef_id'] = self.formdef.id
digests = self.digests or {}
for attribute, template in fields.items():
if template is None:
new_value = None
else:
new_value = Template(template, autoescape=False).render(context)
if new_value != getattr(self, attribute, None):
setattr(self, attribute, new_value)
if attribute.startswith('template:'):
key = attribute[9:]
if new_value != (self.digests or {}).get(key):
digests[key] = new_value
else:
if new_value != getattr(self, attribute, None):
setattr(self, attribute, new_value)
changed = True
if digests:
self.digests = digests
changed = True
return changed
@ -771,7 +790,7 @@ class FormData(StorableObject):
'form_url_backoffice': self.get_url(backoffice=True),
'form_uri': '%s/%s/' % (self.formdef.url_name, self.id),
'form_criticality_level': self.criticality_level,
'form_digest': self.digest,
'form_digest': self.default_digest,
'form_display_name': self.get_display_name(),
}
)
@ -1099,9 +1118,13 @@ class FormData(StorableObject):
def get_display_name(self):
return _('%(name)s #%(id)s') % {'name': self.formdef.name, 'id': self.get_display_id()}
@property
def default_digest(self):
return (self.digests or {}).get('default')
def get_display_label(self):
if self.digest:
return '%s (%s)' % (self.get_display_name(), self.digest)
if self.default_digest:
return '%s (%s)' % (self.get_display_name(), self.default_digest)
return self.get_display_name()
def get_auto_geoloc(self):
@ -1155,7 +1178,7 @@ class FormData(StorableObject):
def get_json_export_dict(self, include_files=True, anonymise=False, user=None):
data = {}
data['id'] = str(self.id)
data['digest'] = self.digest
data['digests'] = self.digests
data['display_id'] = self.get_display_id()
data['display_name'] = self.get_display_name()
data['text'] = self.get_display_label()

23
wcs/qommon/storage.py
View File

@ -280,6 +280,29 @@ class Null(Criteria):
return lambda x: getattr(x, self.attribute, None) is None
class ElementEqual(Criteria):
def __init__(self, attribute, key, value):
self.attribute = attribute
self.key = key
self.value = value
def build_lambda(self):
return lambda x: (getattr(x, self.attribute, None) or {}).get(self.key) == self.value
class ElementILike(Criteria):
def __init__(self, attribute, key, value):
self.attribute = attribute
self.key = key
self.value = value
def build_lambda(self):
return (
lambda x: self.value.lower()
in ((getattr(x, self.attribute, None) or {}).get(self.key) or '').lower()
)
def parse_clause(clause):
# creates a callable out of a clause
# (attribute, operator, value)

83
wcs/sql.py
View File

@ -279,6 +279,24 @@ class FtsMatch(Criteria):
return 'fts @@ plainto_tsquery(%%(c%s)s)' % id(self.value)
class ElementEqual(Criteria):
def __init__(self, attribute, key, value):
super().__init__(attribute, value)
self.key = key
def as_sql(self):
return "%s->>'%s' = %%(c%s)s" % (self.attribute, self.key, id(self.value))
class ElementILike(ElementEqual):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.value = '%' + self.value + '%'
def as_sql(self):
return "%s->>'%s' ILIKE %%(c%s)s" % (self.attribute, self.key, id(self.value))
def get_name_as_sql_identifier(name):
name = qommon.misc.simplify(name)
for char in '<>|{}!?^*+/=\'': # forbidden chars
@ -564,7 +582,7 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
'submission_channel',
'criticality_level',
'last_update_time',
'digest',
'digests',
'user_label',
'prefilling_data',
}
@ -612,8 +630,8 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
if 'last_update_time' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN last_update_time timestamp''' % table_name)
if 'digest' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN digest varchar''' % table_name)
if 'digests' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN digests jsonb''' % table_name)
if 'user_label' not in existing_fields:
cur.execute('''ALTER TABLE %s ADD COLUMN user_label varchar''' % table_name)
@ -1217,7 +1235,7 @@ def get_view_fields(formdef):
'submission_channel',
'backoffice_submission',
'last_update_time',
'digest',
'digests',
'user_label',
):
view_fields.append((field, field))
@ -1858,7 +1876,7 @@ class SqlDataMixin(SqlMixin):
('submission_channel', 'varchar'),
('criticality_level', 'int'),
('last_update_time', 'timestamp'),
('digest', 'varchar'),
('digests', 'jsonb'),
('user_label', 'varchar'),
]
@ -1949,6 +1967,27 @@ class SqlDataMixin(SqlMixin):
conn.commit()
cur.close()
@guard_postgres
def _set_auto_fields(self, cur):
if self.set_auto_fields():
sql_statement = (
'''UPDATE %s
SET id_display = %%(id_display)s,
digests = %%(digests)s,
user_label = %%(user_label)s
WHERE id = %%(id)s'''
% self._table_name
)
cur.execute(
sql_statement,
{
'id': self.id,
'id_display': self.id_display,
'digests': self.digests,
'user_label': self.user_label,
},
)
@guard_postgres
@invalidate_substitution_cache
def store(self):
@ -2030,24 +2069,7 @@ class SqlDataMixin(SqlMixin):
cur.execute(sql_statement, sql_dict)
self.id = cur.fetchone()[0]
if self.set_auto_fields():
sql_statement = (
'''UPDATE %s
SET id_display = %%(id_display)s,
digest = %%(digest)s,
user_label = %%(user_label)s
WHERE id = %%(id)s'''
% self._table_name
)
cur.execute(
sql_statement,
{
'id': self.id,
'id_display': self.id_display,
'digest': self.digest,
'user_label': self.user_label,
},
)
self._set_auto_fields(cur)
if self._evolution:
for evo in self._evolution:
@ -3392,7 +3414,7 @@ def get_period_total(
# latest migration, number + description (description is not used
# programmaticaly but will make sure git conflicts if two migrations are
# separately added with the same number)
SQL_LEVEL = (51, 'add index on formdata blockdef fields')
SQL_LEVEL = (52, 'store digests on formdata and carddata')
def migrate_global_views(conn, cur):
@ -3475,7 +3497,7 @@ def migrate():
raise RuntimeError()
if sql_level < 1: # 1: introduction of tracking_code table
do_tracking_code_table()
if sql_level < 43:
if sql_level < 52:
# 2: introduction of formdef_id in views
# 5: add concerned_roles_array, is_at_endpoint and fts to views
# 7: add backoffice_submission to tables
@ -3495,6 +3517,7 @@ def migrate():
# 33: add anonymised field to global view
# 38: extract submission_agent_id to its own column
# 43: add prefilling_data to formdata
# 52: store digests on formdata and carddata
migrate_views(conn, cur)
if sql_level < 40:
# 3: introduction of _structured for user fields
@ -3557,6 +3580,16 @@ def migrate():
for evo in formdata.evolution:
evo.who = None
formdata.store()
if sql_level < 52:
# 52: store digests on formdata and carddata
from wcs.carddef import CardDef
from wcs.formdef import FormDef
for formdef in FormDef.select() + CardDef.select():
if not formdef.digest_templates:
continue
for formdata in formdef.data_class().select_iterator():
formdata._set_auto_fields(cur) # build digests
if sql_level < 42:
# 42: create snapshots table
do_snapshots_table()

2
wcs/variables.py
View File

@ -406,7 +406,7 @@ class LazyFormData(LazyFormDef):
@property
def digest(self):
return self._formdata.digest
return self._formdata.default_digest
@property
def display_name(self):