general: allow assigning complex types from rendered templates (#41847)

This commit is contained in:
Frédéric Péters 2020-11-29 17:50:37 +01:00
parent 4b67ce1c1f
commit 8c55108916
8 changed files with 277 additions and 26 deletions

View File

@ -23,14 +23,16 @@ from quixote import cleanup, get_response
from wcs.qommon.errors import ConnectionError
from quixote.http_request import Upload as QuixoteUpload
from wcs.qommon.http_request import HTTPRequest
from wcs.qommon.form import PicklableUpload
from wcs.qommon.form import *
from wcs.blocks import BlockDef
from wcs.formdef import FormDef
from wcs.carddef import CardDef
from wcs import sessions
from wcs.fields import (StringField, DateField, MapField, FileField, ItemField,
ItemsField, CommentField, EmailField, PageField, TitleField,
SubtitleField, TextField, BoolField, TableField)
SubtitleField, TextField, BoolField, TableField, BlockField)
from wcs.formdata import Evolution
from wcs.roles import Role
from wcs.workflows import (Workflow, WorkflowStatusItem,
@ -3903,6 +3905,7 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
wf.backoffice_fields_formdef.fields = [
FileField(id='bo1', label='1st backoffice field',
type='file', varname='backoffice_file'),
StringField(id='bo2', label='2nd backoffice field', type='string'),
]
st1 = wf.add_status('Status1')
wf.store()
@ -3963,7 +3966,50 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
assert formdata.data['bo1'].content_type == 'application/vnd.oasis.opendocument.text'
assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'template.odt'), 'rb').read()
# check striping metadata
# check with template string
formdata = formdef.data_class()()
formdata.data = {'00': upload}
formdata.just_created()
formdata.store()
two_pubs.substitutions.feed(formdata)
item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file_raw}}'}]
item.perform(formdata)
assert formdata.data['bo1'].base_filename == 'test.jpeg'
assert formdata.data['bo1'].content_type == 'image/jpeg'
assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
# check with template string, without _raw
formdata = formdef.data_class()()
formdata.data = {'00': upload}
formdata.just_created()
formdata.store()
two_pubs.substitutions.feed(formdata)
item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file}}'}]
item.perform(formdata)
assert formdata.data['bo1'].base_filename == 'test.jpeg'
assert formdata.data['bo1'].content_type == 'image/jpeg'
assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
# check with a template string, into a string field
two_pubs.substitutions.feed(formdata)
item.fields = [{'field_id': 'bo2', 'value': '{{form_var_file}}'}]
item.perform(formdata)
assert formdata.data['bo2'] == 'test.jpeg'
# check with template string and missing file
formdata = formdef.data_class()()
formdata.data = {'00': None}
formdata.just_created()
formdata.store()
assert formdata.data.get('bo1') is None
# check stripping metadata
two_pubs.substitutions.feed(formdata)
item.fields = [{'field_id': 'bo1',
'value': '=utils.attachment(form_var_file_raw,' +
@ -4051,7 +4097,7 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
hello_world = formdata.data['bo1']
# check wrong value
for value in ('="HELLO"', 'BAD', '={}', '=[]'):
for value in ('="HELLO"', 'BAD'):
formdata.data['bo1'] = hello_world
formdata.store()
@ -4364,6 +4410,44 @@ def test_set_backoffice_field_items(two_pubs):
assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']
# from formdata field
formdef.fields = [
ItemsField(id='1', label='field', type='items', varname='items', data_source=datasource),
]
formdef.store()
formdata = formdef.data_class()()
formdata.data = {'1': ['a', 'c']}
formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
formdata.just_created()
formdata.store()
two_pubs.substitutions.feed(formdata)
item = SetBackofficeFieldsWorkflowStatusItem()
item.parent = st1
item.fields = [{'field_id': 'bo1', 'value': "=form_var_items_raw"}]
item.perform(formdata)
assert formdata.data['bo1'] == ['a', 'c']
assert formdata.data['bo1_display'] == 'aa, cc'
assert len(formdata.data['bo1_structured']) == 2
assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']
# with a template
formdata = formdef.data_class()()
formdata.data = {'1': ['a', 'c']}
formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
formdata.just_created()
formdata.store()
two_pubs.substitutions.reset()
two_pubs.substitutions.feed(formdata)
item.fields = [{'field_id': 'bo1', 'value': "{{form_var_items_raw}}"}]
item.perform(formdata)
def test_set_backoffice_field_date(two_pubs):
Workflow.wipe()
@ -4488,6 +4572,88 @@ def test_set_backoffice_field_boolean(two_pubs):
formdata.store()
def test_set_backoffice_field_block(two_pubs, blocks_feature):
BlockDef.wipe()
Workflow.wipe()
FormDef.wipe()
block = BlockDef()
block.name = 'foobar'
block.digest_template = 'X{{foobar_var_foo}}Y'
block.fields = [
StringField(id='123', required=True, label='Test',
type='string', varname='foo'),
StringField(id='234', required=True, label='Test2',
type='string', varname='bar'),
]
block.store()
wf = Workflow(name='xxx')
wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
st1 = wf.add_status('Status1')
wf.backoffice_fields_formdef.fields = [
BlockField(id='bo1', label='1st backoffice field', type='block:foobar'),
StringField(id='bo2', label='2nd backoffice field', type='string'),
]
wf.store()
formdef = FormDef()
formdef.name = 'baz'
formdef.fields = [
BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='foo'),
]
formdef.workflow_id = wf.id
formdef.store()
formdata = formdef.data_class()()
# value from test_block_digest in tests/test_form_pages.py
formdata.data = {
'1': {
'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
'schema': {'123': 'string', '234': 'string'}
},
'1_display': 'XfooY, Xfoo2Y',
}
formdata.just_created()
formdata.store()
get_publisher().substitutions.feed(formdata)
item = SetBackofficeFieldsWorkflowStatusItem()
item.parent = st1
item.fields = [{'field_id': 'bo1', 'value': '{{form_var_foo_raw}}'}]
item.perform(formdata)
formdata = formdef.data_class().get(formdata.id)
assert formdata.data['bo1'] == formdata.data['1']
assert formdata.data['bo1_display'] == formdata.data['1_display']
# without _raw suffix
formdata = formdef.data_class()()
# value from test_block_digest in tests/test_form_pages.py
formdata.data = {
'1': {
'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
'schema': {'123': 'string', '234': 'string'}
},
'1_display': 'XfooY, Xfoo2Y',
}
formdata.just_created()
formdata.store()
get_publisher().substitutions.reset()
get_publisher().substitutions.feed(formdata)
item = SetBackofficeFieldsWorkflowStatusItem()
item.parent = st1
item.fields = [
{'field_id': 'bo1', 'value': '{{form_var_foo}}'},
{'field_id': 'bo2', 'value': '{{form_var_foo}}'},
]
item.perform(formdata)
formdata = formdef.data_class().get(formdata.id)
assert formdata.data['bo1'] == formdata.data['1']
assert formdata.data['bo1_display'] == formdata.data['1_display']
assert formdata.data['bo2'] == formdata.data['1_display']
def test_set_backoffice_field_immediate_use(http_requests, two_pubs):
Workflow.wipe()
FormDef.wipe()
@ -4936,7 +5102,8 @@ def test_create_carddata(two_pubs):
StringField(id='1', label='string'),
ItemField(id='2', label='List', items=['item1', 'item2'],
varname='clist'),
DateField(id='3', label='Date', varname='cdate')
DateField(id='3', label='Date', varname='cdate'),
FileField(id='4', label='File', varname='cfile'),
]
carddef.store()
@ -4951,6 +5118,7 @@ def test_create_carddata(two_pubs):
Mapping(field_id='1', expression='=form_var_undefined'),
Mapping(field_id='2', expression='{{ form_var_list }}'),
Mapping(field_id='3', expression='{{ form_var_date }}'),
Mapping(field_id='4', expression='{{ form_var_file|default_if_none:"" }}'),
]
create.parent = wf.possible_status[1]
wf.possible_status[1].items.insert(0, create)
@ -4959,9 +5127,9 @@ def test_create_carddata(two_pubs):
formdef = FormDef()
formdef.name = 'source form'
formdef.fields = [
ItemField(id='1', label='List', items=['item1', 'item2'],
varname='list'),
DateField(id='2', label='Date', varname='date')
ItemField(id='1', label='List', items=['item1', 'item2'], varname='list'),
DateField(id='2', label='Date', varname='date'),
FileField(id='3', label='File', varname='file'),
]
formdef.workflow_id = wf.id
formdef.store()
@ -4989,16 +5157,23 @@ def test_create_carddata(two_pubs):
formdata = formdef.data_class()()
today = datetime.date.today()
upload = PicklableUpload('test.jpeg', 'image/jpeg')
with open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb') as jpg:
upload.receive([jpg.read()])
formdata.data = {'1': 'item1',
'1_display': 'item1',
'2': today.timetuple()}
'2': today.timetuple(),
'3': upload}
formdata.just_created()
formdata.perform_workflow()
assert formdata.get_substitution_variables()['form_links_mycard_form_number'] == '1-2'
carddata = carddef.data_class().get(id=2)
assert carddata.get_substitution_variables()['form_var_clist'] == 'item1'
assert carddata.get_substitution_variables()['form_var_cdate'] == today
assert carddata.data['2'] == 'item1'
assert carddata.data['2_display'] == 'item1'
assert carddata.data['3'] == today.timetuple()
assert carddata.data['4'].base_filename == 'test.jpeg'
create.condition = {'type': 'python', 'value': '1 == 2'}
wf.store()

View File

@ -176,6 +176,7 @@ class Field(object):
convert_value_from_str = None
convert_value_to_str = None
convert_value_from_anything = None
allow_complex = False
display_locations = []
prefill = None
store_display_value = None
@ -969,6 +970,7 @@ register_field_class(EmailField)
class BoolField(WidgetField):
key = 'bool'
description = N_('Check Box (single choice)')
allow_complex = True
widget_class = CheckboxWidget
required = False
@ -1064,6 +1066,8 @@ register_field_class(BoolField)
class FileField(WidgetField):
key = 'file'
description = N_('File Upload')
allow_complex = True
document_type = None
max_file_size = None
automatic_image_resize = False
@ -1120,7 +1124,7 @@ class FileField(WidgetField):
@classmethod
def convert_value_from_anything(cls, value):
if value is None:
if not value:
return None
from wcs.variables import LazyFieldVarFile
if isinstance(value, LazyFieldVarFile):
@ -1474,6 +1478,7 @@ class MapOptionsMixin:
class ItemField(WidgetField, MapOptionsMixin):
key = 'item'
description = N_('List')
allow_complex = True
items = []
show_as_radio = None
@ -1783,6 +1788,7 @@ register_field_class(ItemField)
class ItemsField(WidgetField):
key = 'items'
description = N_('Multiple choice list')
allow_complex = True
items = []
max_choices = 0
@ -2150,6 +2156,7 @@ register_field_class(PageField)
class TableField(WidgetField):
key = 'table'
description = N_('Table')
allow_complex = True
rows = None
columns = None
@ -2300,6 +2307,7 @@ register_field_class(TableField)
class TableSelectField(TableField):
key = 'table-select'
description = N_('Table of Lists')
allow_complex = True
items = None
@ -2341,6 +2349,7 @@ register_field_class(TableSelectField)
class TableRowsField(WidgetField):
key = 'tablerows'
description = N_('Table with rows')
allow_complex = True
total_row = True
columns = None
@ -2560,6 +2569,7 @@ register_field_class(MapField)
class RankedItemsField(WidgetField):
key = 'ranked-items'
description = N_('Ranked Items')
allow_complex = True
items = []
randomize_items = False
@ -2711,6 +2721,8 @@ register_field_class(PasswordField)
class BlockField(WidgetField):
key = 'block'
allow_complex = True
widget_class = BlockWidget
max_items = 1
extra_attributes = ['block', 'max_items', 'add_element_label', 'label_display']

View File

@ -14,6 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
from contextlib import contextmanager
import json
import os
import random
@ -91,6 +92,8 @@ class WcsPublisher(StubWcsPublisher):
tracking_code_class = TrackingCode
unpickler_class = UnpicklerClass
complex_data_cache = None
@classmethod
def get_backoffice_module(cls):
import backoffice
@ -381,6 +384,38 @@ class WcsPublisher(StubWcsPublisher):
from . import sql
sql.cleanup_connection()
@contextmanager
def complex_data(self):
self.complex_data_cache = {}
try:
yield True
finally:
self.complex_data_cache = None
def cache_complex_data(self, value, str_value=None):
# Keep a temporary cache of assocations between a complex data value
# (value) and a string reprensentation (str(value) or the dedicated
# str_value parameter).
#
# It ensures string values are unique by appending a private unicode
# code point, that will be removed in wcs/qommon/template.py.
if self.complex_data_cache is None:
# it doesn't do anything unless initialized.
return str_value or value
if str_value is None:
str_value = str(value)
str_value += chr(0xE000 + len(self.complex_data_cache))
self.complex_data_cache[str_value] = value
return str_value
def get_cached_complex_data(self, value):
if not isinstance(value, str):
return value
return (self.complex_data_cache or {}).get(value)
set_publisher_class(WcsPublisher)
WcsPublisher.register_extra_dir(os.path.join(os.path.dirname(__file__), 'extra'))

View File

@ -16,6 +16,7 @@
import os
import glob
import re
import xml.etree.ElementTree as ET
import django.template
@ -505,7 +506,10 @@ class Template(object):
raise TemplateError(_('failure to render Django template: %s'), e)
else:
return self.value
return force_str(rendered)
rendered = str(rendered)
if context.get('allow_complex'):
return rendered
return re.sub(r'[\uE000-\uF8FF]', '', rendered)
def ezt_render(self, context={}):
fd = StringIO()

View File

@ -624,7 +624,8 @@ class LazyFieldVar(object):
@property
def raw(self):
if self._field.store_display_value or self._field.key in ('file', 'date'):
return self._data.get(self._field.id)
raw_value = self._data.get(self._field.id)
return get_publisher().cache_complex_data(raw_value)
raise AttributeError('raw')
def get_value(self):
@ -640,7 +641,7 @@ class LazyFieldVar(object):
def __str__(self):
value = self.get_value()
if not isinstance(value, six.string_types):
value = str(value)
value = get_publisher().cache_complex_data(value)
return force_str(value)
def __nonzero__(self):
@ -940,7 +941,11 @@ class LazyFieldVarBlock(LazyFieldVar):
def get_value(self):
# don't give access to underlying data dictionary.
return self._data.get('%s_display' % self._field.id, '---')
value = self._data.get(str(self._field.id))
str_value = self._data.get('%s_display' % self._field.id, '---')
if not value:
return str_value
return get_publisher().cache_complex_data(value, str_value)
def __getitem__(self, key):
try:

View File

@ -125,11 +125,20 @@ class SetBackofficeFieldsWorkflowStatusItem(WorkflowStatusItem):
# assign empty value as None, as that will work for all field types
new_value = None
else:
try:
new_value = self.compute(field['value'], raises=True,
formdata=formdata, status_item=self)
except:
continue
with get_publisher().complex_data():
try:
new_value = self.compute(
field['value'],
raises=True,
allow_complex=formdef_field.allow_complex,
formdata=formdata,
status_item=self)
except Exception:
continue
if formdef_field.allow_complex:
complex_value = get_publisher().get_cached_complex_data(new_value)
if complex_value:
new_value = complex_value
if formdef_field.convert_value_from_anything:
try:

View File

@ -405,11 +405,21 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
except KeyError:
missing_fields.append(mapping.field_id)
continue
try:
value = self.compute(mapping.expression, formdata=src, raises=True, status_item=self)
except Exception:
# already logged by self.compute
continue
with get_publisher().complex_data():
try:
value = self.compute(
mapping.expression,
formdata=src,
raises=True,
allow_complex=dest_field.allow_complex,
status_item=self)
except Exception:
# already logged by self.compute
continue
if dest_field.allow_complex:
complex_value = get_publisher().get_cached_complex_data(value)
if complex_value:
value = complex_value
try:
self._set_value(

View File

@ -1965,7 +1965,7 @@ class WorkflowStatusItem(XmlSerialisable):
return {'type': expression_type, 'value': expression_value}
@classmethod
def compute(cls, var, render=True, raises=False, context=None, formdata=None, status_item=None):
def compute(cls, var, render=True, raises=False, allow_complex=False, context=None, formdata=None, status_item=None):
if not isinstance(var, six.string_types):
return var
@ -1993,6 +1993,7 @@ class WorkflowStatusItem(XmlSerialisable):
exception=exception)
if expression['type'] == 'template':
vars['allow_complex'] = allow_complex
try:
return Template(expression['value'], raises=raises, autoescape=False).render(vars)
except TemplateError as e: