general: add lazy evaluation to substitution subvariables (#22106)

tests/test_formdata.py

@@ -6,14 +6,19 @@ import time
 
 from quixote import cleanup
 from quixote.http_request import Upload
+from qommon.template import Template
+from qommon.form import PicklableUpload
 from wcs.qommon.http_request import HTTPRequest
 from wcs import fields, formdef
+from wcs.conditions import Condition
 from wcs.formdef import FormDef
 from wcs.formdata import Evolution
 from wcs.roles import Role
+from wcs.variables import LazyFormData
 from wcs.workflows import Workflow, WorkflowCriticalityLevel, WorkflowBackofficeFieldsFormDef
 from wcs.wf.anonymise import AnonymiseWorkflowStatusItem
 from wcs.wf.wscall import JournalWsCallErrorPart
+
 from wcs.wf.register_comment import JournalEvolutionPart
 
 from utilities import create_temporary_pub, clean_temporary_pub
@@ -125,7 +130,6 @@ def test_field(pub):
     formdef.store()
     formdata = formdef.data_class()()
     substvars = formdata.get_substitution_variables()
-    assert 'form_f0' in substvars
     assert not substvars.get('form_f0')
 
     formdata.data = {'0': 'test'}
@@ -542,3 +546,143 @@ def test_evolution_get_status(pub):
     d = formdef.data_class().get(d.id)
 
     assert [x.get_status().id for x in d.evolution] == ['1', '1', '1', '2', '2']
+
+@pytest.fixture
+def variable_test_data(pub):
+    pub.user_class.wipe()
+    user = pub.user_class()
+    user.email = 'bar@localhost'
+    user.name_identifiers = ['....']
+    user.store()
+
+    role = Role(name='foobar')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'foobarlazy'
+    formdef.fields = [
+        fields.StringField(id='0', label='string', varname='foo_foo'),
+        fields.BoolField(id='1', label='checkbox', varname='boolfield'),
+        fields.BoolField(id='2', label='checkbox', varname='boolfield2'),
+        fields.DateField(id='3', label='date', varname='datefield'),
+        fields.ItemsField(id='4', label='items', items=['aa', 'ab', 'ac'], varname='itemsfield'),
+        fields.FileField(id='5', label='file', varname='filefield'),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.geolocations = {'base': 'Base'}
+    formdef.store()
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.just_created()
+    formdata.user_id = user.id
+    formdata.data = {
+        '0': 'bar',
+        '1': False,
+        '2': True,
+        '3': time.strptime('2018-07-31', '%Y-%m-%d'),
+        '4': ['aa', 'ac'],
+        '4_display': 'aa, ac',
+        '5': PicklableUpload('test.txt', 'text/plain'),
+    }
+    formdata.data['5'].receive(['hello world'])
+    formdata.geolocations = {'base': {'lat': 1, 'lon': 2}}
+    formdata.store()
+    pub.substitutions.feed(formdef)
+    pub.substitutions.feed(formdata)
+
+def test_lazy_formdata(pub, variable_test_data):
+    formdata = FormDef.select()[0].data_class().select()[0]
+    lazy_formdata = LazyFormData(formdata)
+    assert lazy_formdata.receipt_date == time.strftime('%Y-%m-%d', formdata.receipt_time)
+    assert lazy_formdata.receipt_time == time.strftime('%H:%M', formdata.receipt_time)
+    assert lazy_formdata.name == 'foobarlazy'
+    assert lazy_formdata.url.endswith('/foobarlazy/%s/' % formdata.id)
+    assert lazy_formdata.url_backoffice.endswith('/backoffice/management/foobarlazy/%s/' % formdata.id)
+    assert lazy_formdata.backoffice_url == lazy_formdata.url_backoffice
+    assert lazy_formdata.attachments
+    assert lazy_formdata.geoloc['base'] == {'lat': 1, 'lon': 2}
+    assert lazy_formdata.geoloc['base_lon'] == 2
+    static_vars = formdata.get_static_substitution_variables()
+    for attribute in ('name', 'receipt_date', 'receipt_time', 'previous_status',
+            'uri', 'status_changed', 'comment', 'evolution', 'details',
+            'criticality_level', 'digest'):
+        assert getattr(lazy_formdata, attribute) == static_vars['form_' + attribute]
+
+    assert lazy_formdata.user.email == 'bar@localhost'
+    assert lazy_formdata.var.foo_foo == 'bar'
+    assert lazy_formdata.var.boolfield == 'False'
+    assert bool(lazy_formdata.var.boolfield) is False
+    assert lazy_formdata.var.boolfield2 == 'True'
+    assert bool(lazy_formdata.var.boolfield2) is True
+    assert lazy_formdata.var.datefield == time.strptime('2018-07-31', '%Y-%m-%d')
+    assert lazy_formdata.var.itemsfield == 'aa, ac'
+    assert 'aa' in lazy_formdata.var.itemsfield  # taken as a list
+    assert 'aa,' not in lazy_formdata.var.itemsfield  # not as a string
+    assert lazy_formdata.var.filefield == 'test.txt'
+    assert lazy_formdata.var.filefield.raw.base_filename == 'test.txt'
+    assert lazy_formdata.var.filefield.raw.content_type == 'text/plain'
+
+def test_lazy_variables(pub, variable_test_data):
+    formdata = FormDef.select()[0].data_class().select()[0]
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    assert context['form_number'] == formdata.get_display_id()
+    assert context['form_var_foo_foo'] == 'bar'
+    with pytest.raises(KeyError):
+        context['form_var_xxx']
+    assert 'bar' in context['form_var_foo_foo']
+    assert context['form_var_foo_foo'] + 'ab' == 'barab'
+    for item in enumerate(context['form_var_foo_foo']):
+        assert item in [(0, 'b'), (1, 'a'), (2, 'r')]
+
+def test_lazy_conditions(pub, variable_test_data):
+    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo == "bar"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form.var.foo_foo == "bar"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_field_string == "bar"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name == "foobar"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_user_email == "bar@localhost"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_var_filefield_raw_content_type == "text/plain"'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_user_admin_access'})
+    assert condition.evaluate() is False
+
+    condition = Condition({'type': 'django', 'value': 'form_user_backoffice_access'})
+    assert condition.evaluate() is False
+
+    user = pub.user_class.select()[0]
+    user.is_admin = True
+    user.store()
+
+    condition = Condition({'type': 'django', 'value': 'form_user_admin_access'})
+    assert condition.evaluate() is True
+
+    condition = Condition({'type': 'django', 'value': 'form_user_backoffice_access'})
+    assert condition.evaluate() is True
+
+def test_lazy_templates(pub, variable_test_data):
+    context = pub.substitutions.get_context_variables(mode='lazy')
+    tmpl = Template('{{form_var_foo_foo}}')
+    assert tmpl.render(context) == 'bar'
+
+    tmpl = Template('[form_var_foo_foo]')
+    assert tmpl.render(context) == 'bar'
+
+    tmpl = Template('[form_name]')
+    assert tmpl.render(context) == 'foobarlazy'
+
+    tmpl = Template('[form_user_email]')
+    assert tmpl.render(context) == 'bar@localhost'
+
+    tmpl = Template('{{form_user_name_identifier_0}}')
+    assert tmpl.render(context) == pub.user_class.select()[0].name_identifiers[0]

tests/test_workflows.py

@@ -3036,6 +3036,16 @@ def test_set_backoffice_field(http_requests, two_pubs):
     formdata = formdef.data_class().get(formdata.id)
     assert formdata.data['bo1'] == 'HELLO GOODBYE'
 
+    item.fields = [{'field_id': 'bo1', 'value': '{{ form.var.string }} LAZY'}]
+    item.perform(formdata)
+    formdata = formdef.data_class().get(formdata.id)
+    assert formdata.data['bo1'] == 'HELLO LAZY'
+
+    item.fields = [{'field_id': 'bo1', 'value': '=form.var.string'}]  # lazy python
+    item.perform(formdata)
+    formdata = formdef.data_class().get(formdata.id)
+    assert formdata.data['bo1'] == 'HELLO'
+
     assert LoggedError.count() == 0
 
     item.fields = [{'field_id': 'bo1', 'value': '= ~ invalid python ~'}]

wcs/api.py

@@ -100,7 +100,7 @@ def get_formdata_dict(formdata, user, consider_status_visibility=True):
             'status': status.name,
         }
 
-    d.update(formdata.get_substitution_variables(minimal=True))
+    d.update(formdata.get_static_substitution_variables(minimal=True))
     if get_request().form.get('full') == 'on':
         d.update(formdata.get_json_export_dict(include_files=False))
     return d

wcs/backoffice/management.py

@@ -2207,8 +2207,8 @@ class FormBackOfficeStatusPage(FormStatusPage):
         r += htmltext('</div>')
         r += htmltext('<ul class="biglist form-inspector">')
         r += htmltext(' <li><h3>%s</h3></li>') % _('Substitution variables')
-        substvars = self.filled.get_substitution_variables()
-        substvars.update(self.filled.formdef.get_substitution_variables())
+        substvars = self.filled.get_static_substitution_variables()
+        substvars.update(self.filled.formdef.get_static_substitution_variables())
 
         def safe(v):
             if isinstance(v, str):

wcs/conditions.py

@@ -41,7 +41,8 @@ class Condition(object):
         return '<%s (%s) %r>' % (self.__class__.__name__, self.type, self.value)
 
     def get_data(self):
-        return get_publisher().substitutions.get_context_variables()
+        return get_publisher().substitutions.get_context_variables(
+                mode='%s-condition' % self.type)
 
     def evaluate(self):
         if not self.type or not self.value:

wcs/fields.py

@@ -616,6 +616,12 @@ class StringField(WidgetField):
     def get_admin_attributes(self):
         return WidgetField.get_admin_attributes(self) + ['size', 'validation', 'data_source']
 
+    @classmethod
+    def convert_value_from_anything(cls, value):
+        if value is None:
+            return None
+        return str(value)
+
 register_field_class(StringField)
 
 
@@ -1520,7 +1526,7 @@ class PageCondition(Condition):
             # ie do nothing on first page condition
             get_publisher().substitutions.feed(ConditionVars(id(dict_vars)))
 
-        data = get_publisher().substitutions.get_context_variables()
+        data = super(PageCondition, self).get_data()
         # 2) add live data as var_ variables for local evaluation only, for
         # backward compatibility. They are not added globally as they would
         # interfere with the var_ prefixed variables used in dynamic jsonp

wcs/formdata.py

@@ -649,7 +649,7 @@ class FormData(StorableObject):
         endpoint_status_ids = ['wf-%s' % x.id for x in self.formdef.workflow.get_endpoint_status()]
         return (self.status in endpoint_status_ids)
 
-    def get_substitution_variables(self, minimal=False):
+    def get_static_substitution_variables(self, minimal=False):
         d = {}
 
         if self.id:
@@ -685,7 +685,7 @@ class FormData(StorableObject):
             d['form_submission_context'] = self.submission_context
 
         # formdef and category variables
-        d.update(self.formdef.get_substitution_variables(minimal=minimal))
+        d.update(self.formdef.get_static_substitution_variables(minimal=minimal))
 
         if minimal:
             d = copy.deepcopy(d)
@@ -750,7 +750,35 @@ class FormData(StorableObject):
             if hasattr(klass, 'get_substitution_variables'):
                 d.update(klass.get_substitution_variables(self))
 
+        if self.geolocations:
+            for k, v in self.geolocations.items():
+                d['form_geoloc_%s_lat' % k] = v.get('lat')
+                d['form_geoloc_%s_lon' % k] = v.get('lon')
+                d['form_geoloc_%s' % k] = v
+
+        lazy = self.get_substitution_variables()
+        del lazy['form']
+        del lazy['attachments']
+        d.update(lazy)
+
+        d = copy.deepcopy(d)
+        flatten_dict(d)
+
+        return d
+
+    def get_substitution_variables(self, minimal=False):
+        from qommon.substitution import CompatibilityNamesDict
+        from wcs.variables import LazyFormData
+        from wcs.workflows import AttachmentsSubstitutionProxy
+        variables = CompatibilityNamesDict({
+            'form': LazyFormData(self),
+            'attachments': AttachmentsSubstitutionProxy(self),
+        })
+        if minimal:
+            return variables
+
         if self.workflow_data:
+            d = {}
             # pass over workflow data to:
             #  - attach an extra url attribute to uploaded files
             #  - ignore "private" attributes
@@ -759,6 +787,7 @@ class FormData(StorableObject):
                     continue
                 d[k] = v
             # recompute _url variable of attached files
+            form_url = self.get_url()
             for k, v in self.workflow_data.items():
                 if isinstance(v, Upload):
                     try:
@@ -766,19 +795,14 @@ class FormData(StorableObject):
                     except AttributeError:
                         continue
                     d[k.rsplit('_', 1)[0] + '_url'] = '%sfiles/form-%s-%s/%s' % (
-                            d['form_url'], formvar, fieldvar,
+                            form_url, formvar, fieldvar,
                             self.workflow_data['%s_var_%s' % (formvar, fieldvar)])
 
-        if self.geolocations:
-            for k, v in self.geolocations.items():
-                d['form_geoloc_%s_lat' % k] = v.get('lat')
-                d['form_geoloc_%s_lon' % k] = v.get('lon')
-                d['form_geoloc_%s' % k] = v
+            d = copy.deepcopy(d)
+            flatten_dict(d)
+            variables.update(d)
 
-        d = copy.deepcopy(d)
-        flatten_dict(d)
-
-        return d
+        return variables
 
     @classmethod
     def get_substitution_variables_list(cls):

wcs/formdef.py

@@ -1149,7 +1149,7 @@ class FormDef(StorableObject):
 
         return None
 
-    def get_substitution_variables(self, minimal=False):
+    def get_static_substitution_variables(self, minimal=False):
         d = {
             'form_name': self.name,
             'form_slug': self.url_name,
@@ -1161,6 +1161,11 @@ class FormDef(StorableObject):
         d.update(self.get_variable_options())
         return d
 
+    def get_substitution_variables(self, minimal=False):
+        from qommon.substitution import CompatibilityNamesDict
+        from wcs.variables import LazyFormDef
+        return CompatibilityNamesDict({'form': LazyFormDef(self)})
+
     def get_detailed_evolution(self, formdata):
         if not formdata.evolution:
             return None

wcs/qommon/evalutils.py

@@ -37,9 +37,10 @@ def make_date(date_var):
         return date_var
     if isinstance(date_var, time.struct_time):
         return datetime.date(*date_var[:3])
-    if hasattr(date_var, 'decode'):
-        return get_as_datetime(date_var).date()
-    raise ValueError('invalid date value: %s' % date_var)
+    try:
+        return get_as_datetime(str(date_var)).date()
+    except ValueError:
+        raise ValueError('invalid date value: %s' % date_var)
 
 
 def make_datetime(datetime_var):

wcs/qommon/ezt.py

@@ -318,7 +318,12 @@ class Template:
       # attribute-based object.
       class _data_ob:
         def __init__(self, d):
-          vars(self).update(d)
+          self.data = d
+        def __getattr__(self, k):
+          try:
+            return self.data[k]
+          except KeyError:
+            raise AttributeError(k)
       data = _data_ob(data)
 
     ctx = _context()

wcs/qommon/misc.py

@@ -150,9 +150,9 @@ def simplify(s, space='-'):
         return ''
     if not isinstance(s, unicode):
         if get_publisher() and get_publisher().site_charset:
-            s = unicode(s, get_publisher().site_charset, 'ignore')
+            s = unicode('%s' % s, get_publisher().site_charset, 'ignore')
         else:
-            s = unicode(s, 'iso-8859-1', 'ignore')
+            s = unicode('%s' % s, 'iso-8859-1', 'ignore')
     s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')
     s = re.sub(r'[^\w\s\'%s]' % space, '', s).strip().lower()
     s = re.sub(r'[\s\'%s]+' % space, space, s)

wcs/qommon/publisher.py

@@ -51,7 +51,7 @@ from http_request import HTTPRequest
 from http_response import HTTPResponse, AfterJob
 
 from cron import CronJob
-from substitution import Substitutions
+from substitution import Substitutions, CompatibilityNamesDict
 
 import errors
 import template
@@ -139,8 +139,16 @@ class QommonPublisher(Publisher, object):
         import re
         from decimal import Decimal
         from . import evalutils as utils
+        def compat_locals():
+            import inspect
+            frame = inspect.getouterframes(inspect.currentframe())[1][0]
+            x = CompatibilityNamesDict(frame.f_locals)
+            return x
+
         return {'datetime': datetime,
                 'Decimal': Decimal,
+                'locals': compat_locals,
+                'vars': compat_locals,
                 'random': random.SystemRandom(),
                 're': re,
                 'date': utils.date,
@@ -1020,6 +1028,16 @@ class QommonPublisher(Publisher, object):
             ]
         }[context]
 
+    def get_lazy_variables_modes(self):
+        # possible modes:
+        # * django-condition: used to evaluate django conditions
+        # * python-condition: used to evaluate python conditions
+        # * lazy: used to force lazy mode in tests
+        modes = self.get_site_option('lazy-variables-modes')
+        if modes:
+            return [x.strip() for x in modes.split(',')]
+        return ['lazy', 'django-condition']
+
     def get_substitution_variables(self):
         import misc
         d = {

wcs/qommon/substitution.py

@@ -16,7 +16,7 @@
 
 from contextlib import contextmanager
 
-from quixote import get_request
+from quixote import get_request, get_publisher
 from quixote.html import htmltext, TemplateIO
 
 
@@ -84,19 +84,23 @@ class Substitutions(object):
     @classmethod
     def invalidate_cache(cls):
         request = get_request()
-        if hasattr(request, '_cache_context_variables'):
-            delattr(request, '_cache_context_variables')
+        for value in (True, False):
+            if hasattr(request, '_cache_context_variables%r' % value):
+                delattr(request, '_cache_context_variables%r' % value)
 
-    def get_context_variables(self):
+    def get_context_variables(self, mode=None):
+        lazy = mode in get_publisher().get_lazy_variables_modes() if mode else False
         request = get_request()
-        d = getattr(request, '_cache_context_variables', None)
+        d = getattr(request, '_cache_context_variables%r' % lazy, None)
         if d is not None:
             return d
-        d = {}
+        d = CompatibilityNamesDict()
         for source in self.sources:
             d.update(source.get_substitution_variables())
+            if not lazy and hasattr(source, 'get_static_substitution_variables'):
+                d.update(source.get_static_substitution_variables())
         if request:
-            request._cache_context_variables = d
+            setattr(request, '_cache_context_variables%r' % lazy, d)
         return d
 
     @classmethod
@@ -118,3 +122,47 @@ class Substitutions(object):
         r += htmltext('</tbody>')
         r += htmltext('</table>')
         return r.getvalue()
+
+
+class CompatibilityNamesDict(dict):
+    # custom dictionary that provides automatic fallback to legacy variable
+    # names (namespaced with underscores)
+    def get(self, key, default=None):
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getitem__(self, key):
+        try:
+            val = super(CompatibilityNamesDict, self).__getitem__(key)
+        except KeyError:
+            # fallback to deconstructing namespaces around underscores
+            parts = key.split('_')
+            current_dict = self
+            while parts:
+                for i in range(len(parts), 0, -1):
+                    part = '_'.join(parts[:i])
+                    try:
+                        if current_dict is self:
+                            current_dict = dict.__getitem__(current_dict, part)
+                        elif hasattr(current_dict, '__getitem__'):
+                            current_dict = current_dict[part]
+                        else:
+                            current_dict = getattr(current_dict, part)
+                    except (AttributeError, KeyError):
+                        if i == 1:
+                            raise KeyError(key)
+                    else:
+                        parts = parts[i:]
+                        break
+            return current_dict
+
+        return val
+
+    def __contains__(self, key):
+        try:
+            self.__getitem__(key)
+        except KeyError:
+            return False
+        return True

wcs/variables.py

@@ -0,0 +1,429 @@
+# w.c.s. - web application for online forms
+# Copyright (C) 2005-2018  Entr'ouvert
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+from quixote import get_publisher
+
+import qommon.misc
+from qommon.evalutils import make_datetime
+
+
+class LazyFormDef(object):
+    def __init__(self, formdef):
+        self.formdef = formdef
+
+    @property
+    def name(self):
+        return self.formdef.name
+
+    @property
+    def slug(self):
+        return self.formdef.url_name
+
+    @property
+    def objects(self):
+        from wcs.formdef import FormDefSubstVar
+        return FormDefSubstVar(self.formdef)
+
+    @property
+    def option(self):
+        return LazyFormDefOption(self.formdef)
+
+
+class LazyFormDefOption(object):
+    def __init__(self, formdef):
+        self.formdef = formdef
+
+    _options = None
+    @property
+    def options(self):
+        if self._options is not None:
+            return self._options
+        self._options = {}
+        if not self.formdef.workflow.variables_formdef:
+            return self._options
+        if not self.formdef.workflow_options:
+            return self._options
+        for field in self.formdef.workflow.variables_formdef.fields:
+            if not field.varname:
+                continue
+            self._options[field.varname] = self.formdef.workflow_options.get(field.varname)
+            if field.store_display_value:
+                if '%s_display' % field.varname in self.formdef.workflow_options:
+                    self._options[field.varname + '_raw'] = self._options[field.varname]
+                    self._options[field.varname] = self.formdef.workflow_options[
+                            '%s_display' % field.varname]
+            if field.store_structured_value:
+                if '%s_structured' % field.varname in self.formdef.workflow_options:
+                    self._options[field.varname + '_structured'] = self.formdef.workflow_options.get(
+                            '%s_structured' % field.varname)
+        return self._options
+
+    def __getitem__(self, key):
+        return self.options[key]
+
+
+class LazyFormData(LazyFormDef):
+    def __init__(self, formdata):
+        super(LazyFormData, self).__init__(formdata.formdef)
+        self.formdata = formdata
+
+    @property
+    def receipt_date(self):
+        return qommon.misc.strftime(qommon.misc.date_format(), self.formdata.receipt_time)
+
+    @property
+    def receipt_time(self):
+        return qommon.misc.strftime('%H:%M', self.formdata.receipt_time)
+
+    @property
+    def number(self):
+        return self.formdata.get_display_id()
+
+    @property
+    def number_raw(self):
+        return str(self.formdata.id) if self.formdata.id else None
+
+    @property
+    def url(self):
+        return self.formdata.get_url()
+
+    @property
+    def url_backoffice(self):
+        return self.formdata.get_url(backoffice=True)
+
+    @property
+    def backoffice_url(self):
+        return self.formdata.get_url(backoffice=True)
+
+    @property
+    def uri(self):
+        return '%s/%s/' % (self.formdef.url_name, self.formdata.id)
+
+    @property
+    def criticality_level(self):
+        return self.formdata.criticality_level
+
+    @property
+    def digest(self):
+        return self.formdata.digest
+
+    @property
+    def receipt_datetime(self):
+        return make_datetime(self.formdata.receipt_time) if self.formdata.receipt_time else None
+
+    @property
+    def status(self):
+        return self.formdata.get_status_label()
+
+    @property
+    def status_is_endpoint(self):
+        return self.formdata.is_at_endpoint_status()
+
+    @property
+    def tracking_code(self):
+        return self.formdata.tracking_code
+
+    @property
+    def submission_backoffice(self):
+        return self.formdata.backoffice_submission
+
+    @property
+    def submission_channel(self):
+        return self.formdata.submission_channel
+
+    @property
+    def submission_channel_label(self):
+        return self.formdata.get_submission_channel_label()
+
+    @property
+    def submission_context(self):
+        return self.formdata.submission_context
+
+    @property
+    def status_url(self):
+        return '%sstatus' % self.formdata.get_url()
+
+    @property
+    def details(self):
+        return self.formdef.get_detailed_email_form(self.formdata, self.formdata.get_url())
+
+    @property
+    def user(self):
+        user = self.formdata.get_user()
+        return LazyUser(user) if user else None
+
+    @property
+    def var(self):
+        return LazyFormDataVar(self.formdef.get_all_fields(), self.formdata.data, self.formdata)
+
+    @property
+    def field(self):
+        # no lazy dictionary here as it's legacy.
+        d = {}
+        for field in self.formdef.get_all_fields():
+            if not hasattr(field, 'get_view_value'):
+                continue
+            value = self.formdata.data.get(field.id)
+            if value is not None and field.convert_value_to_str:
+                value = field.convert_value_to_str(value)
+            elif value is None:
+                value = ''
+            identifier_name = qommon.misc.simplify(field.label, space='_')
+            d[identifier_name] = value
+
+        return d
+
+    @property
+    def role(self):
+        from wcs.roles import Role
+        workflow_roles = {}
+        if self.formdef.workflow_roles:
+            workflow_roles.update(self.formdef.workflow_roles)
+        if self.formdata.workflow_roles:
+            workflow_roles.update(self.formdata.workflow_roles)
+
+        d = {}
+        for role_type, role_id in workflow_roles.items():
+            prefix = '%s_' % role_type.replace('-', '_').strip('_')
+            try:
+                d.update(Role.get(role_id).get_substitution_variables(prefix))
+            except KeyError:
+                pass
+        return d
+
+    @property
+    def comment(self):
+        if self.formdata.evolution and self.formdata.evolution[-1].comment:
+            return self.formdata.evolution[-1].comment
+        return ''
+
+    @property
+    def attachments(self):
+        from wcs.workflows import AttachmentsSubstitutionProxy
+        return AttachmentsSubstitutionProxy(self.formdata)
+
+    @property
+    def geoloc(self):
+        data = {}
+        if self.formdata.geolocations:
+            for k, v in self.formdata.geolocations.items():
+                data[k] = v
+                data[k + '_lat'] = v.get('lat')
+                data[k + '_lon'] = v.get('lon')
+        return data
+
+    @property
+    def previous_status(self):
+        if self.formdata.evolution:
+            for evolution in reversed(self.formdata.evolution):
+                if evolution.status and evolution.status != self.formdata.status:
+                    return self.formdata.get_status_label(evolution.status)
+        return ''
+
+    @property
+    def status_changed(self):
+        return self.status != self.previous_status
+
+    @property
+    def evolution(self):
+        return self.formdef.get_detailed_evolution(self.formdata)
+
+    def __getitem__(self, key):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            if key.startswith('f'):
+                for field in self.formdef.get_all_fields():
+                    if str(field.id) == str(key[1:]):
+                        return self.formdata.data.get(field.id)
+            raise
+
+
+class LazyFormDataVar(object):
+    def __init__(self, fields, data, formdata=None):
+        self.fields = fields
+        self.data = data or {}
+        self.formdata = formdata
+
+    _varnames = None
+    @property
+    def varnames(self):
+        if self._varnames is not None:
+            return self._varnames
+        self._varnames = {}
+        for field in self.fields:
+            if not field.varname:
+                continue
+            self._varnames[field.varname] = field
+        return self._varnames
+
+    def __getitem__(self, key):
+        try:
+            field = self.varnames[key]
+        except KeyError:
+            # key was unknown but for some  values we may still have to provide
+            # multiple keys (for example file fields will expect to have both
+            # form_var_foo and form_var_foo_raw set to None) and user
+            # conditions may use the "is" operator that cannot be overridden
+            # (this applies to None as well as boolean values).
+            #
+            # Therefore we catch unknown keys with known suffixes ("foo_raw")
+            # and remove the suffix to get the actual field.  If the data is
+            # None or a boolean type, we return it as is.
+            if not (key.endswith('_raw') or key.endswith('_url')):
+                raise
+            maybe_varname = key.rsplit('_', 1)[0]
+            field = self.varnames[maybe_varname]
+            if self.data.get(field.id) in (None, True, False):
+                # valid suffix and data of the correct type
+                return self.data.get(field.id)
+            raise KeyError(key)
+
+        if self.data.get(field.id) is None:
+            return None
+
+        if str(field.id) not in self.data:
+            raise KeyError(key)
+
+        if field.key == 'date':
+            # for backward compatibility with sites using time.struct_time
+            # methods we still have to return a raw value for date fields.
+            return self.data.get(field.id)
+
+        return LazyFieldVar(self.data, field, self.formdata)
+
+    def __getattr__(self, attr):
+        try:
+            return self.__getitem__(attr)
+        except KeyError:
+            raise AttributeError(attr)
+
+
+class LazyFieldVar(object):
+    def __init__(self, data, field, formdata=None):
+        self.data = data
+        self.field = field
+        self.formdata = formdata
+
+    @property
+    def raw(self):
+        if self.field.store_display_value or self.field.key == 'file':
+            return self.data.get(self.field.id)
+        raise KeyError('raw')
+
+    @property
+    def url(self):
+        if self.field.key != 'file' or not self.formdata:
+            raise KeyError('url')
+        return '%sdownload?f=%s' % (self.formdata.get_url(), self.field.id)
+
+    def get_value(self):
+        if self.field.store_display_value:
+            return self.data.get('%s_display' % self.field.id)
+        value = self.data.get(self.field.id)
+        if self.field.convert_value_to_str:
+            return self.field.convert_value_to_str(value)
+        return value
+
+    def __str__(self):
+        return str(self.get_value())
+
+    def __nonzero__(self):
+        if self.field.key == 'bool':
+            return bool(self.data.get(self.field.id))
+        return bool(self.get_value())
+
+    def __contains__(self, value):
+        if self.field.key == 'items':
+            return value in self.data.get(self.field.id)
+        return value in self.get_value()
+
+    def __unicode__(self):
+        return unicode(str(self), get_publisher().site_charset)
+
+    def __eq__(self, other):
+        return str(self) == str(other)
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            return self.get_value()[key]
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            pass
+        structured_value = self.data.get('%s_structured' % self.field.id)
+        if not (self.field.store_structured_value and structured_value):
+            raise KeyError(key)
+        if isinstance(structured_value, dict):
+            return structured_value[key]
+        if isinstance(structured_value, list):
+            for i, struct_value in enumerate(structured_value):
+                if str(key) == str(i):
+                    return struct_value
+        raise KeyError(key)
+
+    def __iter__(self):
+        return iter(self.get_value())
+
+    def __add__(self, other):
+        return self.get_value().__add__(other)
+
+    def __mul__(self, other):
+        return self.get_value().__mul__(other)
+
+    def __getstate__(self):
+        raise AssertionError('lazy cannot be pickled')
+
+
+class LazyUser(object):
+    def __init__(self, user):
+        self.user = user
+
+    @property
+    def display_name(self):
+        return self.user.display_name
+
+    @property
+    def email(self):
+        return self.user.email
+
+    @property
+    def var(self):
+        return LazyFormDataVar(self.user.formdef.fields, self.user.form_data)
+
+    @property
+    def admin_access(self):
+        return self.user.can_go_in_admin()
+
+    @property
+    def backoffice_access(self):
+        return self.user.can_go_in_backoffice()
+
+    @property
+    def name_identifier(self):
+        d = {}
+        for i, name_identifier in enumerate(self.user.name_identifiers):
+            d[str(i)] = name_identifier
+        return d
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+    def __getattr__(self, attr):
+        try:
+            return super(LazyUser, self).__getattr__(attr)
+        except AttributeError:
+            return getattr(self.user, attr)

wcs/wf/geolocate.py

@@ -141,7 +141,7 @@ class GeolocateWorkflowStatusItem(WorkflowStatusItem):
             return
 
         try:
-            lat, lon = value.split(';')
+            lat, lon = str(value).split(';')
             lat_lon = normalize_geolocation({'lon': lon, 'lat': lat})
         except Exception, e:
             get_logger().error('error geolocating from map variable [%r]', e)

wcs/wf/profile.py

@@ -173,9 +173,9 @@ class UpdateUserProfileStatusItem(WorkflowStatusItem):
                 new_data[field.varname] = field_value
 
         if '__name' in new_data:
-            user.name = new_data.get('__name')
+            user.name = str(new_data.get('__name'))
         if '__email' in new_data:
-            user.email = new_data.get('__email')
+            user.email = str(new_data.get('__email'))
         if not user.form_data and new_user_data:
             user.form_data = {}
         if new_user_data: