wcs/wcs/fields.py

# -*- coding: utf-8 -*-
#
# w.c.s. - web application for online forms
# Copyright (C) 2005-2010  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import base64
import collections
import copy
import datetime
import html
import os
import random
import re
import sys
import time
import xml.etree.ElementTree as ET

from django.utils.encoding import force_bytes
from django.utils.encoding import force_text
from django.utils.encoding import smart_text
from django.utils.formats import date_format as django_date_format
from django.utils.html import urlize
from quixote import get_publisher
from quixote import get_request
from quixote import get_session
from quixote.html import TemplateIO
from quixote.html import htmlescape
from quixote.html import htmltag
from quixote.html import htmltext

from . import data_sources
from . import portfolio
from .blocks import BlockDef
from .blocks import BlockWidget
from .conditions import Condition
from .qommon import N_
from .qommon import _
from .qommon import evalutils
from .qommon import force_str
from .qommon import get_cfg
from .qommon import misc
from .qommon.form import AutocompleteStringWidget
from .qommon.form import CheckboxesWidget
from .qommon.form import CheckboxWidget
from .qommon.form import CommentWidget
from .qommon.form import CompositeWidget
from .qommon.form import ComputedExpressionWidget
from .qommon.form import ConditionWidget
from .qommon.form import DateWidget
from .qommon.form import EmailWidget
from .qommon.form import FileSizeWidget
from .qommon.form import FileWithPreviewWidget
from .qommon.form import HiddenWidget
from .qommon.form import HtmlWidget
from .qommon.form import IntWidget
from .qommon.form import JsonpSingleSelectWidget
from .qommon.form import MapMarkerSelectionWidget
from .qommon.form import MapWidget
from .qommon.form import PasswordEntryWidget
from .qommon.form import RadiobuttonsWidget
from .qommon.form import RankedItemsWidget
from .qommon.form import SingleSelectHintWidget
from .qommon.form import SingleSelectTableWidget
from .qommon.form import SingleSelectWidget
from .qommon.form import StringWidget
from .qommon.form import TableListRowsWidget
from .qommon.form import TableWidget
from .qommon.form import TextWidget
from .qommon.form import ValidationWidget
from .qommon.form import VarnameWidget
from .qommon.form import WcsExtraStringWidget
from .qommon.form import WidgetList
from .qommon.form import WidgetListAsTable
from .qommon.form import WysiwygTextWidget
from .qommon.misc import date_format
from .qommon.misc import ellipsize
from .qommon.misc import get_as_datetime
from .qommon.misc import get_document_type_value_options
from .qommon.misc import get_document_types
from .qommon.misc import strftime
from .qommon.misc import xml_node_text
from .qommon.ods import NS as OD_NS
from .qommon.ods import clean_text as od_clean_text
from .qommon.template import Template
from .qommon.template import TemplateError
from .qommon.upload_storage import PicklableUpload


class PrefillSelectionWidget(CompositeWidget):
    def __init__(self, name, value=None, field=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)

        if not value:
            value = {}

        options = [
            ('none', _('None')),
            ('string', _('String / Template')),
            ('formula', _('Python Expression')),
            ('user', _('User Field')),
            ('geolocation', _('Geolocation')),
        ]

        if field and field.type == 'items':
            # limit choices to python as items field are prefilled with list
            # of strings
            options = [x for x in options if x[0] in ('none', 'formula')]
        elif field and field.type == 'map':
            # limit choices to geolocation
            options = [x for x in options if x[0] in ('none', 'string', 'geolocation')]

        self.add(
            SingleSelectWidget,
            'type',
            options=options,
            value=value.get('type'),
            attrs={'data-dynamic-display-parent': 'true'},
        )

        self.parse()
        if not self.value:
            self.value = {}

        prefill_types = collections.OrderedDict(options)
        self.add(
            StringWidget,
            'value_string',
            size=80,
            value=value.get('value') if value.get('type') == 'string' else None,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('string'),
            },
        )
        self.add(
            StringWidget,
            'value_formula',
            size=80,
            value=value.get('value') if value.get('type') == 'formula' else None,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('formula'),
            },
        )

        formdef = get_publisher().user_class.get_formdef()
        users_cfg = get_cfg('users', {})
        if formdef:
            user_fields = []
            for user_field in formdef.fields:
                if user_field.label in [x[1] for x in user_fields]:
                    # do not allow duplicated field names
                    continue
                user_fields.append((user_field.id, user_field.label))
            if not users_cfg.get('field_email'):
                user_fields.append(('email', _('Email (builtin)')))
        else:
            user_fields = [('name', _('Name')), ('email', _('Email'))]
        self.add(
            SingleSelectWidget,
            'value_user',
            value=value.get('value') if value.get('type') == 'user' else None,
            options=user_fields,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('user'),
            },
        )

        if field and field.type == 'map':
            # different prefilling sources on map fields
            geoloc_fields = [('position', _('Position'))]
        else:
            geoloc_fields = [
                ('house', _('Number')),
                ('road', _('Street')),
                ('number-and-street', _('Number and street')),
                ('postcode', _('Post Code')),
                ('city', _('City')),
                ('country', _('Country')),
            ]
        self.add(
            SingleSelectWidget,
            'value_geolocation',
            value=value.get('value') if value.get('type') == 'geolocation' else None,
            options=geoloc_fields,
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value': prefill_types.get('geolocation'),
            },
        )

        # exclude geolocation from locked prefill as the data necessarily
        # comes from the user device.
        self.add(
            CheckboxWidget,
            'locked',
            value=value.get('locked'),
            attrs={
                'data-dynamic-display-child-of': 'prefill$type',
                'data-dynamic-display-value-in': '|'.join(
                    [x[1] for x in options if x[0] not in ('none', 'geolocation')]
                ),
                'inline_title': _('Locked'),
            },
        )

        self._parsed = False

    def _parse(self, request):
        values = {}
        type_ = self.get('type')
        if type_:
            values['type'] = type_
            values['locked'] = self.get('locked')
            value = self.get('value_%s' % type_)
            if value:
                values['value'] = value
        self.value = values or None
        if values and values['type'] == 'formula' and values.get('value'):
            try:
                compile(values.get('value', ''), '<string>', 'eval')
            except (SyntaxError, TypeError) as e:
                self.set_error(_('invalid expression: %s') % e)
        if values and values['type'] == 'string' and Template.is_template_string(values.get('value')):
            try:
                Template(values.get('value'), raises=True)
            except TemplateError as e:
                self.set_error(str(e))

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_content()
        return r.getvalue()


class Field:
    id = None
    type = None
    varname = None
    label = None
    extra_css_class = None
    convert_value_from_str = None
    convert_value_to_str = None
    convert_value_from_anything = None
    allow_complex = False
    display_locations = []
    prefill = None
    store_display_value = None
    store_structured_value = None
    get_opendocument_node_value = None
    condition = None

    # flag a field for removal by AnonymiseWorkflowStatusItem
    # can be overriden in field' settings
    anonymise = True
    stats = None

    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k.replace('-', '_'), v)

    @classmethod
    def init(cls):
        pass

    def get_type_label(self):
        return _(self.description)

    @property
    def include_in_listing(self):
        return 'listings' in (self.display_locations or [])

    @property
    def include_in_validation_page(self):
        return 'validation' in (self.display_locations or [])

    @property
    def include_in_summary_page(self):
        return 'summary' in (self.display_locations or [])

    @property
    def unhtmled_label(self):
        return force_str(html.unescape(force_text(re.sub('<.*?>', ' ', self.label or ''))).strip())

    def get_admin_attributes(self):
        return ['label', 'type', 'condition']

    def export_to_json(self, include_id=False, anonymise=True):
        field = {}
        if include_id:
            extra_fields = ['id']
        else:
            extra_fields = []
        for attribute in self.get_admin_attributes() + extra_fields:
            if attribute == 'display_locations':
                continue
            if hasattr(self, attribute) and getattr(self, attribute) is not None:
                val = getattr(self, attribute)
                field[attribute] = val
        if not 'type' in field:
            field['type'] = self.key
        return field

    def init_with_json(self, elem, include_id=False):
        if include_id:
            self.id = elem.get('id')
        for attribute in self.get_admin_attributes():
            if attribute in elem:
                setattr(self, attribute, elem.get(attribute))

    def export_to_xml(self, charset, include_id=False):
        field = ET.Element('field')
        if include_id:
            extra_fields = ['id']
        else:
            extra_fields = []
        for attribute in self.get_admin_attributes() + extra_fields:
            if hasattr(self, '%s_export_to_xml' % attribute):
                getattr(self, '%s_export_to_xml' % attribute)(field, charset, include_id=include_id)
                continue
            if hasattr(self, attribute) and getattr(self, attribute) is not None:
                val = getattr(self, attribute)
                if type(val) is dict and not val:
                    continue
                el = ET.SubElement(field, attribute)
                if type(val) is dict:
                    for k, v in sorted(val.items()):
                        if isinstance(v, str):
                            text_value = force_text(v, charset, errors='replace')
                        else:
                            # field having non str value in dictionnary field must overload
                            # import_to_xml to handle import
                            text_value = force_text(v)
                        ET.SubElement(el, k).text = text_value
                elif type(val) is list:
                    if attribute[-1] == 's':
                        atname = attribute[:-1]
                    else:
                        atname = 'item'
                    for v in val:
                        ET.SubElement(el, atname).text = force_text(v, charset, errors='replace')
                elif isinstance(val, str):
                    el.text = force_text(val, charset, errors='replace')
                else:
                    el.text = str(val)
        return field

    def init_with_xml(self, elem, charset, include_id=False, snapshot=False):
        for attribute in self.get_admin_attributes():
            el = elem.find(attribute)
            if hasattr(self, '%s_init_with_xml' % attribute):
                getattr(self, '%s_init_with_xml' % attribute)(
                    el, charset, include_id=include_id, snapshot=False
                )
                continue
            if el is None:
                continue
            if list(el):
                if type(getattr(self, attribute)) is list:
                    v = [xml_node_text(x) for x in el]
                elif type(getattr(self, attribute)) is dict:
                    v = {}
                    for e in el:
                        v[e.tag] = xml_node_text(e)
                else:
                    print('currently:', self.__dict__)
                    print('  attribute:', attribute)
                    # ???
                    raise AssertionError
                setattr(self, attribute, v)
            else:
                if el.text is None:
                    if isinstance(getattr(self, attribute), list):
                        setattr(self, attribute, [])
                    else:
                        setattr(self, attribute, None)
                elif el.text in ('False', 'True'):  # bools
                    setattr(self, attribute, el.text == 'True')
                elif type(getattr(self, attribute)) is int:
                    setattr(self, attribute, int(el.text))
                else:
                    setattr(self, attribute, xml_node_text(el))
        if include_id:
            try:
                self.id = xml_node_text(elem.find('id'))
            except Exception:
                pass

    def condition_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.condition = None
        if node is None:
            return
        if node.findall('type'):
            self.condition = {
                'type': xml_node_text(node.find('type')),
                'value': xml_node_text(node.find('value')),
            }
        elif node.text:
            self.condition = {'type': 'python', 'value': force_str(node.text).strip()}

    def data_source_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.data_source = {}
        if node is None:
            return
        if node.findall('type'):
            self.data_source = {
                'type': xml_node_text(node.find('type')),
                'value': xml_node_text(node.find('value')),
            }
            if self.data_source.get('type') is None:
                self.data_source = {}
            elif self.data_source.get('value') is None:
                del self.data_source['value']

    def prefill_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        self.prefill = {}
        if node is not None and node.findall('type'):
            self.prefill = {
                'type': xml_node_text(node.find('type')),
            }
            if self.prefill['type'] and self.prefill['type'] != 'none':
                self.prefill['value'] = xml_node_text(node.find('value'))
                if xml_node_text(node.find('locked')) == 'True':
                    self.prefill['locked'] = True

    def get_rst_view_value(self, value, indent=''):
        return indent + self.get_view_value(value)

    def get_csv_heading(self):
        return []

    def get_csv_value(self, element, **kwargs):
        return []

    def get_structured_value(self, data):
        if not self.store_structured_value:
            return None
        return data.get('%s_structured' % self.id)

    def get_prefill_value(self, user=None, force_string=True):
        # returns a tuple with two items,
        #  1. value[str], the value that will be used to prefill
        #  2. locked[bool], a flag to know if this is a locked value
        #     (because it has been explicitely marked so or because it
        #     comes from verified identity data).
        t = self.prefill.get('type')
        explicit_lock = bool(self.prefill.get('locked'))
        if t == 'string':
            value = self.prefill.get('value')
            if not Template.is_template_string(value):
                return (value, explicit_lock)

            context = get_publisher().substitutions.get_context_variables(mode='lazy')
            try:
                return (Template(value, autoescape=False, raises=True).render(context), explicit_lock)
            except TemplateError:
                return ('', explicit_lock)
            except AttributeError as e:
                get_publisher().record_error(
                    _('Failed to evaluate prefill on field "%s"') % self.label,
                    formdef=getattr(self, 'formdef', None),
                    exception=e,
                )
                return ('', explicit_lock)

        elif t == 'user' and user:
            x = self.prefill.get('value')
            if x == 'email':
                return (user.email, explicit_lock or 'email' in (user.verified_fields or []))
            elif user.form_data:
                userform = user.get_formdef()
                for userfield in userform.fields:
                    if userfield.id == x:
                        return (
                            user.form_data.get(x),
                            explicit_lock or str(userfield.id) in (user.verified_fields or []),
                        )

        elif t == 'formula':
            formula = self.prefill.get('value')
            try:
                ret = eval(
                    formula,
                    get_publisher().get_global_eval_dict(),
                    get_publisher().substitutions.get_context_variables(),
                )
                if isinstance(ret, datetime.time):
                    ret = misc.site_encode(django_date_format(ret, format='TIME_FORMAT'))
                if isinstance(ret, datetime.date):
                    ret = ret.strftime(date_format())
                if ret:
                    if force_string:
                        # prefilling is done with strings for most fields so
                        # we default to forcing the value as a string.
                        # (items field are prefilled with list of strings, and
                        # will get the native python object)
                        ret = str(ret)
                    return (ret, explicit_lock)
            except Exception:
                pass

        elif t == 'geolocation':
            return (None, False)

        return (None, False)

    def get_prefill_attributes(self):
        if not self.prefill:
            return
        t = self.prefill.get('type')

        if t == 'geolocation':
            return {'geolocation': self.prefill.get('value')}

        if t == 'user':
            formdef = get_publisher().user_class.get_formdef()
            for user_field in formdef.fields or []:
                if user_field.id != self.prefill.get('value'):
                    continue
                try:
                    autocomplete_attribute = re.search(
                        r'\bautocomplete-([a-z0-9-]+)', user_field.extra_css_class
                    ).groups()[0]
                except (TypeError, IndexError, AttributeError):
                    continue
                return {'autocomplete': autocomplete_attribute}

        return None

    def feed_session(self, value, display_value):
        pass

    def migrate(self):
        changed = False
        if getattr(self, 'in_listing', None) is not None:
            if self.in_listing:
                self.display_locations = self.display_locations[:]
                self.display_locations.append('listings')
                changed = True
                self.in_listing = None
        # repair dictionary attributes that may have been kept as bytes in
        # the initial python 2 -> 3 conversion.
        from wcs.qommon.storage import deep_bytes2str

        for key in ('prefill', 'data_source'):
            value = getattr(self, key, None)
            if not value:
                continue
            if b'type' in value:
                setattr(self, key, deep_bytes2str(getattr(self, key)))
        return changed

    def evaluate_condition(self, dict_vars, formdef, condition):
        return PageCondition(condition, {'dict_vars': dict_vars, 'formdef': formdef}).evaluate()

    def is_visible(self, dict, formdef):
        try:
            return self.evaluate_condition(dict, formdef, self.condition)
        except RuntimeError:
            return True

    @classmethod
    def get_referenced_varnames(cls, formdef, value):
        return re.findall(
            r'\b%s[_\.]var[_\.]([a-zA-Z0-9_]+?)(?:_raw|_live_|_structured_|\b)' % formdef.var_prefix,
            value or '',
        )

    def get_condition_varnames(self, formdef):
        return self.get_referenced_varnames(formdef, self.condition['value'])

    def has_live_conditions(self, formdef):
        varnames = self.get_condition_varnames(formdef)
        if not varnames:
            return False
        field_position = formdef.fields.index(self)
        # rewind to field page
        for field_position in range(field_position, -1, -1):
            if formdef.fields[field_position].type == 'page':
                break
        else:
            field_position = -1  # form with no page
        # start from there
        for field in formdef.fields[field_position + 1 :]:
            if field.type == 'page':
                # stop at next page
                break
            if field.varname in varnames:
                return True
        return False

    def set_value(self, data, value):
        data['%s' % self.id] = value
        if self.store_display_value:
            display_value = self.store_display_value(data, self.id)
            if display_value:
                data['%s_display' % self.id] = display_value
            elif '%s_display' % self.id in data:
                del data['%s_display' % self.id]
        if self.store_structured_value and value:
            structured_value = self.store_structured_value(data, self.id)
            if structured_value:
                if isinstance(structured_value, dict) and structured_value.get('id'):
                    # in case of list field, override id
                    data['%s' % self.id] = str(structured_value.get('id'))
                data['%s_structured' % self.id] = structured_value
            elif '%s_structured' % self.id in data:
                del data['%s_structured' % self.id]
        elif self.store_structured_value and '%s_structured' % self.id in data:
            del data['%s_structured' % self.id]

    def __repr__(self):
        return '<%s %s %r>' % (self.__class__.__name__, self.id, self.label and self.label[:64])


class WidgetField(Field):
    hint = None
    required = True
    display_locations = ['validation', 'summary']
    extra_attributes = []
    prefill = {}

    widget_class = None

    def add_to_form(self, form, value=None):
        kwargs = {'required': self.required, 'render_br': False}
        if value:
            kwargs['value'] = value
        for k in self.extra_attributes:
            if hasattr(self, k):
                kwargs[k] = getattr(self, k)
        self.perform_more_widget_changes(form, kwargs)
        if self.hint and self.hint.startswith('<'):
            hint = htmltext(self.hint)
        else:
            hint = self.hint
        form.add(self.widget_class, 'f%s' % self.id, title=self.label, hint=hint, **kwargs)
        widget = form.get_widget('f%s' % self.id)
        widget.field = self
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class
        if self.varname:
            widget.div_id = 'var_%s' % self.varname
        return widget

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        pass

    def add_to_view_form(self, form, value=None):
        kwargs = {'render_br': False}

        self.field_key = 'f%s' % self.id
        self.perform_more_widget_changes(form, kwargs, False)

        for k in self.extra_attributes:
            if hasattr(self, k):
                kwargs[k] = getattr(self, k)

        if self.widget_class is StringWidget and not 'size' in kwargs and value:
            # set a size if there is not one already defined, this will be for
            # example the case with ItemField
            kwargs['size'] = len(value)

        form.add(
            self.widget_class, self.field_key, title=self.label, value=value, readonly='readonly', **kwargs
        )
        widget = form.get_widget(self.field_key)
        widget.transfer_form_value(get_request())
        widget.field = self
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class

    def get_display_locations_options(self):
        return [
            ('validation', _('Validation Page')),
            ('summary', _('Summary Page')),
            ('listings', _('Management Listings')),
        ]

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(CheckboxWidget, 'required', title=_('Required'), value=self.required)
        form.add(
            VarnameWidget,
            'varname',
            title=_('Identifier'),
            value=self.varname,
            size=30,
            advanced=(not self.varname),
            hint=_('This is used as suffix for variable names.'),
        )
        form.add(TextWidget, 'hint', title=_('Hint'), value=self.hint, cols=60, rows=3)
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            advanced=True,
        )
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            advanced=(not self.extra_css_class),
        )
        prefill_in_advanced = not self.prefill or self.prefill.get('type') == 'none'
        if prefill_in_advanced and (
            get_request().form.get('prefill$apply') or get_request().form.get('prefill$value')
        ):
            # check if the apply button has been clicked, and get the field out
            # of the "advanced parameters" section if that's the case.
            prefill_in_advanced = False
        form.add(
            PrefillSelectionWidget,
            'prefill',
            title=_('Prefill'),
            value=self.prefill,
            advanced=prefill_in_advanced,
            field=self,
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            django_only=True,
            advanced=not (bool(self.condition)),
        )
        if 'anonymise' in self.get_admin_attributes():
            # override anonymise flag default value
            form.add(
                CheckboxWidget,
                'anonymise',
                title=_('Anonymise'),
                value=self.anonymise,
                advanced=True,
                hint=_('Marks the field data for removal in the anonymisation processes.'),
            )

    def check_admin_form(self, form):
        return

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + [
            'required',
            'hint',
            'varname',
            'display_locations',
            'extra_css_class',
            'prefill',
        ]

    def get_csv_heading(self):
        return [self.label]

    def get_value_info(self, data):
        # return the selected value and an optional dictionary that will be
        # passed to get_view_value() to provide additional details.
        value_details = {}
        if self.id not in data:
            value = None
        else:
            if self.store_display_value and ('%s_display' % self.id) in data:
                value = data['%s_display' % self.id]
                value_details['value_id'] = data[self.id]
            else:
                value = data[self.id]

            if value is None or value == '':
                value = None
        return (value, value_details)

    def get_view_value(self, value, **kwargs):
        return str(value) if value else ''

    def get_view_short_value(self, value, max_len=30, **kwargs):
        return self.get_view_value(value)

    def get_csv_value(self, element, **kwargs):
        if self.convert_value_to_str:
            return [self.convert_value_to_str(element)]
        return [element]


field_classes = []
field_types = []


def register_field_class(klass):
    if not klass in field_classes:
        field_classes.append(klass)
        if not issubclass(klass, WidgetField):
            field_types.append((klass.key, klass.description))
        else:
            non_widgets = [x for x in field_classes if not issubclass(x, WidgetField)]
            if not non_widgets:
                field_types.append((klass.key, klass.description))
            else:
                idx = field_types.index([x for x in field_types if x[0] == non_widgets[0].key][0])
                field_types.insert(idx, (klass.key, klass.description))
        klass.init()


class TitleField(Field):
    key = 'title'
    description = N_('Title')
    html_tag = 'h3'
    display_locations = ['validation', 'summary']

    def add_to_form(self, form, value=None):
        import wcs.workflows

        extra_attributes = ' data-field-id="%s"' % self.id
        if self.extra_css_class:
            extra_attributes += ' class="%s"' % self.extra_css_class
        title_markup = '<{html_tag}{extra_attributes}>%s</{html_tag}>'.format(
            html_tag=self.html_tag,
            extra_attributes=extra_attributes,
        )
        label = wcs.workflows.template_on_formdata(None, self.label, autoescape=False)
        widget = HtmlWidget(htmltext(title_markup) % label)
        widget.field = self
        form.widgets.append(widget)
        return widget

    add_to_view_form = add_to_form

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            advanced=(not self.extra_css_class),
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            django_only=True,
            advanced=not (bool(self.condition)),
        )
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            advanced=True,
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['extra_css_class', 'display_locations']


register_field_class(TitleField)


class SubtitleField(TitleField):
    key = 'subtitle'
    description = N_('Subtitle')
    html_tag = 'h4'


register_field_class(SubtitleField)


class CommentField(Field):
    key = 'comment'
    description = N_('Comment')
    display_locations = []

    def get_text(self):
        import wcs.workflows

        label = self.get_html_content()
        return wcs.workflows.template_on_html_string(label)

    def add_to_form(self, form, value=None):
        widget = CommentWidget(content=self.get_text(), extra_css_class=self.extra_css_class)
        form.widgets.append(widget)
        widget.field = self
        return widget

    def add_to_view_form(self, *args, **kwargs):
        if self.include_in_validation_page:
            return self.add_to_form(*args, **kwargs)
        return None

    def get_html_content(self):
        if not self.label:
            return ''
        if self.label.startswith('<'):
            return self.label
        if '\n\n' in self.label:
            # blank lines to paragraphs
            label = '</p>\n<p>'.join([str(htmlescape(x)) for x in re.split('\n\n+', self.label)])
            return '<p>' + label + '</p>'
        return '<p>%s</p>' % str(htmlescape(self.label))

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        if self.label and (self.label[0] != '<' and '[end]' in self.label):
            form.add(
                TextWidget,
                'label',
                title=_('Label'),
                value=self.label,
                validation_function=ComputedExpressionWidget.validate_template,
                required=True,
                cols=70,
                rows=3,
                render_br=False,
            )
        else:
            form.add(
                WysiwygTextWidget,
                'label',
                title=_('Label'),
                validation_function=ComputedExpressionWidget.validate_template,
                value=self.get_html_content(),
                required=True,
            )
        form.add(
            StringWidget,
            'extra_css_class',
            title=_('Extra classes for CSS styling'),
            value=self.extra_css_class,
            size=30,
            advanced=(not self.extra_css_class),
        )
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
            django_only=True,
            advanced=not (bool(self.condition)),
        )
        form.add(
            CheckboxesWidget,
            'display_locations',
            title=_('Display Locations'),
            options=self.get_display_locations_options(),
            value=self.display_locations,
            advanced=True,
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['extra_css_class', 'display_locations']


register_field_class(CommentField)


def is_datasource_advanced(value):
    data_source_in_advanced = not value
    if data_source_in_advanced and (
        get_request().form.get('data_source$apply') or get_request().form.get('data_source$value')
    ):
        data_source_in_advanced = False
    return data_source_in_advanced


class StringField(WidgetField):
    key = 'string'
    description = N_('Text (line)')

    widget_class = WcsExtraStringWidget
    size = None
    extra_attributes = ['size']
    validation = {}
    data_source = {}

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if self.data_source:
            data_source = data_sources.get_object(self.data_source)
            if data_source.can_jsonp():
                kwargs['url'] = data_source.get_jsonp_url()
                self.widget_class = AutocompleteStringWidget

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        if self.size:
            form.add(
                StringWidget,
                'size',
                title=_('Line length'),
                hint=_(
                    'Deprecated option, it is advised to use CSS classes '
                    'to size the fields in a manner compatible with all devices.'
                ),
                value=self.size,
            )
        else:
            form.add(HiddenWidget, 'size', value=None)
        form.add(
            ValidationWidget,
            'validation',
            title=_('Validation'),
            value=self.validation,
            advanced=(not self.validation),
        )
        form.add(
            data_sources.DataSourceSelectionWidget,
            'data_source',
            value=self.data_source,
            title=_('Data Source'),
            hint=_('This will allow autocompletion from an external source.'),
            advanced=is_datasource_advanced(self.data_source),
            required=False,
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + ['size', 'validation', 'data_source', 'anonymise']

    def get_view_value(self, value, **kwargs):
        value = value or ''
        if value.startswith('http://') or value.startswith('https://'):
            charset = get_publisher().site_charset
            value = force_text(value, charset)
            return htmltext(force_str(urlize(value, nofollow=True, autoescape=True)))
        return str(value)

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        if value.startswith('http://') or value.startswith('https://'):
            node = ET.Element('{%s}a' % OD_NS['text'])
            node.attrib['{%s}href' % OD_NS['xlink']] = value
        else:
            node = ET.Element('{%s}span' % OD_NS['text'])
        node.text = od_clean_text(force_text(value))
        return node

    def get_rst_view_value(self, value, indent=''):
        return indent + str(value or '')

    def convert_value_from_str(self, value):
        return value

    @classmethod
    def convert_value_from_anything(cls, value):
        if value is None:
            return None
        return str(value)

    def migrate(self):
        changed = super().migrate()
        if isinstance(self.validation, str):
            self.validation = {'type': 'regex', 'value': self.validation}
            changed = True
        return changed

    def init_with_xml(self, element, charset, include_id=False, snapshot=False):
        super().init_with_xml(element, charset, include_id=include_id)
        self.migrate()


register_field_class(StringField)


class TextField(WidgetField):
    key = 'text'
    description = N_('Long Text')

    widget_class = TextWidget
    cols = None
    rows = None
    pre = False
    maxlength = None
    extra_attributes = ['cols', 'rows', 'maxlength']

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        if self.cols:
            form.add(
                StringWidget,
                'cols',
                title=_('Line length'),
                hint=_(
                    'Deprecated option, it is advised to use CSS classes '
                    'to size the fields in a manner compatible with all devices.'
                ),
                value=self.cols,
            )
        else:
            form.add(HiddenWidget, 'cols', value=None)
        form.add(StringWidget, 'rows', title=_('Number of rows'), value=self.rows)
        form.add(StringWidget, 'maxlength', title=_('Maximum number of characters'), value=self.maxlength)
        form.add(CheckboxWidget, 'pre', title=_('Preformatted Text'), value=self.pre, advanced=not (self.pre))

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + ['cols', 'rows', 'pre', 'maxlength']

    def convert_value_from_str(self, value):
        return value

    def get_view_value(self, value, **kwargs):
        if self.pre:
            return htmltext('<pre>') + value + htmltext('</pre>')
        else:
            try:
                return (
                    htmltext('<p>')
                    + htmltext('\n').join([(x or htmltext('</p><p>')) for x in value.splitlines()])
                    + htmltext('</p>')
                )
            except Exception:
                return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        paragraphs = []
        for paragraph in value.splitlines():
            if paragraph.strip():
                p = ET.Element('{%s}p' % OD_NS['text'])
                p.text = paragraph
                paragraphs.append(p)
        return paragraphs

    def get_view_short_value(self, value, max_len=30):
        return ellipsize(str(value), max_len)


register_field_class(TextField)


class EmailField(WidgetField):
    key = 'email'
    description = N_('Email')

    widget_class = EmailWidget

    def convert_value_from_str(self, value):
        return value

    def get_view_value(self, value, **kwargs):
        return htmltext('<a href="mailto:%s">%s</a>') % (value, value)

    def get_rst_view_value(self, value, indent=''):
        return indent + value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        a = ET.Element('{%s}a' % OD_NS['text'])
        a.text = od_clean_text(value)
        a.attrib['{%s}href' % OD_NS['xlink']] = 'mailto:' + a.text
        return a


register_field_class(EmailField)


class BoolField(WidgetField):
    key = 'bool'
    description = N_('Check Box (single choice)')
    allow_complex = True

    widget_class = CheckboxWidget
    required = False
    anonymise = False

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if not edit:
            kwargs['disabled'] = 'disabled'
            value = get_request().get_field(self.field_key)
            form.add_hidden(self.field_key, value=str(value))
            widget = form.get_widget(self.field_key)
            widget.field = self
            if value and not value == 'False':
                self.field_key = 'f%sdisabled' % self.id
                get_request().form[self.field_key] = 'yes'
            self.field_key = 'f%sdisabled' % self.id

    def get_view_value(self, value, **kwargs):
        if value is True or value == 'True':
            return _('Yes')
        elif value is False or value == 'False':
            return _('No')
        else:
            return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def convert_value_from_anything(self, value):
        if isinstance(value, str):
            return self.convert_value_from_str(value)
        return bool(value)

    def convert_value_from_str(self, value):
        if value is None:
            return None
        for true_word in (N_('True'), N_('Yes')):
            if str(value).lower() in (true_word.lower(), _(true_word).lower()):
                return True
        return False

    def convert_value_to_str(self, value):
        if value is True:
            return 'True'
        elif value is False:
            return 'False'
        return value

    def stats(self, values):
        no_records = len(values)
        if not no_records:
            return
        r = TemplateIO(html=True)
        r += htmltext('<table class="stats">')
        r += htmltext('<thead><tr><th colspan="4">')
        r += self.label
        r += htmltext('</th></tr></thead>')
        options = (True, False)
        r += htmltext('<tbody>')
        for o in options:
            r += htmltext('<tr>')
            r += htmltext('<td class="label">')
            if o is True:
                r += _('Yes')
                value = True
            else:
                r += _('No')
                value = False
            r += htmltext('</td>')
            no = len([None for x in values if self.convert_value_from_str(x.data.get(self.id)) is value])

            r += htmltext('<td class="percent">')
            r += htmltext(' %.2f&nbsp;%%') % (100.0 * no / no_records)
            r += htmltext('</td>')
            r += htmltext('<td class="total">')
            r += '(%d/%d)' % (no, no_records)
            r += htmltext('</td>')
            r += htmltext('</tr>')
            r += htmltext('<tr>')
            r += htmltext('<td class="bar" colspan="3">')
            r += htmltext('<span style="width: %d%%"></span>' % (100 * no / no_records))
            r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody>')
        r += htmltext('</table>')
        return r.getvalue()


register_field_class(BoolField)


class FileField(WidgetField):
    key = 'file'
    description = N_('File Upload')
    allow_complex = True

    document_type = None
    max_file_size = None
    automatic_image_resize = False
    allow_portfolio_picking = False
    storage = 'default'

    widget_class = FileWithPreviewWidget
    extra_attributes = [
        'file_type',
        'max_file_size',
        'allow_portfolio_picking',
        'automatic_image_resize',
        'storage',
    ]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.document_type = self.document_type or {}

    @property
    def file_type(self):
        return (self.document_type or {}).get('mimetypes', [])

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        options = get_document_type_value_options(self.document_type)
        form.add(
            SingleSelectWidget,
            'document_type',
            title=_('File type suggestion'),
            value=self.document_type,
            options=options,
            advanced=not (self.document_type),
        )
        form.add(
            FileSizeWidget,
            'max_file_size',
            title=_('Max file size'),
            value=self.max_file_size,
            advanced=not (self.max_file_size),
        )
        form.add(
            CheckboxWidget,
            'automatic_image_resize',
            title=_('Automatically resize uploaded images'),
            value=self.automatic_image_resize,
            advanced=True,
        )
        if portfolio.has_portfolio():
            form.add(
                CheckboxWidget,
                'allow_portfolio_picking',
                title=_('Allow user to pick a file from a portfolio'),
                value=self.allow_portfolio_picking,
                advanced=(self.allow_portfolio_picking is FileField.allow_portfolio_picking),
            )
        storages = get_publisher().get_site_storages()
        if storages:
            storage_options = [('default', '---', {})]
            storage_options += [(key, value['label'], key) for key, value in storages.items()]
            form.add(
                SingleSelectWidget,
                'storage',
                title=_('File storage system'),
                value=self.storage,
                options=storage_options,
                advanced=bool(not self.storage or self.storage == 'default'),
            )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'document_type',
            'max_file_size',
            'allow_portfolio_picking',
            'automatic_image_resize',
            'storage',
        ]

    @classmethod
    def convert_value_from_anything(cls, value):
        if not value:
            return None
        from wcs.variables import LazyFieldVarFile

        if isinstance(value, LazyFieldVarFile):
            value = value.get_value()  # unbox
        if hasattr(value, 'base_filename'):
            upload = PicklableUpload(value.base_filename, value.content_type or 'application/octet-stream')
            upload.receive([value.get_content()])
            return upload
        from wcs.workflows import NamedAttachmentsSubstitutionProxy

        if isinstance(value, NamedAttachmentsSubstitutionProxy):
            upload = PicklableUpload(value.filename, value.content_type)
            upload.receive([value.content])
            return upload
        if isinstance(value, dict):
            # if value is a dictionary we expect it to have a content or
            # b64_content key and a filename keys and an optional
            # content_type key.
            if 'b64_content' in value:
                value['content'] = base64.decodebytes(force_bytes(value['b64_content']))
            if 'filename' in value and 'content' in value:
                content_type = value.get('content_type') or 'application/octet-stream'
                if content_type.startswith('text/'):
                    charset = 'utf-8'
                else:
                    charset = None
                upload = PicklableUpload(value['filename'], content_type, charset)
                upload.receive([force_bytes(value['content'])])
                return upload
        raise ValueError('invalid data for file type (%r)' % value)

    def get_view_short_value(self, value, max_len=30, **kwargs):
        return self.get_view_value(value, include_image_thumbnail=False, max_len=max_len, **kwargs)

    def get_download_query_string(self, **kwargs):
        if kwargs.get('parent_field'):
            return 'f=%s$%s$%s' % (kwargs['parent_field'].id, kwargs['parent_field_index'], self.id)
        return 'f=%s' % self.id

    def get_view_value(self, value, include_image_thumbnail=True, max_len=None, **kwargs):
        show_link = True
        if value.has_redirect_url():
            is_in_backoffice = bool(get_request() and get_request().is_in_backoffice())
            show_link = bool(value.get_redirect_url(backoffice=is_in_backoffice))
        t = TemplateIO(html=True)
        t += htmltext('<div class="file-field">')
        if show_link or include_image_thumbnail:
            download_qs = self.get_download_query_string(**kwargs)
        if show_link:
            attrs = {
                'href': '[download]?%s' % download_qs,
            }
            if max_len:
                attrs['title'] = value
            t += htmltag('a', **attrs)
        if include_image_thumbnail and value.can_thumbnail():
            t += htmltext('<img alt="" src="[download]?%s&thumbnail=1"/>') % download_qs
        filename = str(value)
        if max_len and len(filename) > max_len:
            basename, ext = os.path.splitext(filename)
            basename = ellipsize(basename, max_len - 5)
            filename = basename + ext
        t += htmltext('<span>%s</span>') % filename
        if show_link:
            t += htmltext('</a>')
        t += htmltext('</div>')
        return t.getvalue()

    def get_download_url(self, formdata, **kwargs):
        return '%sdownload?%s' % (formdata.get_url(), self.get_download_query_string(**kwargs))

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        show_link = True
        if value.has_redirect_url():
            is_in_backoffice = bool(get_request() and get_request().is_in_backoffice())
            show_link = bool(value.get_redirect_url(backoffice=is_in_backoffice))
        if show_link and formdata:
            node = ET.Element('{%s}a' % OD_NS['text'])
            node.attrib['{%s}href' % OD_NS['xlink']] = self.get_download_url(formdata, **kwargs)
        else:
            node = ET.Element('{%s}span' % OD_NS['text'])
        node.text = od_clean_text(force_text(value))
        return node

    def get_csv_value(self, value, **kwargs):
        return [str(value) if value else '']

    def get_json_value(self, value, formdata=None, include_file_content=True, **kwargs):
        out = value.get_json_value(include_file_content=include_file_content)
        if formdata:
            out['url'] = self.get_download_url(formdata, **kwargs)
        out['field_id'] = self.id
        return out

    def from_json_value(self, value):
        if value and 'filename' in value and 'content' in value:
            content = base64.b64decode(value['content'])
            content_type = value.get('content_type', 'application/octet-stream')
            if content_type.startswith('text/'):
                charset = 'utf-8'
            else:
                charset = None
            upload = PicklableUpload(value['filename'], content_type, charset)
            upload.receive([content])
            return upload
        return None

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        if not edit:
            value = get_request().get_field(self.field_key)
            if value and hasattr(value, 'token'):
                get_request().form[self.field_key + '$token'] = value.token

    def migrate(self):
        changed = super().migrate()
        if 'file_type' in self.__dict__:
            self.document_type = {}
            if self.__dict__['file_type']:
                file_type = self.__dict__['file_type']
                document_types = get_document_types(self.document_type)
                parts = []
                for key, value in document_types.items():
                    if file_type == value.get('mimetypes'):
                        self.document_type = value.copy()
                        self.document_type['id'] = key
                        break
                    if not value.get('mimetypes'):
                        continue
                    if ','.join(value['mimetypes']) in file_type:
                        parts.append(value['label'])
                else:
                    # self.file_type is a combination of file type, we create a
                    # virtual one from them
                    if parts and len(parts) > 1:
                        label = ', '.join(parts)
                    else:
                        label = ','.join(file_type)
                    self.document_type = {
                        'id': '_legacy',
                        'label': label,
                        'mimetypes': file_type,
                    }
            del self.__dict__['file_type']
            changed = True
        return changed

    def export_to_xml(self, charset, include_id=False):
        # convert some sub-fields to strings as export_to_xml() only supports
        # dictionnaries with strings values
        if self.document_type and self.document_type.get('mimetypes'):
            old_value = self.document_type['mimetypes']
            self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes'])
        result = super().export_to_xml(charset, include_id=include_id)
        if self.document_type and self.document_type.get('mimetypes'):
            self.document_type['mimetypes'] = old_value
        return result

    def init_with_xml(self, element, charset, include_id=False, snapshot=False):
        super().init_with_xml(element, charset, include_id=include_id)
        # translate fields flattened to strings
        if self.document_type and self.document_type.get('mimetypes'):
            self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|')
        if self.document_type and self.document_type.get('fargo'):
            self.document_type['fargo'] = self.document_type['fargo'] == 'True'


register_field_class(FileField)


class DateField(WidgetField):
    key = 'date'
    description = N_('Date')

    widget_class = DateWidget
    minimum_date = None
    maximum_date = None
    minimum_is_future = False
    date_in_the_past = False
    date_can_be_today = False
    extra_attributes = [
        'minimum_date',
        'minimum_is_future',
        'maximum_date',
        'date_in_the_past',
        'date_can_be_today',
    ]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(DateWidget, 'minimum_date', title=_('Minimum Date'), value=self.minimum_date)
        form.add(
            CheckboxWidget,
            'minimum_is_future',
            title=_('Date must be in the future'),
            value=self.minimum_is_future,
            hint=_('This option is obviously not compatible with setting a minimum date'),
        )
        form.add(DateWidget, 'maximum_date', title=_('Maximum Date'), value=self.maximum_date)
        form.add(
            CheckboxWidget,
            'date_in_the_past',
            title=_('Date must be in the past'),
            value=self.date_in_the_past,
            hint=_('This option is obviously not compatible with setting a maximum date'),
        )
        form.add(
            CheckboxWidget,
            'date_can_be_today',
            title=_('Date can be present day'),
            value=self.date_can_be_today,
            hint=_('This option is only useful combined with one of the previous checkboxes.'),
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'minimum_date',
            'minimum_is_future',
            'maximum_date',
            'date_in_the_past',
            'date_can_be_today',
        ]

    @classmethod
    def convert_value_from_anything(cls, value):
        if value is None or value == '':
            return None
        date_value = evalutils.make_date(value).timetuple()  # could raise ValueError
        return date_value

    def convert_value_from_str(self, value):
        if not value:
            return None
        try:
            return get_as_datetime(value).timetuple()
        except ValueError:
            return None

    def convert_value_to_str(self, value):
        if value is None:
            return ''
        try:
            return strftime(date_format(), value)
        except TypeError:
            return ''

    def add_to_form(self, form, value=None):
        if value and type(value) is not str:
            value = self.convert_value_to_str(value)
        return WidgetField.add_to_form(self, form, value=value)

    def add_to_view_form(self, form, value=None):
        value = strftime(misc.date_format(), value)
        return super().add_to_view_form(form, value=value)

    def get_view_value(self, value, **kwargs):
        try:
            return strftime(misc.date_format(), value)
        except TypeError:
            return value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def get_json_value(self, value, **kwargs):
        try:
            return strftime('%Y-%m-%d', value)
        except TypeError:
            return ''

    def from_json_value(self, value):
        try:
            return time.strptime(value, '%Y-%m-%d')
        except (TypeError, ValueError):
            return None


register_field_class(DateField)


def item_items_stats(field, values):
    if field.data_source:
        options = data_sources.get_items(field.data_source)
    else:
        options = field.items or []
    if len(options) == 0:
        return None
    no_records = len(values)
    if no_records == 0:
        return None
    r = TemplateIO(html=True)
    r += htmltext('<table class="stats">')
    r += htmltext('<thead><tr><th colspan="4">')
    r += field.label
    r += htmltext('</th></tr></thead>')
    r += htmltext('<tbody>')
    for option in options:
        if type(option) in (tuple, list):
            option_label = option[1]
            option_value = str(option[0])
        else:
            option_label = option
            option_value = option
        if field.type == 'item':
            no = len([None for x in values if x.data.get(field.id) == option_value])
        else:
            no = len([None for x in values if option_value in (x.data.get(field.id) or [])])
        r += htmltext('<tr>')
        r += htmltext('<td class="label">')
        r += option_label
        r += htmltext('</td>')

        r += htmltext('<td class="percent">')
        r += htmltext(' %.2f&nbsp;%%') % (100.0 * no / no_records)
        r += htmltext('</td>')
        r += htmltext('<td class="total">')
        r += '(%d/%d)' % (no, no_records)
        r += htmltext('</td>')
        r += htmltext('</tr>')
        r += htmltext('<tr>')
        r += htmltext('<td class="bar" colspan="3">')
        r += htmltext('<span style="width: %d%%"></span>' % (100 * no / no_records))
        r += htmltext('</td>')
        r += htmltext('</tr>')
    r += htmltext('</tbody>')
    r += htmltext('</table>')
    return r.getvalue()


class MapOptionsMixin:
    initial_zoom = None
    min_zoom = None
    max_zoom = None

    def fill_zoom_admin_form(self, form, **kwargs):
        # 0: whole world, 9: wide area, 11: area, 13: town, 16: small road
        zoom_levels = [
            (None, '---'),
            ('0', _('Whole world')),
            ('6', _('Country')),
            ('9', _('Wide area')),
            ('11', _('Area')),
            ('13', _('Town')),
            ('16', _('Small road')),
            ('18', _('Neighbourhood')),
            ('19', _('Ant')),
        ]
        form.add(
            SingleSelectWidget,
            'initial_zoom',
            title=_('Initial zoom level'),
            value=self.initial_zoom or '13',
            options=zoom_levels,
            **kwargs,
        )
        form.add(
            SingleSelectWidget,
            'min_zoom',
            title=_('Minimal zoom level'),
            value=self.min_zoom,
            options=zoom_levels,
            required=False,
            **kwargs,
        )
        form.add(
            SingleSelectWidget,
            'max_zoom',
            title=_('Maximal zoom level'),
            value=self.max_zoom,
            options=zoom_levels,
            required=False,
            **kwargs,
        )

    def check_zoom_admin_form(self, form):
        initial_zoom = form.get_widget('initial_zoom').parse()
        min_zoom = form.get_widget('min_zoom').parse()
        max_zoom = form.get_widget('max_zoom').parse()
        if min_zoom and max_zoom:
            if int(min_zoom) > int(max_zoom):
                form.get_widget('min_zoom').set_error(
                    _('Minimal zoom level cannot be greater than maximal zoom level.')
                )
        if (initial_zoom and min_zoom and int(initial_zoom) < int(min_zoom)) or (
            (initial_zoom and max_zoom and int(initial_zoom) > int(max_zoom))
        ):
            form.get_widget('initial_zoom').set_error(
                _('Initial zoom level must be between minimal and maximal zoom levels.')
            )


class ItemFieldMixin:
    def add_items_fields_admin_form(self, form):
        real_data_source = data_sources.get_real(self.data_source)
        form.add(
            RadiobuttonsWidget,
            'data_mode',
            title=_('Data'),
            options=[
                ('simple-list', _('Simple List'), 'simple-list'),
                ('data-source', _('Data Source'), 'data-source'),
            ],
            value='data-source' if real_data_source else 'simple-list',
            attrs={'data-dynamic-display-parent': 'true'},
            extra_css_class='widget-inline-radio no-bottom-margin',
        )
        form.add(
            WidgetList,
            'items',
            element_type=StringWidget,
            value=self.items,
            required=False,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
            attrs={'data-dynamic-display-child-of': 'data_mode', 'data-dynamic-display-value': 'simple-list'},
        )
        form.add(
            data_sources.DataSourceSelectionWidget,
            'data_source',
            value=self.data_source,
            required=False,
            hint=_('This will get the available items from an external source.'),
            attrs={'data-dynamic-display-child-of': 'data_mode', 'data-dynamic-display-value': 'data-source'},
        )

    def check_items_admin_form(self, form):
        data_mode = form.get_widget('data_mode').parse()
        if data_mode == 'simple-list':
            items = form.get_widget('items').parse()
            d = {}
            for v in items or []:
                if v in d:
                    form.set_error('items', _('Duplicated Items'))
                    return
                d[v] = None

            data_source_type = form.get_widget('data_source').get_widget('type')
            data_source_type.set_value(None)
            data_source_type.transfer_form_value(get_request())


class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin):
    key = 'item'
    description = N_('List')
    allow_complex = True

    items = []
    show_as_radio = None
    anonymise = False
    widget_class = SingleSelectHintWidget
    data_source = {}
    in_filters = False
    display_disabled_items = False
    display_mode = 'list'
    initial_date_alignment = None

    def __init__(self, **kwargs):
        self.items = []
        WidgetField.__init__(self, **kwargs)

    def migrate(self):
        changed = super().migrate()
        if isinstance(getattr(self, 'show_as_radio', None), bool):
            if self.show_as_radio:
                self.display_mode = 'radio'
            else:
                self.display_mode = 'list'
            self.show_as_radio = None
            changed = True
        return changed

    def init_with_xml(self, element, charset, include_id=False, snapshot=False):
        super().init_with_xml(element, charset, include_id=include_id)
        if getattr(element.find('show_as_radio'), 'text', None) == 'True':
            self.display_mode = 'radio'

    @property
    def extra_attributes(self):
        if self.display_mode == 'map':
            return ['initial_zoom', 'min_zoom', 'max_zoom', 'data_source']
        return []

    def get_options(self, mode=None):
        if self.data_source:
            return [x[:3] for x in data_sources.get_items(self.data_source, mode=mode)]
        if self.items:
            return [(x, x) for x in self.items]
        return []

    def get_extended_options(self):
        if self.data_source:
            return data_sources.get_structured_items(self.data_source, mode='lazy')
        if self.items:
            return [{'id': x, 'text': x} for x in self.items]
        return []

    def get_display_mode(self, data_source=None):
        if not data_source:
            data_source = data_sources.get_object(self.data_source)

        if data_source and data_source.type == 'jsonp':
            # a source defined as JSONP can only be used in autocomplete mode
            return 'autocomplete'

        return self.display_mode

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        data_source = data_sources.get_object(self.data_source)
        display_mode = self.get_display_mode(data_source)

        if display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
            self.url = kwargs['url'] = data_source.get_jsonp_url()
            self.widget_class = JsonpSingleSelectWidget
            return

        if self.display_mode != 'map':
            if self.data_source:
                items = data_sources.get_items(self.data_source, include_disabled=self.display_disabled_items)
                kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
                kwargs['options_with_attributes'] = items[:]
            else:
                kwargs['options'] = self.get_options()
            if not kwargs.get('options'):
                kwargs['options'] = [(None, '---', None)]
        if display_mode == 'radio':
            self.widget_class = RadiobuttonsWidget
            if type(kwargs['options'][0]) is str:
                first_items = [x for x in kwargs['options'][:3]]
            else:
                first_items = [x[1] for x in kwargs['options'][:3]]
            length_first_items = sum([len(x) for x in first_items])
            # display radio buttons on a single line if there's just a few
            # short options.
            # TODO: absence/presence of delimitor should be an option
            self.inline = bool(len(kwargs['options']) <= 3 and length_first_items <= 40)
        elif display_mode == 'autocomplete':
            kwargs['select2'] = True
        elif display_mode == 'map':
            self.widget_class = MapMarkerSelectionWidget
        elif display_mode == 'timetable':
            # SingleSelectHintWidget with custom template
            kwargs['template-name'] = 'qommon/forms/widgets/select-timetable.html'

    def get_display_value(self, value):
        data_source = data_sources.get_object(self.data_source)
        if data_source is None:
            return value or ''

        if data_source.type == 'jsonp':
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            return get_session().jsonp_display_values.get('%s_%s' % (data_source.get_jsonp_url(), value))

        display_value = data_source.get_display_value(value)
        if self.display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
            # store display value in session to be used by select2
            url = data_source.get_jsonp_url()
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            get_session().jsonp_display_values['%s_%s' % (url, value)] = display_value

        return display_value

    def get_view_value(self, value, value_id=None, **kwargs):
        value = super().get_view_value(value)
        if not (
            value_id
            and get_request()
            and get_request().is_in_backoffice()
            and self.data_source
            and self.data_source.get('type', '').startswith('carddef:')
        ):
            return value
        from wcs.carddef import CardDef

        try:
            carddef = CardDef.get_by_urlname(self.data_source['type'][8:])
            carddata = carddef.data_class().get(value_id)
        except KeyError:
            return value
        if not carddef.is_user_allowed_read(get_request().user, carddata):
            return value
        return htmltext('<a href="%s">' % carddata.get_url(backoffice=True)) + htmltext('%s</a>') % value

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(force_text(value))
        return span

    def add_to_view_form(self, form, value=None):
        real_value = value
        label_value = ''
        if value is not None:
            label_value = self.get_display_value(value)
        self.field_key = 'f%s' % self.id

        form.add(
            StringWidget,
            self.field_key + '_label',
            title=self.label,
            value=label_value,
            readonly='readonly',
            size=len(label_value or '') + 2,
            render_br=False,
        )
        label_widget = form.get_widget(self.field_key + '_label')
        # don't let subwidget overwrite label widget value
        label_widget.secondary = True
        get_request().form[label_widget.name] = label_value
        label_widget.field = self
        form.add(HiddenWidget, self.field_key, value=real_value)
        form.get_widget(self.field_key).field = self
        widget = form.get_widget(self.field_key + '_label')
        if self.extra_css_class:
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                widget.extra_css_class = '%s %s' % (widget.extra_css_class, self.extra_css_class)
            else:
                widget.extra_css_class = self.extra_css_class

    def store_display_value(self, data, field_id):
        value = data.get(field_id)
        if not value:
            return ''
        data_source = data_sources.get_object(self.data_source)
        if data_source and data_source.type == 'jsonp':
            if get_request():
                display_value = get_request().form.get('f%s_display' % field_id)
                real_data_source = data_source.data_source
                if display_value is None:
                    if not get_session().jsonp_display_values:
                        get_session().jsonp_display_values = {}
                    display_value = get_session().jsonp_display_values.get(
                        '%s_%s' % (real_data_source.get('value'), value)
                    )
                else:
                    if not get_session().jsonp_display_values:
                        get_session().jsonp_display_values = {}
                    get_session().jsonp_display_values[
                        '%s_%s' % (real_data_source.get('value'), value)
                    ] = display_value
                return display_value
        return self.get_display_value(value)

    def store_structured_value(self, data, field_id):
        data_source = data_sources.get_object(self.data_source)
        if data_source is None:
            return

        if data_source.type == 'jsonp':
            return

        value = data_source.get_structured_value(data.get(field_id))
        if value is None or set(value.keys()) == set(['id', 'text']):
            return
        return value

    def convert_value_from_anything(self, value):
        if value is None:
            return None
        return str(value)

    def convert_value_from_str(self, value):
        # caller should also call store_display_value and store_structured_value
        return value

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(
            CheckboxWidget,
            'in_filters',
            title=_('Display in default filters'),
            value=self.in_filters,
            advanced=True,
        )
        options = [
            ('list', _('List'), 'list'),
            ('radio', _('Radio buttons'), 'radio'),
            ('autocomplete', _('Autocomplete'), 'autocomplete'),
            ('map', _('Map (requires geographical data)'), 'map'),
            ('timetable', _('Timetable'), 'timetable'),
        ]
        form.add(
            RadiobuttonsWidget,
            'display_mode',
            title=_('Display Mode'),
            options=options,
            value=self.display_mode,
            attrs={'data-dynamic-display-parent': 'true'},
            extra_css_class='widget-inline-radio',
        )
        self.add_items_fields_admin_form(form)
        form.add(
            CheckboxWidget,
            'display_disabled_items',
            title=_('Display disabled items'),
            value=self.display_disabled_items,
            advanced=not (self.display_disabled_items),
        )
        form.add(
            StringWidget,
            'initial_date_alignment',
            title=_('Initial date alignment'),
            value=self.initial_date_alignment,
            validation_function=ComputedExpressionWidget.validate_template,
            attrs={
                'data-dynamic-display-child-of': 'display_mode',
                'data-dynamic-display-value': 'timetable',
            },
        )
        self.fill_zoom_admin_form(
            form, attrs={'data-dynamic-display-child-of': 'display_mode', 'data-dynamic-display-value': 'map'}
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'items',
            'display_mode',
            'data_source',
            'in_filters',
            'anonymise',
            'display_disabled_items',
            'initial_zoom',
            'min_zoom',
            'max_zoom',
            'initial_date_alignment',
        ]

    def check_admin_form(self, form):
        self.check_items_admin_form(form)
        self.check_zoom_admin_form(form)

    def stats(self, values):
        return item_items_stats(self, values)

    def get_initial_date_alignment(self):
        if not self.initial_date_alignment:
            return
        import wcs.workflows

        try:
            date = wcs.workflows.template_on_formdata(None, self.initial_date_alignment, autoescape=False)
        except TemplateError:
            return
        try:
            return misc.get_as_datetime(date)
        except ValueError:
            return

    def feed_session(self, value, display_value):
        real_data_source = data_sources.get_real(self.data_source)
        if real_data_source and real_data_source.get('type') == 'jsonp':
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            get_session().jsonp_display_values[
                '%s_%s' % (real_data_source.get('value'), value)
            ] = display_value

    def get_csv_heading(self):
        if self.data_source:
            return ['%s (%s)' % (self.label, _('identifier')), self.label]
        return [self.label]

    def get_csv_value(self, value, display_value=None, **kwargs):
        values = [value]
        if self.data_source:
            values.append(display_value)
        return values

    def export_to_json(self, include_id=False, anonymise=True):
        field = super().export_to_json(include_id=include_id, anonymise=anonymise)
        if self.data_source and not anonymise:
            structured_items = data_sources.get_structured_items(self.data_source)
            if structured_items:
                field['structured_items'] = structured_items
                if not field.get('items'):
                    field['items'] = [item['text'] for item in structured_items]
        return field


register_field_class(ItemField)


class ItemsField(WidgetField, ItemFieldMixin):
    key = 'items'
    description = N_('Multiple choice list')
    allow_complex = True

    items = []
    min_choices = 0
    max_choices = 0
    data_source = {}
    in_filters = False
    display_disabled_items = False

    widget_class = CheckboxesWidget

    _cached_data_source = None

    def __init__(self, **kwargs):
        self.items = []
        WidgetField.__init__(self, **kwargs)

    def get_options(self):
        if self.data_source:
            if self._cached_data_source:
                return self._cached_data_source
            self._cached_data_source = [x[:3] for x in data_sources.get_items(self.data_source)]
            return self._cached_data_source[:]
        elif self.items:
            return [(x, x) for x in self.items]
        else:
            return []

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['options'] = self.get_options()
        kwargs['min_choices'] = self.min_choices
        kwargs['max_choices'] = self.max_choices
        if self.data_source:
            items = data_sources.get_items(self.data_source, include_disabled=self.display_disabled_items)
            kwargs['options'] = [x[:3] for x in items if not x[-1].get('disabled')]
            kwargs['options_with_attributes'] = items[:]

        if len(kwargs['options']) > 3:
            kwargs['inline'] = False

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        form.add(
            CheckboxWidget,
            'in_filters',
            title=_('Display in default filters'),
            value=self.in_filters,
            advanced=True,
        )
        self.add_items_fields_admin_form(form)
        form.add(
            IntWidget,
            'min_choices',
            title=_('Minimum number of choices'),
            value=self.min_choices,
            required=False,
            size=4,
        )
        form.add(
            IntWidget,
            'max_choices',
            title=_('Maximum number of choices'),
            value=self.max_choices,
            required=False,
            size=4,
        )
        form.add(
            CheckboxWidget,
            'display_disabled_items',
            title=_('Display disabled items'),
            value=self.display_disabled_items,
            advanced=not (self.display_disabled_items),
        )

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'items',
            'min_choices',
            'max_choices',
            'data_source',
            'in_filters',
            'anonymise',
            'display_disabled_items',
        ]

    def check_admin_form(self, form):
        self.check_items_admin_form(form)

    def get_prefill_value(self, user=None, force_string=True):
        return super().get_prefill_value(user=user, force_string=False)

    def convert_value_to_str(self, value):
        return value

    def convert_value_from_str(self, value):
        if not isinstance(value, str):
            return value
        if not value.strip():
            return None
        return [x.strip() for x in value.split('|') if x.strip()]

    def convert_value_from_anything(self, value):
        if isinstance(value, str):
            return self.convert_value_from_str(value)
        if isinstance(value, int):
            return [value]
        if not value:
            return None
        try:
            return list(value)
        except TypeError:
            raise ValueError('invalid data for items type (%r)' % value)

    def get_value_info(self, data):
        value, value_details = super().get_value_info(data)
        labels = []
        if not self.data_source:
            value_id = value_details.get('value_id')
            if value_id:
                labels = value_id.copy()
        else:
            structured_values = self.get_structured_value(data)
            if structured_values:
                labels = [x['text'] for x in structured_values]
        value_details['labels'] = labels
        return (value, value_details)

    def get_view_value(self, value, **kwargs):
        if kwargs.get('labels'):
            # summary page and labels are available
            r = TemplateIO(html=True)
            r += htmltext('<ul>')
            for x in kwargs['labels']:
                r += htmltext('<li>%s</li>' % x)
            r += htmltext('</ul>')
            return r.getvalue()

        if type(value) is str:  # == display_value
            return value
        if value:
            try:
                return ', '.join([(x) for x in value])
            except TypeError:
                pass
        return ''

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        span = ET.Element('{%s}span' % OD_NS['text'])
        span.text = od_clean_text(self.get_view_value(value))
        return span

    def stats(self, values):
        return item_items_stats(self, values)

    def get_csv_heading(self):
        labels = [self.label]
        if self.max_choices:
            labels.extend([''] * (self.max_choices - 1))
        elif len(self.get_options()):
            labels.extend([''] * (len(self.get_options()) - 1))
        return labels

    def get_csv_value(self, value, **kwargs):
        values = []
        for one_value in value:
            values.append(one_value)
        if self.max_choices:
            nb_columns = self.max_choices
        elif len(self.get_options()):
            nb_columns = len(self.get_options())
        else:
            nb_columns = 1

        if len(values) > nb_columns:
            # this would happen if max_choices is set after forms were already
            # filled with more values
            values = values[:nb_columns]
        elif len(values) < nb_columns:
            values.extend([''] * (nb_columns - len(values)))
        return values

    def store_display_value(self, data, field_id):
        options = self.get_options()
        if not options:
            return ''
        choices = []
        for choice in data.get(field_id) or []:
            if type(options[0]) is str:
                choices.append(choice)
            elif type(options[0]) in (tuple, list):
                if len(options[0]) == 2:
                    for key, option_value in options:
                        if str(key) == str(choice):
                            choices.append(option_value)
                            break
                elif len(options[0]) == 3:
                    for key, option_value, dummy in options:
                        if str(key) == str(choice):
                            choices.append(option_value)
                            break
        return ', '.join(choices)

    def store_structured_value(self, data, field_id):
        if not self.data_source:
            return
        structured_options = data_sources.get_structured_items(self.data_source)
        if not structured_options:
            return
        structured_value = []
        for structured_option in structured_options:
            for choice in data.get(field_id) or []:
                if str(structured_option.get('id')) == str(choice):
                    structured_value.append(structured_option)
        return structured_value

    def export_to_json(self, include_id=False, anonymise=True):
        field = super().export_to_json(include_id=include_id, anonymise=True)
        if self.data_source and not anonymise:
            structured_items = data_sources.get_structured_items(self.data_source)
            if structured_items:
                field['structured_items'] = structured_items
                if not field.get('items'):
                    field['items'] = [item['text'] for item in structured_items]
        return field


register_field_class(ItemsField)


class PostConditionsRowWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        if not value:
            value = {}
        self.add(
            ConditionWidget, name='condition', title=_('Condition'), value=value.get('condition'), size=50
        )
        self.add(
            StringWidget,
            name='error_message',
            title=_('Error message if condition is not met'),
            value=value.get('error_message'),
            size=50,
        )

    def _parse(self, request):
        if self.get('condition') or self.get('error_message'):
            self.value = {'condition': self.get('condition'), 'error_message': self.get('error_message')}
        else:
            self.value = None


class PostConditionsTableWidget(WidgetListAsTable):
    readonly = False

    def __init__(self, name, **kwargs):
        super().__init__(name, element_type=PostConditionsRowWidget, **kwargs)

    def parse(self, request=None):
        super().parse(request=request)
        for post_condition in self.value or []:
            if not (post_condition.get('error_message') and post_condition.get('condition')):
                self.set_error(_('Both condition and error message are required.'))
                break
        return self.value


class PageCondition(Condition):
    log_errors = True
    record_errors = False

    def get_data(self):
        dict_vars = self.context['dict_vars']
        formdef = self.context['formdef']

        # create variables with values currently being evaluated, not yet
        # available in the formdata.
        from .formdata import get_dict_with_varnames

        live_data = {}
        form_live_data = {}
        if dict_vars is not None:
            live_data = get_dict_with_varnames(formdef.fields, dict_vars)
            form_live_data = dict(('form_' + x, y) for x, y in live_data.items())

        # 1) feed the form_var_* variables in the global substitution system,
        # they will shadow formdata context variables with their new "live"
        # value, this may be useful when evaluating data sources.
        class ConditionVars:
            def __init__(self, id_dict_var):
                # keep track of reference dictionary
                self.id_dict_var = id_dict_var

            def get_substitution_variables(self):
                return {}

            def get_static_substitution_variables(self):
                # only for backward compatibility with python evaluations
                return form_live_data

            def __eq__(self, other):
                # Assume all ConditionVars are equal when initialized with
                # the same live data dictionary; this avoids filling
                # the substitution sources with duplicates and invalidating its
                # cache.
                return self.id_dict_var == getattr(other, 'id_dict_var', None)

        if dict_vars is not None:
            # Add them only if there is a real dict_vars in context,
            # ie do nothing on first page condition
            get_publisher().substitutions.feed(ConditionVars(id(dict_vars)))

            # alter top-of-stack formdata with data from submitted form
            from wcs.formdata import FormData

            for source in reversed(get_publisher().substitutions.sources):
                if isinstance(source, FormData):
                    source.data.update(dict_vars)
                    break

        data = super().get_data()
        # 2) add live data as var_ variables for local evaluation only, for
        # backward compatibility. They are not added globally as they would
        # interfere with the var_ prefixed variables used in dynamic jsonp
        # fields. (#9786)
        data = copy.copy(data)
        data.update(live_data)
        if dict_vars is None:
            # ConditionsVars is not set when evaluating first page condition,
            # but we need to have form_var_* variables already; add them from
            # form_live_data (where all variables will have been set to None).
            data.update(form_live_data)
        return data


class PageField(Field):
    key = 'page'
    description = N_('Page')

    post_conditions = None

    def post_conditions_init_with_xml(self, node, charset, include_id=False, snapshot=False):
        if node is None:
            return
        self.post_conditions = []
        for post_condition_node in node.findall('post_condition'):
            if post_condition_node.findall('condition/type'):
                condition = {
                    'type': xml_node_text(post_condition_node.find('condition/type')),
                    'value': xml_node_text(post_condition_node.find('condition/value')),
                }
            elif post_condition_node.find('condition').text:
                condition = {
                    'type': 'python',
                    'value': xml_node_text(post_condition_node.find('condition')),
                }
            else:
                continue
            self.post_conditions.append(
                {
                    'condition': condition,
                    'error_message': xml_node_text(post_condition_node.find('error_message')),
                }
            )

    def post_conditions_export_to_xml(self, node, charset, include_id=False):
        if not self.post_conditions:
            return

        conditions_node = ET.SubElement(node, 'post_conditions')
        for post_condition in self.post_conditions:
            post_condition_node = ET.SubElement(conditions_node, 'post_condition')
            condition_node = ET.SubElement(post_condition_node, 'condition')
            ET.SubElement(condition_node, 'type').text = force_text(
                post_condition['condition'].get('type') or '', charset, errors='replace'
            )
            ET.SubElement(condition_node, 'value').text = force_text(
                post_condition['condition'].get('value') or '', charset, errors='replace'
            )
            ET.SubElement(post_condition_node, 'error_message').text = force_text(
                post_condition['error_message'] or '', charset, errors='replace'
            )

    def fill_admin_form(self, form):
        form.add(StringWidget, 'label', title=_('Label'), value=self.label, required=True, size=50)
        form.add(
            ConditionWidget,
            'condition',
            title=_('Display Condition'),
            value=self.condition,
            required=False,
            size=50,
        )
        form.add(
            PostConditionsTableWidget,
            'post_conditions',
            title=_('Post Conditions'),
            value=self.post_conditions,
            advanced=not (self.post_conditions),
        )

    def get_admin_attributes(self):
        return Field.get_admin_attributes(self) + ['post_conditions']

    def migrate(self):
        changed = super().migrate()
        if isinstance(self.condition, str):
            if self.condition:
                self.condition = {'type': 'python', 'value': self.condition}
            else:
                self.condition = {}
            changed = True
        for post_condition in self.post_conditions or []:
            condition = post_condition.get('condition')
            if isinstance(condition, str):
                if condition:
                    post_condition['condition'] = {'type': 'python', 'value': condition}
                else:
                    post_condition['condition'] = {}
                changed = True
        return changed

    def add_to_view_form(self, *args, **kwargs):
        pass


register_field_class(PageField)


class TableField(WidgetField):
    key = 'table'
    description = N_('Table')
    allow_complex = True

    rows = None
    columns = None

    widget_class = TableWidget

    def __init__(self, **kwargs):
        self.rows = []
        self.columns = []
        WidgetField.__init__(self, **kwargs)

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['rows'] = self.rows
        kwargs['columns'] = self.columns

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'rows',
            title=_('Rows'),
            element_type=StringWidget,
            value=self.rows,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add row'),
        )
        form.add(
            WidgetList,
            'columns',
            title=_('Columns'),
            element_type=StringWidget,
            value=self.columns,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add column'),
        )

    def get_admin_attributes(self):
        t = WidgetField.get_admin_attributes(self) + ['rows', 'columns']
        try:
            t.remove('prefill')
        except ValueError:
            pass
        return t

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr><td></td>')
        for column in self.columns:
            r += htmltext('<th>%s</th>') % column
        r += htmltext('</tr></thead><tbody>')
        for i, row in enumerate(self.rows):
            r += htmltext('<tr><th>%s</th>') % row
            for j, column in enumerate(self.columns):
                r += htmltext('<td>')
                if value:
                    try:
                        r += value[i][j]
                    except IndexError:
                        pass
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody></table>')

        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        if not value:
            return indent
        r = []
        max_width = 0
        for column in self.columns:
            max_width = max(max_width, len(smart_text(column)))

        for i, row in enumerate(value):
            value[i] = [x or '' for x in row]

        def get_value(i, j):
            try:
                return smart_text(value[i][j])
            except IndexError:
                return '-'

        for i, row in enumerate(self.rows):
            max_width = max(max_width, len(row))
            for j, column in enumerate(self.columns):
                max_width = max(max_width, len(get_value(i, j)))

        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        r.append(' '.join([smart_text(column).center(max_width) for column in ['/'] + self.columns]))
        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        for i, row in enumerate(self.rows):
            r.append(
                ' '.join(
                    [
                        cell.center(max_width)
                        for cell in [smart_text(row)] + [get_value(i, x) for x in range(len(self.columns))]
                    ]
                )
            )
        r.append(' '.join(['=' * max_width] * (len(self.columns) + 1)))
        return misc.site_encode('\n'.join([indent + x for x in r]))

    def get_csv_heading(self):
        if not self.columns:
            return [self.label]
        labels = []
        for col in self.columns:
            for row in self.rows:
                t = '%s / %s' % (col, row)
                if len(labels) == 0:
                    labels.append('%s - %s' % (self.label, t))
                else:
                    labels.append(t)
        return labels

    def get_csv_value(self, value, **kwargs):
        if not self.columns:
            return ['']
        values = []
        for i in range(len(self.columns)):
            for j in range(len(self.rows)):
                try:
                    values.append(value[j][i])
                except IndexError:
                    values.append('')
        return values

    def get_opendocument_node_value(self, value, formdata=None, **kwargs):
        table = ET.Element('{%s}table' % OD_NS['table'])
        ET.SubElement(table, '{%s}table-column' % OD_NS['table'])
        for col in self.columns:
            ET.SubElement(table, '{%s}table-column' % OD_NS['table'])
        row = ET.SubElement(table, '{%s}table-row' % OD_NS['table'])
        ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
        for col in self.columns:
            table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
            cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
            cell_value.text = col
        for i, row_label in enumerate(self.rows):
            row = ET.SubElement(table, '{%s}table-row' % OD_NS['table'])
            table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
            cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
            cell_value.text = row_label
            for j, col in enumerate(self.columns):
                table_cell = ET.SubElement(row, '{%s}table-cell' % OD_NS['table'])
                cell_value = ET.SubElement(table_cell, '{%s}p' % OD_NS['text'])
                try:
                    cell_value.text = value[i][j]
                except IndexError:
                    pass
        return table


register_field_class(TableField)


class TableSelectField(TableField):
    key = 'table-select'
    description = N_('Table of Lists')
    allow_complex = True

    items = None

    widget_class = SingleSelectTableWidget

    def __init__(self, **kwargs):
        self.items = []
        TableField.__init__(self, **kwargs)

    def fill_admin_form(self, form):
        TableField.fill_admin_form(self, form)
        form.add(
            WidgetList,
            'items',
            title=_('Items'),
            element_type=StringWidget,
            value=self.items,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
        )

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        TableField.perform_more_widget_changes(self, form, kwargs, edit=edit)
        if edit:
            kwargs['options'] = self.items or [(None, '---')]
        else:
            self.widget_class = TableWidget

    def get_admin_attributes(self):
        return TableField.get_admin_attributes(self) + ['items']

    def check_admin_form(self, form):
        items = form.get_widget('items').parse()
        d = {}
        for v in items or []:
            if v in d:
                form.set_error('items', _('Duplicated Items'))
                return
            d[v] = None


register_field_class(TableSelectField)


class TableRowsField(WidgetField):
    key = 'tablerows'
    description = N_('Table with rows')
    allow_complex = True

    total_row = True
    columns = None

    widget_class = TableListRowsWidget

    def __init__(self, **kwargs):
        self.columns = []
        WidgetField.__init__(self, **kwargs)

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['columns'] = self.columns
        kwargs['add_element_label'] = _('Add row')

    def get_display_locations_options(self):
        return [('validation', _('Validation Page')), ('summary', _('Summary Page'))]

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'columns',
            title=_('Columns'),
            element_type=StringWidget,
            value=self.columns,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add column'),
        )
        form.add(CheckboxWidget, 'total_row', title=_('Total Row'), value=self.total_row)

    def get_admin_attributes(self):
        t = WidgetField.get_admin_attributes(self) + ['columns', 'total_row']
        try:
            t.remove('prefill')
        except ValueError:
            pass
        return t

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr>')
        for column in self.columns:
            r += htmltext('<th>%s</th>') % column
        r += htmltext('</tr></thead><tbody>')
        for row in value:
            r += htmltext('<tr>')
            for j, column in enumerate(self.columns):
                r += htmltext('<td>')
                if value:
                    try:
                        r += row[j]
                    except IndexError:
                        pass
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody>')

        if self.total_row:
            sums_row = []
            for j, column in enumerate(self.columns):
                sum_column = 0
                for row_value in value:
                    try:
                        cell_value = row_value[j]
                    except IndexError:
                        continue
                    if cell_value in (None, ''):
                        continue
                    try:
                        sum_column += float(cell_value)
                    except ValueError:
                        sums_row.append(None)
                        break
                else:
                    sums_row.append(sum_column)
            if [x for x in sums_row if x is not None]:
                r += htmltext('<tfoot><tr>')
                for sum_column in sums_row:
                    if sum_column is None:
                        r += htmltext('<td></td>')
                    else:
                        r += htmltext('<td>%.2f</td>' % sum_column)
                r += htmltext('</tr></tfoot>')

        r += htmltext('</table>')

        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        if not value:
            return indent
        r = []
        max_width = 0
        for column in self.columns:
            max_width = max(max_width, len(smart_text(column)))

        for i, row in enumerate(value):
            value[i] = [x or '' for x in row]

        def get_value(i, j):
            try:
                return smart_text(value[i][j])
            except IndexError:
                return '-'

        for i, row_value in enumerate(value):
            for j, column in enumerate(self.columns):
                try:
                    max_width = max(max_width, len(smart_text(row_value[j])))
                except IndexError:
                    # ignore errors for shorter than expected rows, this is
                    # typical of the field gaining new columns after some forms
                    # were already saved.
                    pass

        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        r.append(' '.join([smart_text(column).center(max_width) for column in self.columns]))
        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        for i, row_value in enumerate(value):
            r.append(
                ' '.join(
                    [cell.center(max_width) for cell in [get_value(i, x) for x in range(len(self.columns))]]
                )
            )
        r.append(' '.join(['=' * max_width] * (len(self.columns))))
        return misc.site_encode('\n'.join([indent + x for x in r]))

    def get_csv_value(self, element, **kwargs):
        return [_('unimplemented')]  # XXX


register_field_class(TableRowsField)


class MapField(WidgetField, MapOptionsMixin):
    key = 'map'
    description = N_('Map')

    default_position = None
    init_with_geoloc = False

    widget_class = MapWidget
    extra_attributes = ['initial_zoom', 'min_zoom', 'max_zoom', 'default_position', 'init_with_geoloc']

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        self.fill_zoom_admin_form(form)
        form.add(
            MapWidget,
            'default_position',
            title=_('Initial Position'),
            value=self.default_position,
            default_zoom='9',
            required=False,
        )
        form.add(
            CheckboxWidget,
            'init_with_geoloc',
            title=_('Initialize position using device geolocation'),
            value=self.init_with_geoloc,
            required=False,
        )

    def check_admin_form(self, form):
        self.check_zoom_admin_form(form)

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + [
            'initial_zoom',
            'min_zoom',
            'max_zoom',
            'default_position',
            'init_with_geoloc',
        ]

    def get_prefill_value(self, user=None, force_string=True):
        if self.prefill.get('type') != 'string' or not self.prefill.get('value'):
            return (None, False)
        # template string must produce lat;lon to be interpreted as coordinates,
        # otherwise it will be interpreted as an address that will be geocoded.
        prefill_value, explicit_lock = super().get_prefill_value()
        if re.match(r'-?\d+(\.\d+)?;-?\d+(\.\d+)?$', prefill_value):
            return (prefill_value, explicit_lock)

        from wcs.wf.geolocate import GeolocateWorkflowStatusItem

        geolocate = GeolocateWorkflowStatusItem()
        geolocate.method = 'address_string'
        geolocate.address_string = prefill_value
        coords = geolocate.geolocate_address_string(None, compute_template=False)
        if not coords:
            return (None, False)
        return ('%(lat)s;%(lon)s' % coords, False)

    def get_view_value(self, value, **kwargs):
        widget = self.widget_class('x%s' % random.random(), value, readonly=True)
        return widget.render_widget_content()

    def get_rst_view_value(self, value, indent=''):
        return indent + value

    def convert_value_from_str(self, value):
        try:
            dummy, dummy = [float(x) for x in value.split(';')]
        except (AttributeError, ValueError):
            return None
        return value

    def get_json_value(self, value, **kwargs):
        if not value or ';' not in value:
            return None
        lat, lon = value.split(';')
        try:
            lat = float(lat)
            lon = float(lon)
        except ValueError:
            return None
        return {'lat': lat, 'lon': lon}

    def from_json_value(self, value):
        if 'lat' in value and 'lon' in value:
            return '%s;%s' % (float(value['lat']), float(value['lon']))
        else:
            return None

    def get_structured_value(self, data):
        return self.get_json_value(data.get(self.id))


register_field_class(MapField)


class RankedItemsField(WidgetField):
    key = 'ranked-items'
    description = N_('Ranked Items')
    allow_complex = True

    items = []
    randomize_items = False
    widget_class = RankedItemsWidget
    anonymise = False

    def perform_more_widget_changes(self, form, kwargs, edit=True):
        kwargs['elements'] = self.items or []
        kwargs['randomize_items'] = self.randomize_items

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        try:
            form.remove('prefill')
        except KeyError:  # perhaps it was already removed
            pass
        form.add(
            WidgetList,
            'items',
            title=_('Items'),
            element_type=StringWidget,
            value=self.items,
            required=True,
            element_kwargs={'render_br': False, 'size': 50},
            add_element_label=_('Add item'),
        )
        form.add(CheckboxWidget, 'randomize_items', title=_('Randomize Items'), value=self.randomize_items)

    def get_admin_attributes(self):
        attrs = WidgetField.get_admin_attributes(self) + ['items', 'randomize_items']
        if 'prefill' in attrs:
            attrs.remove('prefill')
        return attrs

    def get_view_value(self, value, **kwargs):
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        items = list(value.items())
        items.sort(key=lambda x: x[1] or sys.maxsize)
        counter = 0
        last_it = None
        for it in items:
            if it[1] is not None:
                if last_it != it[1]:
                    counter += 1
                    last_it = it[1]
                r += htmltext('<li>%s: %s</li>') % (counter, it[0])
        r += htmltext('</ul>')
        return r.getvalue()

    def get_rst_view_value(self, value, indent=''):
        items = list(value.items())
        items.sort(key=lambda x: x[1] or sys.maxsize)
        counter = 0
        last_it = None
        values = []
        for it in items:
            if it[1] is not None:
                if last_it != it[1]:
                    counter += 1
                    last_it = it[1]
                values.append('%s: %s' % (counter, it[0]))
        return indent + ' / '.join(values)

    def get_csv_heading(self):
        if not self.items:
            return [self.label]
        return [self.label] + [''] * (len(self.items) - 1)

    def get_csv_value(self, value, **kwargs):
        if not self.items:
            return ['']
        if type(value) is not dict:
            value = {}
        items = [x for x in value.items() if x[1] is not None]
        items.sort(key=lambda x: x[1])
        ranked = [x[0] for x in items]
        return ranked + ['' for x in range(len(self.items) - len(ranked))]


register_field_class(RankedItemsField)


class PasswordField(WidgetField):
    key = 'password'
    description = N_('Password')

    min_length = 0
    max_length = 0
    count_uppercase = 0
    count_lowercase = 0
    count_digit = 0
    count_special = 0
    confirmation = True
    confirmation_title = None
    strength_indicator = True
    formats = ['sha1']
    extra_attributes = [
        'formats',
        'min_length',
        'max_length',
        'count_uppercase',
        'count_lowercase',
        'count_digit',
        'count_special',
        'confirmation',
        'confirmation_title',
        'strength_indicator',
    ]

    widget_class = PasswordEntryWidget

    def get_admin_attributes(self):
        return WidgetField.get_admin_attributes(self) + self.extra_attributes

    def fill_admin_form(self, form):
        WidgetField.fill_admin_form(self, form)
        formats = [
            ('cleartext', _('Clear text')),
            ('md5', _('MD5')),
            ('sha1', _('SHA1')),
        ]
        form.add(
            CheckboxesWidget,
            'formats',
            title=_('Storage formats'),
            value=self.formats,
            options=formats,
            inline=True,
        )
        form.add(IntWidget, 'min_length', title=_('Minimum length'), value=self.min_length)
        form.add(
            IntWidget,
            'max_length',
            title=_('Maximum password length'),
            value=self.max_length,
            hint=_('0 for unlimited length'),
        )
        form.add(
            IntWidget,
            'count_uppercase',
            title=_('Minimum number of uppercase characters'),
            value=self.count_uppercase,
        )
        form.add(
            IntWidget,
            'count_lowercase',
            title=_('Minimum number of lowercase characters'),
            value=self.count_lowercase,
        )
        form.add(IntWidget, 'count_digit', title=_('Minimum number of digits'), value=self.count_digit)
        form.add(
            IntWidget,
            'count_special',
            title=_('Minimum number of special characters'),
            value=self.count_special,
        )
        form.add(
            CheckboxWidget,
            'strength_indicator',
            title=_('Add a password strength indicator'),
            value=self.strength_indicator,
        )
        form.add(CheckboxWidget, 'confirmation', title=_('Add a confirmation input'), value=self.confirmation)
        form.add(
            StringWidget,
            'confirmation_title',
            size=50,
            title=_('Label for confirmation input'),
            value=self.confirmation_title,
        )

    def get_view_value(self, value, **kwargs):
        return '●' * 8

    def get_csv_value(self, value, **kwargs):
        return [self.get_view_value(value)]

    def get_rst_view_value(self, value, indent=''):
        return indent + self.get_view_value(value)


register_field_class(PasswordField)


class BlockField(WidgetField):
    key = 'block'
    allow_complex = True

    widget_class = BlockWidget
    max_items = 1
    extra_attributes = ['block', 'max_items', 'add_element_label', 'label_display', 'remove_button']
    add_element_label = ''
    label_display = 'normal'
    remove_button = False

    # cache
    _block = None

    @property
    def block(self):
        if self._block:
            return self._block
        self._block = BlockDef.get_on_index(self.type[6:], 'slug')
        return self._block

    def get_type_label(self):
        return _('Field Block (%s)') % self.block.name

    def fill_admin_form(self, form):
        super().fill_admin_form(form)
        if form.get_widget('prefill'):
            form.remove('prefill')
        form.add(IntWidget, 'max_items', title=_('Maximum number of items'), value=self.max_items)
        form.add(
            StringWidget, 'add_element_label', title=_('Label of "Add" button'), value=self.add_element_label
        )
        display_options = [
            ('normal', _('Normal')),
            ('subtitle', _('Subtitle')),
            ('hidden', _('Hidden')),
        ]
        form.add(
            SingleSelectWidget,
            'label_display',
            title=_('Label display'),
            value=self.label_display or 'normal',
            options=display_options,
        )
        form.add(CheckboxWidget, 'remove_button', title=_('Include remove button'), value=self.remove_button)

    def get_admin_attributes(self):
        return super().get_admin_attributes() + [
            'max_items',
            'add_element_label',
            'label_display',
            'remove_button',
        ]

    def store_display_value(self, data, field_id):
        value = data.get(field_id)
        parts = []
        if value and value.get('data'):
            for subvalue in value.get('data'):
                parts.append(self.block.get_display_value(subvalue))
        return ', '.join(parts)

    def get_view_value(self, value, summary=False, **kwargs):
        from wcs.workflows import template_on_formdata

        if 'value_id' not in kwargs:
            # when called from get_rst_view_value()
            return str(value or '')
        value = kwargs['value_id']
        r = TemplateIO(html=True)
        for i, row_value in enumerate(value['data']):
            for field in self.block.fields:
                if summary and not field.include_in_summary_page:
                    continue
                if not hasattr(field, 'get_value_info'):
                    # inert field
                    if field.include_in_summary_page:
                        if field.type == 'title':
                            label = template_on_formdata(None, field.label, autoescape=False)
                            r += htmltext('<div class="title %s"><h3>%s</h3></div>') % (
                                field.extra_css_class or '',
                                label,
                            )
                        elif field.type == 'subtitle':
                            label = template_on_formdata(None, field.label, autoescape=False)
                            r += htmltext('<div class="subtitle %s"><h4>%s</h4></div>') % (
                                field.extra_css_class or '',
                                label,
                            )
                        elif field.type == 'comment':
                            r += htmltext(
                                '<div class="comment-field %s">%s</div>'
                                % (field.extra_css_class or '', field.get_text())
                            )
                    continue
                css_classes = ['field', 'field-type-%s' % field.key]
                if field.extra_css_class:
                    css_classes.append(field.extra_css_class)
                r += htmltext('<div class="%s">' % ' '.join(css_classes))
                r += htmltext('<span class="label">%s</span> ') % field.label
                sub_value, sub_value_details = field.get_value_info(row_value)
                if sub_value is None:
                    r += htmltext('<div class="value"><i>%s</i></div>') % _('Not set')
                else:
                    r += htmltext('<div class="value">')
                    kwargs = {'parent_field': self, 'parent_field_index': i}
                    kwargs.update(**sub_value_details)
                    r += field.get_view_value(sub_value, **kwargs)
                    r += htmltext('</div>')
                r += htmltext('</div>\n')
        return r.getvalue()

    def get_csv_heading(self):
        nb_items = self.max_items or 1
        if nb_items == 1:
            return [self.label]
        headings = ['%s - %s' % (self.label, x + 1) for x in range(nb_items)]
        return headings

    def get_csv_value(self, value, **kwargs):
        nb_items = self.max_items or 1
        cells = [''] * nb_items
        if value and value.get('data'):
            for i, subvalue in enumerate(value.get('data')[:nb_items]):
                if subvalue:
                    cells[i] = self.block.get_display_value(subvalue)
        return cells

    def get_json_value(self, value, **kwargs):
        from wcs.formdata import FormData

        result = []
        if not value or not value.get('data'):
            return result
        for subvalue_data in value.get('data'):
            result.append(
                FormData.get_json_data_dict(
                    subvalue_data, self.block.fields, include_files=kwargs.get('include_file_content')
                )
            )
        return result


def get_field_class_by_type(type):
    for k in field_classes:
        if k.key == type:
            return k
    if type.startswith('block:'):
        # make sure block type exists (raises KeyError on missing data)
        BlockDef.get_on_index(type[6:], 'slug')
        return BlockField
    raise KeyError()


def get_field_types():
    return field_types


def get_field_options(blacklisted_types):
    widgets, non_widgets = [], []
    for klass in field_classes:
        if klass.key in blacklisted_types:
            continue
        if issubclass(klass, WidgetField):
            widgets.append((klass.key, _(klass.description), klass.key))
        else:
            non_widgets.append((klass.key, _(klass.description), klass.key))
    options = widgets + [('', '—', '')] + non_widgets
    if get_publisher().has_site_option('fields-blocks') and (
        not blacklisted_types or 'blocks' not in blacklisted_types
    ):
        position = len(options)
        for blockdef in BlockDef.select(order_by='name'):
            options.append(('block:%s' % blockdef.slug, blockdef.name, 'block:%s' % blockdef.slug))
        if len(options) != position:
            # add separator
            options.insert(position, ('', '—', ''))
    return options