wcs/wcs/qommon/form.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2010  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import base64
import collections
import copy
import fnmatch
import mimetypes
import os
import re
import socket
import sys
import tempfile
import time
import random
import datetime
import itertools
import hashlib
import json

try:
    from PIL import Image
except ImportError:
    Image = None

from .storage import atomic_write

try:
    from feedparser import _sanitizeHTML
except ImportError:
    _sanitizeHTML = None

try:
    import DNS
except (ImportError, IOError):
    DNS = None

try:
    import magic
except ImportError:
    magic = None

import quixote
import quixote.form.widget

from quixote import get_publisher, get_request, get_response, get_session
from quixote.form import *
from quixote.html import htmltext, htmltag, htmlescape, TemplateIO
from quixote.http_request import Upload
from quixote.util import randbytes

from django.utils.encoding import force_bytes, force_text
from django.utils import six
from django.utils.six.moves.html_parser import HTMLParser

from django.conf import settings
from django.utils.safestring import mark_safe

from .template import render as render_template, Template, TemplateError
from ..portfolio import has_portfolio
from wcs.conditions import Condition, ValidationError

from . import _, ngettext, N_, force_str
from . import misc
from .humantime import humanduration2seconds, seconds2humanduration, timewords
from .misc import strftime, C_, HAS_PDFTOPPM, json_loads
from .publisher import get_cfg
from .template_utils import render_block_to_string
from .upload_storage import PicklableUpload, UploadStorageError


QuixoteForm = Form

Widget.REQUIRED_ERROR = N_('required field')
get_error_orig = Widget.get_error


def get_i18n_error(self, request=None):
    error = get_error_orig(self, request)
    if error == Widget.REQUIRED_ERROR:
        return _(error)
    return error


def is_prefilled(self):
    if hasattr(self, 'prefilled'):
        return self.prefilled
    else:
        return False


def render_title(self, title):
    if title:
        if self.required:
            title += htmltext('<span title="%s" class="required">*</span>') % _(
                    'This field is required.')
        attrs = {
            'class': 'field--label',
            'id': 'form_label_%s' % self.name,
            'for': 'form_%s' % self.name,
        }
        hint = self.get_hint()
        if hint:
            attrs['aria-describedby'] = 'form_hint_%s' % self.name
        error = self.get_error()
        if error:
            attrs['aria-invalid'] = 'true'
            attrs['aria-errormessage'] = 'form_error_%s' % self.name
        label = htmltag('label', **attrs)
        return htmltext('<div class="title">') + label + htmltext('%s</label></div>') % title
    else:
        return ''


def render_hint(self, hint):
    if not hint:
        return ''
    return htmltext('<div id="form_hint_%s" class="hint">%s</div>') % (self.name, hint)


def render_error(self, error):
    if not error:
        return ''
    return htmltext('<div id="form_error_%s" class="error">%s</div>') % (self.name, error)


def get_template_names(widget):
    template_names = []
    widget_template_name = getattr(widget, 'template_name', None)
    for extra_css_class in (getattr(widget, 'extra_css_class', '') or '').split():
        if not extra_css_class.startswith('template-'):
            continue
        template_name = extra_css_class.split('-', 1)[1]
        # full template
        template_names.append('qommon/forms/widgets/%s.html' % template_name)
        if widget_template_name:
            # widget specific variation
            template_names.append(widget_template_name.replace(
                '.html', '--%s.html' % template_name))
    if widget_template_name:
        template_names.append(widget_template_name)
    template_names.append('qommon/forms/widget.html')
    return template_names


def render(self):
    # quixote/form/widget.py, Widget::render
    def safe(text):
        return mark_safe(str(htmlescape(text)))
    if hasattr(self, 'add_media'):
        self.add_media()
    self.class_name = self.__class__.__name__
    self.rendered_title = lambda: safe(self.render_title(self.get_title()))
    self.rendered_error = lambda: safe(self.render_error(self.get_error()))
    self.rendered_hint = lambda: safe(self.render_hint(self.get_hint()))
    if get_publisher():
        context = get_publisher().substitutions.get_context_variables()
    else:
        context = {}
    context['widget'] = self
    template_names = get_template_names(self)
    return htmltext(render_template(template_names, context))

def render_widget_content(self):
    # widget content (without label, hint, etc.) is reused on status page;
    # render the appropriate block.
    self.add_media()
    template_names = get_template_names(self)
    context = {'widget': self}
    return htmltext(force_str(render_block_to_string(template_names, 'widget-content', context)))


Widget.get_error = get_i18n_error
Widget.render = render
Widget.cleanup = None
Widget.render_error = render_error
Widget.render_hint = render_hint
Widget.render_title = render_title
Widget.is_prefilled = is_prefilled
Widget.render_widget_content = render_widget_content


def string_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if getattr(self, 'prefill_attributes', None) and 'autocomplete' in self.prefill_attributes:
        attrs['autocomplete'] = self.prefill_attributes['autocomplete']
    if self.attrs:
        attrs.update(self.attrs)
    if getattr(self, 'inputmode', None):
        attrs['inputmode'] = self.inputmode
    return htmltag("input", xml_end=True, type=self.HTML_TYPE, name=self.name,
                   value=self.value, **attrs)
StringWidget.render_content = string_render_content


def file_render_content(self):
    # remove trailing $file for identifier
    attrs = {'id': 'form_' + str(self.name).rsplit('$', 1)[0]}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    return htmltag("input", xml_end=True, type=self.HTML_TYPE, name=self.name,
                   value=self.value, **attrs)
FileWidget.render_content = file_render_content


def text_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    if not attrs.get('cols'):
        attrs['cols'] = 72
    if not attrs.get('rows'):
        attrs['rows'] = 5
    if attrs.get('readonly') and not self.value:
        attrs['rows'] = 1
    return (htmltag("textarea", name=self.name, **attrs) +
        htmlescape(self.value or "") + htmltext("</textarea>"))
TextWidget.render_content = text_render_content


class SubmitWidget(quixote.form.widget.SubmitWidget):
    def __init__(self, *args, **kwargs):
        self.extra_css_class = kwargs.pop('extra_css_class', None)
        super().__init__(*args, **kwargs)

    def render_content(self):
        if self.name in ('cancel', 'previous', 'save-draft'):
            self.attrs['formnovalidate'] = 'formnovalidate'
        value = (self.label and htmlescape(self.label) or None)
        return htmltag('button', name=self.name, value=value, **self.attrs) + \
                self.label + htmltext('</button>')


class RadiobuttonsWidget(quixote.form.RadiobuttonsWidget):
    template_name = 'qommon/forms/widgets/radiobuttons.html'
    has_inside_labels = True
    content_extra_attributes = {'role': 'radiogroup'}

    def __init__(self, name, value=None, **kwargs):
        self.options_with_attributes = kwargs.pop('options_with_attributes', None)
        super(RadiobuttonsWidget, self).__init__(name, value=value, **kwargs)

    def get_options(self):
        options = self.options_with_attributes or self.options
        for i, option in enumerate(options):
            object, description, key = option[:3]
            yield {
                'value': key,
                'label': description,
                'disabled': bool(self.options_with_attributes and option[-1].get('disabled')),
                'selected': self.is_selected(object)
            }


def checkbox_render_content(self, standalone=True):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    inline_title = self.attrs.pop('inline_title', '')
    if self.attrs:
        attrs.update(self.attrs)
    checkbox = htmltag("input", xml_end=True, type="checkbox", name=self.name,
                       value="yes", checked=self.value and "checked" or None,
                       **attrs)
    if standalone:
        data_attrs = ' '.join('%s="%s"' % x for x in attrs.items() if x[0].startswith('data-'))
        # more elaborate markup so standalone checkboxes can be applied a
        # custom style.
        return htmltext('<label %s>%s<span>' % (data_attrs, checkbox)) + inline_title + htmltext('</span></label>')
    return checkbox
CheckboxWidget.render_content = checkbox_render_content


def select_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    tags = [htmltag("select", name=self.name, **attrs)]
    for object, description, key in self.options:
        if self.is_selected(object):
            selected = 'selected'
        else:
            selected = None
        if description is None:
            description = ""
        r = htmltag("option", value=key, selected=selected)
        tags.append(r + htmlescape(description) + htmltext('</option>'))
    tags.append(htmltext("</select>"))
    return htmltext("\n").join(tags)
SelectWidget.render_content = select_render_content


def get_selection_error_text(*args):
    return _('invalid value selected')

SelectWidget.SELECTION_ERROR = property(get_selection_error_text)


def transfer_form_value(self, request):
    # transfer form value (set in constructor, or using set_value) to
    # request.form{}
    request.form[self.name] = self.value


Widget.transfer_form_value = transfer_form_value


class Form(QuixoteForm):
    TOKEN_NOTICE = N_(
            "The form you have submitted is invalid.  Most "
            "likely it has been successfully submitted once "
            "already.  Please review the form data "
            "and submit the form again.")
    ERROR_NOTICE = N_("There were errors processing your form.  See below for details.")

    info = None
    captcha = None
    advanced_label = '+'

    def __init__(self, *args, **kwargs):
        if kwargs.get('advanced_label'):
            self.advanced_label = kwargs.pop('advanced_label')
        QuixoteForm.__init__(self, *args, **kwargs)
        self.global_error_messages = None

    def add_captcha(self, hint=None):
        if not self.captcha and not (get_session().won_captcha or get_session().user):
            self.captcha = CaptchaWidget('captcha', hint=hint)

    def add_submit(self, name, value=None, **kwargs):
        return self.add(SubmitWidget, name, value, **kwargs)

    def add(self, widget_class, name, *args, **kwargs):
        if kwargs and 'render_br' not in kwargs:
            kwargs['render_br'] = False
        advanced = False
        if kwargs and kwargs.get('advanced', False):
            advanced = True
            del kwargs['advanced']
        QuixoteForm.add(self, widget_class, name, *args, **kwargs)
        widget = self._names[name]
        if 'id' not in kwargs and 'id' in widget.attrs:
            # don't let quixote3 assign a default id to widgets
            del widget.attrs['id']
        widget.advanced = advanced
        return widget

    def remove(self, name):
        widget = self._names.get(name)
        if widget:
            del self._names[name]
            self.widgets.remove(widget)

    def get_all_widgets(self):
        l = QuixoteForm.get_all_widgets(self)
        if self.captcha:
            l.append(self.captcha)
        return l

    def add_global_errors(self, error_messages):
        self.global_error_messages = error_messages

    def _get_default_action(self):
        if get_request().get_header('x-popup') == 'true':
            # do not leave action empty for popups, as they get embedded into
            # another URI
            return get_request().get_path()
        return QuixoteForm._get_default_action(self)

    def render_button(self, button):
        r = TemplateIO(html=True)
        classnames = '%s widget %s-button %s' % (
                button.__class__.__name__, button.name,
                getattr(button, 'extra_css_class', None) or '')
        r += htmltext('<div class="%s">') % classnames
        r += htmltext('<div class="content">')
        r += button.render_content()
        r += htmltext('</div>')
        r += htmltext('</div>')
        r += htmltext('\n')
        return r.getvalue()

    def _render_submit_widgets(self):
        r = TemplateIO(html=True)
        if self.submit_widgets:
            r += htmltext('<div class="buttons submit">')
            for widget in self.submit_widgets:
                if widget.name == 'prefill':
                    continue
                r += self.render_button(widget)
            r += htmltext('</div>')
        return r.getvalue()

    def _render_info_notice(self):
        return htmltext('<div class="infonotice">%s</div>' % _(self.info))

    def _render_error_notice(self):
        errors = []
        classnames = ['errornotice']
        if self.has_errors():
            errors.append(_(QuixoteForm._render_error_notice(self)))
        if self.global_error_messages:
            errors.extend(self.global_error_messages)
            classnames.append('global-errors')
        t = TemplateIO(html=True)
        t += htmltext('<div class="%s" role="status">' % ' '.join(classnames))
        for error in errors:
            t += htmltext('<p>%s</p>') % error
        t += htmltext('</div>')
        return t.getvalue()

    def _render_body(self):
        r = TemplateIO(html=True)
        if self.has_errors() or self.global_error_messages:
            r += self._render_error_notice()
        if self.info:
            r += self._render_info_notice()
        if 'prefill' in self._names:
            r += self._render_prefill_widgets()

        r += self._render_widgets()
        if self.captcha:
            r += self.captcha.render()
        r += self._render_submit_widgets()
        return r.getvalue()

    def _render_prefill_widgets(self):
        r = TemplateIO(html=True)
        if self.submit_widgets:
            r += htmltext('<div class="prefill buttons">')
            for widget in self.submit_widgets:
                if widget.name == 'prefill':
                    r += widget.render()
            r += htmltext('</div>')
        return r.getvalue()

    def _render_widgets(self):
        r = TemplateIO(html=True)
        advanced_widgets = []
        for widget in self.widgets:
            if not hasattr(widget, 'advanced') or not widget.advanced:
                r += widget.render()
            else:
                advanced_widgets.append(widget)
        if advanced_widgets:
            get_response().add_javascript(['jquery.js', 'qommon.forms.js'])
            r += htmltext('<fieldset class="form-plus">')
            r += htmltext('<legend>%s</legend>') % self.advanced_label
            for widget in advanced_widgets:
                r += widget.render()
            r += htmltext('</fieldset>')
        return r.getvalue()

    def add_media(self):
        for widget in self.get_all_widgets():
            if hasattr(widget, 'add_media'):
                widget.add_media()


class HtmlWidget(object):
    error = None
    name = None

    def __init__(self, string, title = None, *args, **kwargs):
        self.string = string
        self.title = title

    def render(self):
        return self.render_content()

    def render_content(self):
        content = self.title or self.string or ''
        if getattr(self, 'is_hidden', False):
            content = htmltext(str(content).replace('>', ' style="display: none">', 1))
        return htmltext(content)

    def has_error(self, request):
        return False

    def parse(self, *args):
        pass

    def clear_error(self, request=None):
        pass

    def transfer_form_value(self, request):
        pass


class CommentWidget(Widget):
    template_name = 'qommon/forms/widgets/comment.html'

    def __init__(self, content, extra_css_class):
        super(CommentWidget, self).__init__(name='')
        self.content = content
        self.extra_css_class = extra_css_class

    def has_error(self, request):
        return False

    def parse(self, *args, **kwargs):
        pass

    def clear_error(self, request=None):
        pass


class CompositeWidget(quixote.form.CompositeWidget):
    content_extra_attributes = {'role': 'group'}

    def add_hidden(self, name, value=None, **kwargs):
        self.add(HiddenWidget, name, value, **kwargs)

    def transfer_form_value(self, request):
        for widget in self.get_widgets():
            widget.transfer_form_value(request)

    def render_as_thead(self):
        r = TemplateIO(html=True)
        r += htmltext('<tr>\n')
        for widget in self.get_widgets():
            r += htmltext('<th>')
            r += widget.get_title()
            r += htmltext('</th>')
        r += htmltext('</tr>\n')
        return r.getvalue()

    def render_content_as_tr(self):
        r = TemplateIO(html=True)
        r += htmltext('<tr>\n')
        for widget in self.get_widgets():
            extra_attributes = ''
            classnames = '%s widget' % widget.__class__.__name__
            if hasattr(widget, 'extra_css_class') and widget.extra_css_class:
                classnames += ' ' + widget.extra_css_class
            if hasattr(widget, 'content_extra_attributes'):
                extra_attributes = ' '.join(['%s=%s' % x for x in widget.content_extra_attributes.items()])
            r += htmltext('<td><div class="%s"><div class="content" %s>' % (
                    classnames, extra_attributes))
            r += widget.render_content()
            r += htmltext('</div></div></td>')
        r += htmltext('</tr>\n')
        return r.getvalue()


class StringWidget(quixote.form.StringWidget):
    def __init__(self, name, *args, **kwargs):
        if 'readonly' in kwargs and not kwargs.get('readonly'):
            del kwargs['readonly']
        elif 'readonly' in kwargs:
            kwargs['readonly'] = 'readonly'
        self.validation_function = kwargs.pop('validation_function', None)
        super(StringWidget, self).__init__(name, *args, **kwargs)

    def _parse(self, request):
        quixote.form.StringWidget._parse(self, request)
        if self.value:
            self.value = self.value.strip()
        if self.value and self.validation_function:
            try:
                self.validation_function(self.value)
            except ValueError as e:
                self.error = str(e)


class DurationWidget(StringWidget):
    def __init__(self, name, value=None, **kwargs):
        if value:
            value = seconds2humanduration(int(value))
        if 'hint' in kwargs:
            kwargs['hint'] += htmltext('<br>')
        else:
            kwargs['hint'] = ''
        kwargs['hint'] += htmltext(
                _('Usable units of time: %s.')) % ', '.join(timewords())
        super(DurationWidget, self).__init__(name, value=value, **kwargs)

    def parse(self, request=None):
        value = super(DurationWidget, self).parse(request)
        return str(humanduration2seconds(self.value)) if value else None


class TextWidget(quixote.form.TextWidget):
    prefill_attributes = None

    def __init__(self, name, *args, **kwargs):
        self.validation_function = kwargs.pop('validation_function', None)
        super(TextWidget, self).__init__(name, *args, **kwargs)

    def add_media(self):
        if self.prefill_attributes and 'geolocation' in self.prefill_attributes:
            get_response().add_javascript(['qommon.geolocation.js'])

    def _parse(self, request, use_validation_function=True):
        quixote.form.TextWidget._parse(self, request)
        if self.value is not None:
            try:
                maxlength = int(self.attrs.get('maxlength', 0))
            except (TypeError, ValueError):
                maxlength = 0
            if maxlength:
                if len(self.value) > maxlength:
                    self.error = _('too many characters (limit is %d)') % maxlength
            if use_validation_function and self.validation_function:
                try:
                    self.validation_function(self.value)
                except ValueError as e:
                    self.error = str(e)


class CheckboxWidget(quixote.form.CheckboxWidget):
    '''
    Widget just like CheckboxWidget but with an effective support for the
    required attribute, if required the checkbox will have to be checked.
    '''
    template_name = 'qommon/forms/widgets/checkbox.html'

    def _parse(self, request):
        self.value = self.name in request.form and not request.form[self.name] in (False, '', 'False')
        if self.required and not self.value:
            self.set_error(self.REQUIRED_ERROR)

    def set_value(self, value):
        if value in (None, False, '', 'False'):
            self.value = False
        else:
            self.value = True


class UploadedFile: #pylint: disable=C1001
    def __init__(self, directory, filename, upload):
        self.directory = directory
        self.base_filename = upload.base_filename
        self.content_type = upload.content_type

        # Find a good filename
        if filename:
            self.filename = filename
        elif self.base_filename:
            self.filename = self.base_filename
        else:
            t = datetime.datetime.now().isoformat()
            fd, path = tempfile.mkstemp(prefix=t, suffix='.upload',
                    dir=self.dir_path())
            os.close(fd)
            self.filename = os.path.basename(filename)

        if upload.fp:
            self.set_content(upload.fp.read())

    def set_content(self, content):
        self.size = len(content)
        file_path = self.build_file_path()
        if not os.path.exists(self.dir_path()):
            os.mkdir(self.dir_path())
        open(file_path, 'wb').write(content)

    def dir_path(self):
        return os.path.join(get_publisher().app_dir, self.directory)

    def build_file_path(self):
        return os.path.join(get_publisher().app_dir, self.directory,
                self.filename)

    def get_file(self):
        return open(self.build_file_path(), 'rb')

    def get_content(self):
        return self.get_file().read()

    def build_response(self):
        response = get_response()
        response.content_type = self.content_type
        response.set_header('content-disposition',
                'attachment; filename="%s"' % self.base_filename)
        return self.get_file().read()


class UploadValidationError(Exception):
    pass


class UploadWidget(CompositeWidget):
    def __init__(self, name, value=None, directory=None, filename=None,
            validation=None, **kwargs):
        CompositeWidget.__init__(self, name, **kwargs)
        del kwargs['title']
        kwargs.pop('hint')
        self.value = value
        self.directory = directory or 'uploads'
        self.filename = filename
        self.validation = validation
        if value:
            self.add(CheckboxWidget, 'orexisting',
                            title = _('Use previous file.'), value=True)
            self.widgets[-1].render_br = False
            self.add(FileWidget, 'file', title = _('Or upload a new one'), **kwargs)
        else:
            self.add(FileWidget, 'file', **kwargs)

    def _parse(self, request):
        if self.value and self.get('orexisting'):
            pass
        elif self.get('file'):
            upload = self.get('file')
            self.value = UploadedFile(self.directory, self.filename, upload)
            if self.validation:
                try:
                    self.validation(upload)
                except UploadValidationError as e:
                    self.error = str(e)
        else:
            self.value = None


class FileWithPreviewWidget(CompositeWidget):
    """Widget that proposes a File Upload widget but that stores the file
    ondisk so it has a "readonly" mode where the filename is shown."""

    template_name = 'qommon/forms/widgets/file.html'

    extra_css_class = 'file-upload-widget'
    max_file_size = None
    file_type = None

    max_file_size_bytes = None # will be filled automatically

    def __init__(self, name, value=None, **kwargs):
        self.storage = kwargs.pop('storage', None)
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.value = value
        self.readonly = kwargs.get('readonly')
        self.max_file_size = kwargs.pop('max_file_size', None)
        self.automatic_image_resize = kwargs.pop('automatic_image_resize', False)
        self.allow_portfolio_picking = has_portfolio() and kwargs.pop('allow_portfolio_picking', True)
        if self.max_file_size:
            self.max_file_size_bytes = FileSizeWidget.parse_file_size(self.max_file_size)
        self.add(HiddenWidget, 'token')
        if not self.readonly:
            attrs = {'data-url': get_publisher().get_root_url() + 'tmp-upload'}
            if self.storage:
                attrs['data-url'] += '?storage=%s' % self.storage
            self.file_type = kwargs.pop('file_type', None)
            if self.file_type:
                attrs['accept'] = ','.join(self.file_type)
            if self.max_file_size_bytes:
                # this could be used for client size validation of file size
                attrs['data-max-file-size'] = str(self.max_file_size_bytes)
            self.add(FileWidget, 'file', render_br=False, attrs=attrs)
        if value:
            self.set_value(value)

    def is_image(self):
        if getattr(self, 'file_type', None):
            return all([x.startswith('image/') for x in getattr(self, 'file_type', None) or []])
        return False

    def set_value(self, value):
        self.value = value
        if hasattr(self.value, 'token'):
            self.get_widget('token').set_value(self.value.token)
            if not get_session().get_tempfile(self.value.token):
                # oops, it has a token but it's not in the session; this is
                # probably because it was restored from a draft file created
                # from an expired session.
                self.value.token = get_session().add_tempfile(self.value, storage=self.storage).get('token')
                self.get_widget('token').set_value(self.value.token)

    def add_media(self):
        get_response().add_javascript(['qommon.fileupload.js'])
        if not self.readonly and get_request().user and self.allow_portfolio_picking:
            get_response().add_javascript(['../../i18n.js', 'fargo.js'])

    def tempfile(self):
        return get_session().get_tempfile(self.get('token')) or {}

    def has_tempfile_image(self):
        temp = self.tempfile()
        if not temp:
            return False
        if not temp.get('size'):  # empty or RemoteOpaque file
            return False

        filetype = (mimetypes.guess_type(temp.get('orig_filename', '')) or [''])[0]
        if not filetype:
            return False

        if filetype == 'application/pdf':
            return HAS_PDFTOPPM

        if not filetype.startswith('image/'):
            return False

        if Image:
            image_content = get_session().get_tempfile_content(self.get('token'))
            try:
                image = Image.open(image_content.fp)
            except Exception:
                return False

        return True

    def _parse(self, request):
        self.value = None
        if self.get('token'):
            token = self.get('token')
        elif self.get('file'):
            try:
                token = get_session().add_tempfile(self.get('file'), storage=self.storage)['token']
            except UploadStorageError as e:
                self.error = _('failed to store file (system error)')
                return
            request.form[self.get_widget('token').get_name()] = token
        else:
            token = None

        session = get_session()
        if token:
            self.value = session.get_tempfile_content(token)

        if self.value is None:
            # there's no file, the other checks are irrelevant.
            return

        if self.storage and self.storage != self.storage:
            self.error = _('unknown storage system (system error)')
            return

        # Don't trust the browser supplied MIME type, update the Upload object
        # with a MIME type created with magic (or based on the extension if the
        # module is missing).
        #
        # This also helps people uploading PDF files that were downloaded from
        # sites setting a wrong MIME type (like application/force-download) for
        # various reasons.
        if magic and self.value.fp:
            if hasattr(magic, 'MagicException'):
                mime = magic.Magic(mime=True)
                filetype = mime.from_file(self.value.fp.name)
            else:  # bindings from libmagic package, obsolete after jessie
                magic_object = magic.open(magic.MIME)
                magic_object.load()
                filetype = magic_object.file(self.value.fp.name).split(';')[0]
                magic_object.close()
        else:
            filetype = getattr(self.value, 'storage_attrs', {}).get('content_type')
            if not filetype:
                filetype, encoding = mimetypes.guess_type(self.value.base_filename)

        if not filetype:
            filetype = 'application/octet-stream'

        self.value.content_type = filetype

        if self.max_file_size:
            # validate file size
            if self.value.file_size > self.max_file_size_bytes:
                self.error = _('over file size limit (%s)') % self.max_file_size
                return

        if self.file_type:
            # validate file type
            accepted_file_types = []
            for file_type in self.file_type:
                accepted_file_types.extend(file_type.split(','))

            valid_file_type = False
            for accepted_file_type in accepted_file_types:
                # fnmatch is used to handle generic mimetypes, like
                # image/*
                if fnmatch.fnmatch(self.value.content_type, accepted_file_type):
                    valid_file_type = True
                    break
            if not valid_file_type:
                self.error = _('invalid file type')

        blacklisted_file_types = get_publisher().get_site_option('blacklisted-file-types')
        if blacklisted_file_types:
            blacklisted_file_types = [x.strip() for x in blacklisted_file_types.split(',')]
        else:
            blacklisted_file_types = ['.exe', '.bat', '.com', '.pif', '.php', '.js',
                    'application/x-ms-dos-executable']
        if (os.path.splitext(self.value.base_filename)[-1].lower() in blacklisted_file_types or
                filetype in blacklisted_file_types):
            self.error = _('forbidden file type')


class EmailWidget(StringWidget):
    HTML_TYPE = 'email'

    def __init__(self, *args, **kwargs):
        StringWidget.__init__(self, *args, **kwargs)
        if not 'size' in kwargs:
            self.attrs['size'] = '35'

    def add_media(self):
        get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js'])
        get_response().add_javascript_code('''
            const WCS_WELL_KNOWN_DOMAINS = %s;
            const WCS_VALID_KNOWN_DOMAINS = %s;
        ''' % (json.dumps(get_publisher().get_email_well_known_domains()),
               json.dumps(get_publisher().get_email_valid_known_domains())))

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            # basic tests first
            if not '@' in self.value[1:-1]:
                self.error = _('must be a valid email address')
            elif self.value[0] != '"' and ' ' in self.value:
                self.error = _('must be a valid email address')
            elif self.value[0] != '"' and self.value.count('@') != 1:
                self.error = _('must be a valid email address')
            elif get_cfg('emails', {}).get('check_domain_with_dns', True):
                # testing for domain existence
                domain = self.value.split('@')[-1]
                if [x for x in domain.split('.') if not x]:
                    # empty parts in domain, ex: @example..net, or
                    # @.example.net
                    self.error = _('invalid address domain')
                    return
                domain = force_text(domain, 'utf-8', errors='ignore')
                try:
                    domain = force_str(domain.encode('idna'))
                except UnicodeError:
                    self.error = _('invalid address domain')
                    return
                if domain == 'localhost':
                    return
                if DNS is not None:
                    DNS.ParseResolvConf()
                    try:
                        if not DNS.mxlookup(force_str(domain), timeout=7):
                            self.error = _('invalid address domain')
                    except DNS.DNSError:
                        self.error = _('invalid address domain')


class ValidationCondition(Condition):
    def __init__(self, django_condition, value):
        super(ValidationCondition, self).__init__({'type': 'django', 'value': django_condition})
        self.evaluated_value = value

    def get_data(self):
        return {'value': self.evaluated_value}


class ValidationWidget(CompositeWidget):
    validation_methods = collections.OrderedDict([
        ('digits', {
            'title': N_('Digits'),
            'regex': r'\d+',
            'html_inputmode': 'numeric'}
        ),
        ('phone', {
            'title': N_('Phone Number'),
            'regex': r'\+?[-\(\)\d\.\s/]+',
            'html_input_type': 'tel'}
        ),
        ('phone-fr', {
            'title': N_('Phone Number (France)'),
            'function': 'validate_phone_fr',
            'html_input_type': 'tel'}
        ),
        ('zipcode-fr', {
            'title': N_('Zip Code (France)'),
            'regex': r'\d{5}',
            'html_inputmode': 'numeric'}
        ),
        ('siren-fr', {
            'title': N_('SIREN Code (France)'),
            'function': 'validate_siren',
            'html_inputmode': 'numeric'}
        ),
        ('siret-fr', {
            'title': N_('SIRET Code (France)'),
            'function': 'validate_siret',
            'html_inputmode': 'numeric'}
        ),
        ('nir-fr', {
            'title': N_('NIR (France)'),
            'function': 'validate_nir'}
        ),
        ('iban', {
            'title': N_('IBAN'),
            'function': 'validate_iban'}
        ),
        ('regex', {'title': N_('Regular Expression')}),
        ('django', {'title': N_('Django Condition')}),
    ])

    def __init__(self, name, value=None, **kwargs):
        super(ValidationWidget, self).__init__(name, value=value, **kwargs)
        if not value:
            value = {}

        options = [(None, _('None'))] + [(x, _(y['title'])) for x, y in self.validation_methods.items()]

        self.add(SingleSelectWidget, 'type', options=options, value=value.get('type'),
                 attrs={'data-dynamic-display-parent': 'true'})
        self.parse()
        if not self.value:
            self.value = {}

        validation_labels = collections.OrderedDict(options)
        self.add(RegexStringWidget, 'value_regex', size=80,
                 value=value.get('value') if value.get('type') == 'regex' else None,
                 attrs={'data-dynamic-display-child-of': 'validation$type',
                        'data-dynamic-display-value': validation_labels.get('regex')})
        self.add(DjangoConditionWidget, 'value_django', size=80,
                 value=value.get('value') if value.get('type') == 'django' else None,
                 attrs={'data-dynamic-display-child-of': 'validation$type',
                        'data-dynamic-display-value': validation_labels.get('django')})
        self._parsed = False

    def _parse(self, request):
        values = {}
        type_ = self.get('type')
        if type_:
            values['type'] = type_
            value = self.get('value_%s' % type_)
            if value:
                values['value'] = value
        self.value = values or None

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_error(widget.get_error())
        for widget in self.get_widgets():
            r += widget.render_content()
        return r.getvalue()

    @classmethod
    def get_validation_function(cls, validation):
        pattern = cls.get_validation_pattern(validation)
        if pattern:
            def regex_validation(value):
                return bool(re.match(r'^(?:%s)$' % pattern, value))
            return regex_validation
        if validation['type'] == 'django' and validation.get('value'):
            def django_validation(value):
                condition = ValidationCondition(validation['value'], value=value)
                return condition.evaluate()
            return django_validation
        validation_method = cls.validation_methods.get(validation['type'])
        if validation_method and 'function' in validation_method:
            return getattr(misc, validation_method['function'])

    @classmethod
    def get_validation_pattern(cls, validation):
        validation_method = cls.validation_methods.get(validation['type'])
        if validation_method and validation_method.get('regex'):
            return validation_method.get('regex')
        if validation['type'] == 'regex':
            return validation.get('value')
        return None

    @classmethod
    def get_html_input_type(cls, validation):
        validation_method = cls.validation_methods.get(validation['type'])
        if validation_method and validation_method.get('html_input_type'):
            return validation_method.get('html_input_type')
        return 'text'

    @classmethod
    def get_html_inputmode(cls, validation):
        validation_method = cls.validation_methods.get(validation['type'])
        if validation_method and validation_method.get('html_inputmode'):
            return validation_method.get('html_inputmode')


class WcsExtraStringWidget(StringWidget):
    field = None
    prefill = False
    prefill_attributes = None

    def add_media(self):
        if self.prefill_attributes and 'geolocation' in self.prefill_attributes:
            get_response().add_javascript(['qommon.geolocation.js'])

    def render_content(self):
        if self.field and self.field.validation:
            self.HTML_TYPE = ValidationWidget.get_html_input_type(self.field.validation)
            self.inputmode = ValidationWidget.get_html_inputmode(self.field.validation)
        return super(WcsExtraStringWidget, self).render_content()

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.field and self.field.validation and self.value is not None:
            validation_function = ValidationWidget.get_validation_function(self.field.validation)
            if validation_function and not validation_function(self.value):
                self.error = _('invalid value')


class DateWidget(StringWidget):
    '''StringWidget which checks the value entered is a correct date'''
    template_name = 'qommon/forms/widgets/date.html'

    minimum_date = None
    maximum_date = None
    content_extra_css_class = 'date'

    def __init__(self, name, value=None, **kwargs):
        minimum_date = kwargs.pop('minimum_date', None)
        if minimum_date:
            self.minimum_date = misc.get_as_datetime(minimum_date)
        maximum_date = kwargs.pop('maximum_date', None)
        if maximum_date:
            self.maximum_date = misc.get_as_datetime(maximum_date)
        if kwargs.pop('minimum_is_future', False):
            if kwargs.get('date_can_be_today'):
                self.minimum_date = datetime.date.today()
            else:
                self.minimum_date = (datetime.datetime.today() + datetime.timedelta(1))
        if kwargs.pop('date_in_the_past', False):
            if kwargs.get('date_can_be_today'):
                self.maximum_date = datetime.date.today()
            else:
                self.maximum_date = (datetime.datetime.today() - datetime.timedelta(1))

        if 'date_can_be_today' in kwargs:
            del kwargs['date_can_be_today']

        if isinstance(value, datetime.datetime) or isinstance(value, datetime.date):
            value = value.strftime(misc.date_format())

        StringWidget.__init__(self, name, value=value, **kwargs)
        self.attrs['size'] = '12'
        self.attrs['maxlength'] = '10'

    def transfer_form_value(self, request):
        if isinstance(self.value, time.struct_time):
            request.form[self.name] = strftime(misc.date_format(), self.value)
        else:
            super().transfer_form_value(request)

    def parse(self, request=None):
        StringWidget.parse(self, request=request)
        return self.value

    @classmethod
    def get_format_string(cls):
        return misc.date_format()

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            try:
                value = misc.get_as_datetime(self.value).timetuple()
                self.value = strftime(self.get_format_string(), value)
            except ValueError:
                self.error = _('invalid date')
                self.value = None
                return
            if value[0] < 1800 or value[0] > 2099:
                self.error = _('invalid date')
                self.value = None
            elif self.minimum_date and value[:3] < self.minimum_date.timetuple()[:3]:
                self.error = _('invalid date: date must be on or after %s') % strftime(
                        misc.date_format(), self.minimum_date)
            elif self.maximum_date and value[:3] > self.maximum_date.timetuple()[:3]:
                self.error = _('invalid date; date must be on or before %s') % strftime(
                        misc.date_format(), self.maximum_date)

    def add_media(self):
        self.prepare_javascript()

    @classmethod
    def prepare_javascript(cls):
        get_response().add_javascript([
            'jquery.js', 'bootstrap-datetimepicker.js', 'qommon.forms.js'])
        current_language = get_request().language
        if current_language != 'en' and current_language in get_publisher().supported_languages or []:
            get_response().add_javascript(['bootstrap-datetimepicker.%s.js' % current_language])

        get_response().add_css_include('datetimepicker.css')

    def date_format(self):
        return self.get_format_string().replace('%Y', 'yyyy').replace(
                '%m', 'mm').replace('%d', 'dd').replace('%H', 'hh').replace(
                '%M', 'ii').replace('%S', 'ss')

    def start_date(self):
        return self.date_format().replace(
                    'yyyy', '%04d' % self.minimum_date.year).replace(
                    'mm', '%02d' % self.minimum_date.month).replace(
                    'dd', '%02d' % self.minimum_date.day).replace(
                    'hh', '00').replace('ii', '00').replace('ss', '00')

    def end_date(self):
        return self.date_format().replace(
                    'yyyy', '%04d' % self.maximum_date.year).replace(
                    'mm', '%02d' % self.maximum_date.month).replace(
                    'dd', '%02d' % self.maximum_date.day).replace(
                    'hh', '00').replace('ii', '00').replace('ss', '00')


class TimeWidget(DateWidget):
    template_name = 'qommon/forms/widgets/time.html'

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            try:
                value = datetime.datetime.strptime(self.value, self.get_format_string())
                self.value = strftime(self.get_format_string(), value)
            except ValueError:
                self.error = _('invalid time')
                self.value = None
                return

    @classmethod
    def get_format_string(cls):
        return '%H:%M'


class DateTimeWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        super(DateTimeWidget, self).__init__(name, value=value, **kwargs)
        date_value = None
        time_value = None
        if value:
            date_value = misc.get_as_datetime(value).strftime(DateWidget.get_format_string())
            time_value = misc.get_as_datetime(value).strftime(TimeWidget.get_format_string())
        self.add(DateWidget, 'date', value=date_value, render_br=False)
        self.add(TimeWidget, 'time', value=time_value, render_br=False)

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_widget_content()
        return r.getvalue()

    def _parse(self, request):
        date = self.get('date')
        time = self.get('time')
        if not date and not time:
            self.value = None
            return self.value
        time = time or '00:00'
        try:
            misc.get_as_datetime('%s %s' % (date, time))
        except ValueError:
            self.error = _('invalid value')
        self.value = '%s %s' % (date, time)
        return self.value


class RegexStringWidget(StringWidget):
    '''StringWidget which checks the value entered is a correct regex'''
    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            try:
                re.compile(self.value)
            except:
                self.error = _('invalid regular expression')
                self.value = None


class CheckboxesWidget(CompositeWidget):
    readonly = False
    template_name = 'qommon/forms/widgets/checkboxes.html'
    has_inside_labels = True

    def __init__(self, name, value=None, options=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.element_names = collections.OrderedDict()

        if 'title' in kwargs:
            del kwargs['title']
        if 'readonly' in kwargs:
            del kwargs['readonly']
            self.readonly = True
        self.inline = kwargs.get('inline', True)
        self.max_choices = int(kwargs.get('max_choices', 0) or 0)

        self.options_with_attributes = kwargs.pop('options_with_attributes', None)
        self.disabled_options = []
        if self.options_with_attributes:
            options = self.options_with_attributes

        for i, option in enumerate(options):
            if len(option) == 2:
                key, title = option[:2]
                name = 'element%d' % i
            else:
                _, title, key  = option[:3]
                name = 'element%s' % str(key)

            key = str(key)

            element_kwargs = kwargs.copy()
            if self.options_with_attributes and option[-1].get('disabled'):
                element_kwargs['disabled'] = 'disabled'
                self.disabled_options.append(name)
            # don't mark individual checkboxes as required
            element_kwargs.pop('required', None)

            if value and key in value:
                checked = True
            else:
                checked = False
            self.add(CheckboxWidget, name, title=title, value=checked, **element_kwargs)
            self.element_names[name] = key

    def _parse(self, request):
        if self.readonly:
            return
        values = []
        for name in self.element_names:
            if name in self.disabled_options:
                continue
            value = self.get(name)
            if value is True:
                values.append(self.element_names[name])
        self.value = values
        if self.required and not self.value:
            self.set_error(self.REQUIRED_ERROR)
        if self.value and self.max_choices and len(self.value) > self.max_choices:
            self.set_error(_('You must select at most %d answers.') % self.max_choices)

    def set_value(self, value):
        self.value = value
        for name in self.element_names:
            widget = self.get_widget(name)
            if self.element_names[name] in self.value:
                widget.set_value(True)
            else:
                widget.set_value(False)
        self._parsed = True

    def has_error(self, request=None):
        return Widget.has_error(self, request=request)

    def render_content(self):
        r = TemplateIO(html=True)
        if self.inline:
            r += htmltext('<ul class="inline">')
        else:
            r += htmltext('<ul>')
        for widget in self.get_widgets():
            if widget.attrs and 'disabled' in widget.attrs:
                r += htmltext('<li class="disabled"><label>')
            else:
                r += htmltext('<li><label>')
            if self.readonly:
                widget.attrs['disabled'] = 'disabled'
                if widget.value:
                    r += htmltext('<input type="hidden" name="%s" value="yes" >') % widget.name
                widget.name = widget.name + 'xx'
            r += widget.render_content(standalone=False)
            r += htmltext('<span>%s</span>') % widget.title
            r += htmltext('</label>')
            r += htmltext('</li>')
        r += htmltext('</ul>')
        return r.getvalue()


class ValidatedStringWidget(StringWidget):
    '''StringWidget which checks the value entered is correct according to a regex'''
    regex = None

    def __init__(self, *args, **kwargs):
        if 'regex' in kwargs:
            self.regex = kwargs.pop('regex')
        super(ValidatedStringWidget, self).__init__(*args, **kwargs)

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.regex and self.value is not None:
            match = re.match(self.regex, self.value)
            if not match or not match.group() == self.value:
                self.error = _('wrong format')


class UrlWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a correct url starting with http or https'''
    regex = r'^https?://.+'

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('must start with http:// or https:// and have a domain name')


class VarnameWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a syntactically correct
       variable name.'''
    regex = r'^[a-zA-Z][a-zA-Z0-9_]*'

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('must only consist of letters, numbers, or underscore')


class FileSizeWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a syntactically correct
       file size.'''
    regex = r'^\s*([\d]+)\s*([MKk]i?)?[oB]?\s*$'

    def __init__(self, *args, **kwargs):
        hint = kwargs.pop('hint', _('Accepted units: MB (megabytes), kB (kilobytes), for example: 3 MB'))
        ValidatedStringWidget.__init__(self, *args, hint=hint, **kwargs)

    @classmethod
    def parse_file_size(cls, value):
        try:
            value, unit = re.search(cls.regex, value).groups()
        except AttributeError: #  None has no .groups()
            raise ValueError()
        coeffs = {
            'Mi': 2**20,
            'Ki': 2**10,
            'ki': 2**10,
            'M':  10**6,
            'K':  10**3,
            'k':  10**3,
            None: 1,
        }
        return int(value) * coeffs.get(unit)

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('invalid file size')


class CaptchaWidget(CompositeWidget):
    def __init__(self, name, value = None, mode = 'arithmetic-simple', *args, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.render_br = False
        if value:
            token = value
        else:
            token = get_session().create_captcha_token()
        hidden_input = htmltext('<input type="hidden" name="%s$token" value="%s"></input>')
        self.add(HtmlWidget, 'token', title = hidden_input % (self.name, token['token']))

        # create question, and fill token['answer']
        if mode == 'arithmetic' or mode == 'arithmetic-simple':
            a, b = random.randint(2, 9), random.randint(2, 9)
            while b == a:
                # don't get twice the same number
                b = random.randint(2, 9)
            if mode == 'arithmetic-simple':
                operator = random.choice([N_('plus'), N_('minus')])
            else:
                operator = random.choice([N_('times'), N_('plus'), N_('minus')])
            if operator == 'times':
                answer = a * b
            elif operator == 'plus':
                answer = a + b
            elif operator == 'minus':
                if b > a:
                    a, b = b, a
                answer = a - b
            self.question = _('What is the result of %(a)d %(op)s %(b)d?') % {
                    'a': a, 'b': b, 'op': _(operator)}
            self.hint = kwargs.get('hint')
            if self.hint is None:
                self.hint = _('Please answer this simple mathematical question as proof you are not a bot.')
        self.add(StringWidget, 'q', required=True, attrs={'required': 'required'})
        token['answer'] = str(answer)

    def _parse(self, request):
        v = {'answer': self.get('q'),
             'token': request.form.get('%s$token' % self.name) }
        token = get_session().get_captcha_token(v['token'])
        if v['answer'] and token and token['answer'] == v['answer'].strip():
            get_session().won_captcha = True
            self.value = v
        elif v['answer']:
            self.error = _('wrong answer')

    def get_title(self):
        return self.question

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_content()
        return r.getvalue()


class WidgetList(quixote.form.widget.WidgetList):
    def __init__(self, name, value=None,
                 element_type=StringWidget,
                 element_kwargs={},
                 add_element_label="Add row",
                 max_items=None,
                 **kwargs):

        if add_element_label == 'Add row':
            add_element_label = _('Add row')

        CompositeWidget.__init__(self, name, value=value, **kwargs)
        self.element_type = element_type
        self.element_kwargs = element_kwargs
        self.element_names = []
        self.max_items = max_items

        # Add element widgets for initial value
        if value is not None:
            for element_value in value:
                self.add_element(value=element_value)
        if not self.element_names:
           # Add at least an element widget
            self.add_element()

        if not kwargs.get('readonly'):
            # add element widgets to match submitted list
            prefix = '%s$element' % self.name
            if get_request().form:
                known_prefixes = {x.split('$', 2)[1] for x in get_request().form.keys() if x.startswith(prefix)}
                for i in range(len(known_prefixes) - len(self.element_names)):
                    self.add_element()

            current_len = len(self.element_names)
            if (not max_items) or current_len < max_items:
                # Add submit to add more element widgets
                self.add(SubmitWidget, 'add_element', value=add_element_label,
                        render_br=False, extra_css_class='list-add')
                if self.get('add_element'):
                    self.add_element()
                current_len = len(self.element_names)
                if max_items and current_len >= max_items:
                    self.widgets.remove(self.get_widget('add_element'))
                    del self._names['add_element']

    def add_element(self, value=None):
        name = "element%d" % len(self.element_names)
        self.add(self.element_type, name, value=value, **self.element_kwargs)
        self.element_names.append(name)

    def add_media(self):
        get_response().add_javascript(['jquery.js', 'widget_list.js'])

    def transfer_form_value(self, request):
        for widget in self.get_widgets():
            widget.transfer_form_value(request)

    def _parse(self, request):
        super()._parse(request)
        if self.max_items and self.value and len(self.value) > self.max_items:
            self.set_error(_('Too many elements (maximum: %s)') % self.max_items)

    def set_value(self, value):
        for i in range(len(value) - len(self.element_names)):
            self.add_element()
        for element_name, subvalue in zip(self.element_names, value):
            self.get_widget(element_name).set_value(subvalue)

    def render(self):
        return render(self)

    def render_content(self):
        r = TemplateIO(html=True)
        add_element_widget = self.get_widget('add_element')
        clear_errors = False
        if self.value is None and self.required:
            # if there's no value and it's marked required, there won't be
            # values in subwidgets either, clear them instead of filling the
            # screen with "required field" messages.
            clear_errors = True
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            if clear_errors:
                widget.clear_error()
            r += widget.render()
        if add_element_widget:
            r += add_element_widget.render()
        return r.getvalue()


class WidgetDict(quixote.form.widget.WidgetDict):
    # Fix the title and hint setting
    # FIXME: to be fixed in Quixote upstream : title and hint parameters should be removed
    def __init__(self, name, value=None, title='', hint='',
                 element_key_type=StringWidget,
                 element_value_type=StringWidget,
                 element_key_kwargs={},
                 element_value_kwargs={},
                 add_element_label='Add row', **kwargs):

        if add_element_label == 'Add row':
            add_element_label = _('Add row')

        quixote.form.widget.WidgetDict.__init__(self, name, value,
                element_key_type=element_key_type,
                element_value_type=element_value_type,
                element_key_kwargs=element_key_kwargs,
                element_value_kwargs=element_value_kwargs,
                add_element_label=add_element_label,
                **kwargs)
        if title:
            self.title = title
        if hint:
            self.hint = hint

    def render_content(self):
        r = TemplateIO(html=True)

        lines = []
        for name in self.element_names:
            if name in ('add_element', 'added_elements'):
                continue
            key_widget = self.get_widget(name + 'key')
            value_widget = self.get_widget(name + 'value')
            lines.append({'key': key_widget, 'value': value_widget})

        def sort_key(line):
            if not isinstance(line['key'], StringWidget) or not line['key'].value:
                return (1, None)  # empty keys always at the end
            return (0, line['key'].value)
        lines.sort(key=sort_key)

        for line in lines:
            r += htmltext('<div class="dict-key">%s</div>'
                          '<div class="dict-separator">: </div>'
                          '<div class="dict-value">%s</div>') % (
                line['key'].render(),
                line['value'].render())
            r += htmltext('\n')
        add_element_widget = self.get_widget('add_element')
        add_element_widget.render_br = False
        add_element_widget.extra_css_class = 'list-add'
        r += add_element_widget.render()
        r += self.get_widget('added_elements').render()
        return r.getvalue()


class WysiwygTextWidget(TextWidget):
    def _parse(self, request):
        TextWidget._parse(self, request, use_validation_function=False)
        if self.value:
            if _sanitizeHTML:
                self.value = _sanitizeHTML(self.value, get_request().charset, 'text/html')
            if self.value.startswith('<br />'):
                self.value = self.value[6:]
            if self.value.endswith('<br />'):
                self.value = self.value[:-6]
            # unescape Django template tags
            parser = HTMLParser()
            charset = get_publisher().site_charset
            def unquote_django(matchobj):
                return force_str(parser.unescape(force_text(matchobj.group(0), charset)))
            self.value = re.sub('{[{%](.*?)[%}]}', unquote_django, self.value)
            if self.validation_function:
                try:
                    self.validation_function(self.value)
                except ValueError as e:
                    self.error = str(e)

    def add_media(self):
        get_response().add_javascript(['qommon.wysiwyg.js'])

    def render_content(self):
        from ckeditor.widgets import DEFAULT_CONFIG as CKEDITOR_DEFAULT_CONFIG
        attrs = self.attrs.copy()
        config = copy.deepcopy(CKEDITOR_DEFAULT_CONFIG)
        config.update(settings.CKEDITOR_CONFIGS['default'])
        attrs['data-config'] = json.dumps(config)
        return (htmltag('textarea', name=self.name, **attrs) +
                htmlescape(self.value or '') +
                htmltext("</textarea>"))


class TableWidget(CompositeWidget):
    readonly = False

    def __init__(self, name, value = None, rows = None, columns = None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.rows = rows
        self.columns = columns

        if 'title' in kwargs:
            del kwargs['title']

        if kwargs.get('readonly'):
            self.readonly = True

        for i, row in enumerate(rows):
            for j, column in enumerate(columns):
                widget = self.add_widget(kwargs, i, j)
                widget = self.get_widget('c-%s-%s' % (i, j))
                if value and self.readonly:
                    widget.set_value(value[i][j])
                    widget.transfer_form_value(get_request())

    def add_widget(self, kwargs, i, j):
        widget_kwargs = {}
        if kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(StringWidget, 'c-%s-%s' % (i, j), **widget_kwargs)

    def render_content(self):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr><td></td>')
        for column in self.columns:
            r += htmltext('<th><span>%s</span></th>') % column
        r += htmltext('</tr></thead><tbody>')
        for i, row in enumerate(self.rows):
            r += htmltext('<tr><th>%s</th>') % row
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                r += htmltext('<td>')
                r += widget.render_content()
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody></table>')
        return r.getvalue()

    def parse(self, request=None):
        CompositeWidget.parse(self, request=request)
        if request is None:
            request = get_request()
        if (request.form or request.get_method() == 'POST') and self.required:
            if not self.value:
                self.set_error(self.REQUIRED_ERROR)
            else:
                for row in self.value:
                    for column in row:
                        if column:
                            break
                    else:
                        continue
                    break
                else:
                    self.set_error(self.REQUIRED_ERROR)
        return self.value

    def _parse(self, request):
        if self.readonly:
            return
        table = []
        for i, row in enumerate(self.rows):
            row = []
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                row.append(widget.parse())
            table.append(row)
        self.value = table

    def set_value(self, value):
        self.value = value
        if not value:
            return
        for i, row in enumerate(self.rows):
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                try:
                    widget.set_value(value[i][j])
                except IndexError:
                    pass


class SingleSelectTableWidget(TableWidget):
    def add_widget(self, kwargs, i, j):
        widget_kwargs = {'options': kwargs.get('options')}
        if kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(SingleSelectWidget, 'c-%s-%s' % (i, j), **widget_kwargs)


class CheckboxesTableWidget(TableWidget):
    def add_widget(self, kwargs, i, j):
        widget_kwargs = {'options': kwargs.get('options')}
        if kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(CheckboxWidget, 'c-%s-%s' % (i, j), **widget_kwargs)


class SingleSelectHintWidget(SingleSelectWidget):
    template_name = 'qommon/forms/widgets/select.html'
    readonly = None

    def __init__(self, name, value=None, **kwargs):
        self.options_with_attributes = kwargs.pop('options_with_attributes', None)
        self.select2 = kwargs.pop('select2', None)
        super(SingleSelectHintWidget, self).__init__(name, value=value, **kwargs)

    def add_media(self):
        if self.select2:
            get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
            get_response().add_css_include('../js/select2/select2.css')

    def separate_hint(self):
        return (self.hint and len(self.hint) > 80)

    def get_options(self):
        if self.options_with_attributes:
            options = self.options_with_attributes[:]
        else:
            options = self.options[:]
        if options[0][0] is None:
            options = self.options[1:]

        for option in options:
            object, description, key = option[:3]
            html_attrs = {}
            html_attrs['value'] = key
            if self.is_selected(object):
                html_attrs['selected'] = 'selected'
            elif self.readonly and self.value:
                # if readonly only include the selected option
                continue
            if self.options_with_attributes and option[-1].get('disabled'):
                html_attrs['disabled'] = 'disabled'
            if description is None:
                description = ''
            yield {'description': description, 'attrs': html_attrs,
                    'options': option[-1] if self.options_with_attributes else None}

    def has_valid_options(self):
        # helper function for templates, return True if there's at least a
        # valid option.
        for option in self.get_options():
            if not option['attrs'].get('disabled'):
                return True
        return False

    def get_hint(self):
        if self.separate_hint():
            return SingleSelectWidget.get_hint(self)
        return None


class WidgetListAsTable(WidgetList):
    def render_content(self):
        r = TemplateIO(html=True)
        add_element_widget = self.get_widget('add_element')
        if add_element_widget:
            add_element_widget.render_br = False
            add_element_widget.extra_css_class = 'list-add'
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            if not hasattr(widget, 'render_content_as_tr'):
                r += widget.render()
        r += htmltext('<table>')
        r += htmltext('<thead>')
        r += self.get_widgets()[0].render_as_thead()
        r += htmltext('</thead>')
        r += htmltext('<tbody>')
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            if hasattr(widget, 'render_content_as_tr'):
                r += widget.render_content_as_tr()
        r += htmltext('</tbody>')
        r += htmltext('</table>')
        if add_element_widget and not self.readonly:
            r += add_element_widget.render()
        return r.getvalue()

    def render(self):
        return render(self)


class TableRowWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        for i, column in enumerate(self.columns):
            self.add(StringWidget, name='col%s'%i, title=column, **kwargs)


class TableListRowsWidget(WidgetListAsTable):
    readonly = False

    def create_row_class(self, columns):
        class klass(TableRowWidget): pass
        setattr(klass, 'columns', columns)
        return klass

    def add_element(self, value=None):
        name = "element%d" % len(self.element_names)
        self.add(self.table_row_class, name, value=value, **self.widget_kwargs)
        self.element_names.append(name)

    def __init__(self, name, value=None, columns=None, min_rows=5, **kwargs):
        self.table_row_class = self.create_row_class(columns)
        self.widget_kwargs = {}
        if 'readonly' in kwargs:
            del kwargs['readonly']
            self.readonly = True
            self.widget_kwargs['readonly'] = 'readonly'
        WidgetListAsTable.__init__(self, name, value,
                element_type=self.table_row_class,
                element_kwargs=self.widget_kwargs, **kwargs)
        self.columns = columns

        while len(self.element_names) < min_rows:
            self.add_element()

        self.set_value(value)

    def parse(self, request=None):
        WidgetListAsTable.parse(self, request=request)
        if request is None:
            request = get_request()
        add_element_pushed = self.get_widget('add_element').parse()
        if (request.form or request.get_method() == 'POST') and self.required:
            if not self.value and not add_element_pushed:
                self.set_error(self.REQUIRED_ERROR)
            for row in self.value or []:
                for column in row:
                    if column:
                        break
                else:
                    continue
                break
            else:
                if not add_element_pushed:
                    self.set_error(self.REQUIRED_ERROR)
        return self.value

    def _parse(self, request):
        if self.readonly:
            return
        table = []
        for i, row_name in enumerate(self.element_names):
            row = []
            row_widget = self.get_widget(row_name)
            notnull = False
            for j, column in enumerate(self.columns):
                widget = row_widget.get_widget('col%s'%j)
                row.append(widget.parse())
                if row[-1]:
                    notnull = True
            if notnull:
                table.append(row)
        self.value = table

    def set_value(self, value):
        self.value = value
        if not value:
            return
        while len(self.element_names) < len(value):
            self.add_element()
        for i, row_name in enumerate(self.element_names):
            widget_row = self.get_widget(row_name)
            for j, column in enumerate(self.columns):
                widget = widget_row.get_widget('col%s'%j)
                try:
                    widget.set_value(value[i][j])
                    widget.transfer_form_value(get_request())
                except IndexError:
                    pass


class RankedItemsWidget(CompositeWidget):
    readonly = False
    has_inside_labels = True

    def __init__(self, name, value=None, elements=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.element_names = {}

        if 'title' in kwargs:
            del kwargs['title']
        if 'readonly' in kwargs:
            if kwargs['readonly']:
                self.readonly = True
            del kwargs['readonly']
        if 'required' in kwargs:
            if kwargs['required']:
                self.required = True
            del kwargs['required']

        self.randomize_items = False
        if 'randomize_items' in kwargs:
            if kwargs['randomize_items']:
                self.randomize_items = True
            del kwargs['randomize_items']

        for v in elements:
            if type(v) is tuple:
                title = v[1]
                key = v[0]
                if type(key) is int:
                    name = 'element%d' % v[0]
                elif type(key) in (str, htmltext):
                    name = str('element%s' % v[0])
                    key = str(key)
                else:
                    raise NotImplementedError()
            else:
                title = v
                key = v
                name = 'element%d' % len(self.element_names.keys())

            if value:
                position = value.get(key)
            else:
                position = None
            self.add(IntWidget, name, title=title, value=position, size=5,
                     required=False, **kwargs)
            self.element_names[name] = key

        if self.randomize_items:
            random.shuffle(self.widgets)

        if self.readonly:
            self.widgets.sort(key=lambda x: x.value or sys.maxsize)

            if value:
                # in readonly mode, we mark all fields as already parsed, to
                # avoid getting them reinitialized on a has_error() call
                for name in self.element_names:
                    self.get_widget(name)._parsed = True
                self._parsed = True

    def _parse(self, request):
        values = {}
        for name in self.element_names:
            value = self.get(name)
            if value is not None:
                values[self.element_names[name]] = value
            if value is not None and type(value) is not int:
                self.get_widget(name).set_error(IntWidget.TYPE_ERROR)
        self.value = values or None

    def set_value(self, value):
        self.value = value
        if value:
            for name in self.element_names:
                self.get_widget(name).set_value(self.value.get(self.element_names[name]))

    def render_content(self):
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        for widget in self.get_widgets():
            if widget.has_error():
                r += htmltext('<li class="error"><label>')
            else:
                r += htmltext('<li><label>')
            if self.readonly:
                widget.attrs['disabled'] = 'disabled'
                if widget.value:
                    r += htmltext('<input type="hidden" name="%s" value="%s" >') % (
                            widget.name, widget.value)
                widget.name = widget.name + 'xx'
            r += widget.render_content()
            r += widget.title
            r += htmltext('</label>')
            r += htmltext('</li>')
        r += htmltext('</ul>')
        return r.getvalue()


class JsonpSingleSelectWidget(Widget):
    template_name = 'qommon/forms/widgets/select_jsonp.html'

    def __init__(self, name, value=None, url=None, **kwargs):
        super(JsonpSingleSelectWidget, self).__init__(name, value=value, **kwargs)
        self.url = url

    def add_media(self):
        get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
        get_response().add_css_include('../js/select2/select2.css')

    def get_display_value(self):
        if self.value is None:
            value = None
        else:
            value = htmlescape(self.value)
        if not value or not get_session().jsonp_display_values:
            return None
        key = '%s_%s' % (self.url, value)
        return get_session().jsonp_display_values.get(key)

    def get_select2_url(self):
        if Template.is_template_string(self.url):
            vars = get_publisher().substitutions.get_context_variables(mode='lazy')
            # skip variables that were not set (None)
            vars = dict((x, y) for x, y in vars.items() if y is not None)
            url = misc.get_variadic_url(self.url, vars, encode_query=False)
        else:
            url = self.url
        return url

    def parse(self, request=None):
        if request and request.form.get(self.name) and request.form.get(self.name + '_display'):
            # store text value associated to the jsonp value
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            value = request.form.get(self.name)
            display_value = request.form.get(self.name + '_display')
            get_session().jsonp_display_values['%s_%s' % (self.url, value)] = display_value

        return Widget.parse(self, request=request)


class AutocompleteStringWidget(WcsExtraStringWidget):
    url = None

    def __init__(self, *args, **kwargs):
        self.url = kwargs.pop('url', None)
        WcsExtraStringWidget.__init__(self, *args, **kwargs)

    def add_media(self):
        get_response().add_javascript(['jquery.js', 'jquery-ui.js'])

    def render_content(self):
        if Template.is_template_string(self.url):
            vars = get_publisher().substitutions.get_context_variables(mode='lazy')
            # skip variables that were not set (None)
            vars = dict((x, y) for x, y in vars.items() if y is not None)
            url = misc.get_variadic_url(self.url, vars, encode_query=False)
        else:
            url = self.url

        r = TemplateIO(html=True)
        r += WcsExtraStringWidget.render_content(self)
        if not url:
            # there's no autocomplete URL, get out now.
            return r.getvalue()

        data_type = 'json' if url.startswith('/api/autocomplete/') else 'jsonp'
        r += htmltext("""
<script id="script_%(id)s">
$(function() {
  $("#form_%(id)s").autocomplete({
    source: function( request, response ) {
      $.ajax({
        url: $("#form_%(id)s").data('uiAutocomplete').options.url,
        dataType: "%(data_type)s",
        data: {
          q: request.term
        },
        success: function( data ) {
          response( $.map(data.data, function(item) {
            return {label: item.text, value: item.label};
           }));
        }
      });
    },
    minLength: 2,
    open: function() {
      $(this).removeClass("ui-corner-all").addClass("ui-corner-top");
    },
    close: function() {
      $(this).removeClass("ui-corner-top").addClass("ui-corner-all");
    }
  });
""" % {'id': self.name, 'data_type': data_type})

        if not '[var_' in url:
            r += htmltext("""
$("#form_%(id)s").data('uiAutocomplete').options.url = '%(url)s';
""" % {'id': self.name, 'url': url})

        if '[var_' in url:
            # if this is a parametric url, store template url and hook to the
            # appropriate onchange event to give the url to autocomplete
            r += htmltext("""
$("#form_%(id)s").data('uiAutocomplete').options.wcs_base_url = '%(url)s';
""" % {'id': self.name, 'url': url})
            variables = re.findall(r'\[(var_.+?)\]', url)
            r += htmltext("""
function url_replace_%(id)s() {
    var url = $("#form_%(id)s").data('uiAutocomplete').options.wcs_base_url;""" % {'id': self.name})
            for variable in variables:
                r += htmltext("""
    selector = '#' + $('#%(variable)s').data('valuecontainerid');
    url = url.replace('[%(variable)s]', $(selector).val() || '');""" % {'variable': variable})
            r += htmltext("""
    $("#form_%(id)s").data('uiAutocomplete').options.url = url;
    if ($("form_%(id)s").val() != $("form_%(id)s").attr('value'))
        $("#form_%(id)s").val('');
}
""" % {'id': self.name} )
            for variable in variables:
                r += htmltext("""
$('#%(variable)s').change(url_replace_%(id)s);
$('#%(variable)s').change();
""" % {'id': self.name, 'variable': variable})

        r += htmltext("""
});
</script>""")

        return r.getvalue()


class ColourWidget(SingleSelectWidget):

    def __init__(self, *args, **kwargs):
        colours = ['%s%s%s' % x for x in itertools.product(('00', '66', '99', 'FF'), repeat=3)]
        SelectWidget.__init__(self, options=colours, *args, **kwargs)
        self.attrs['class'] = 'colour-picker'

    def add_media(self):
        get_response().add_javascript(['jquery.js', 'jquery.colourpicker.js'])

    def _parse(self, request):
        SingleSelectWidget._parse(self, request)
        if self.value is not None:
            self.value = self.value.upper()


class PasswordEntryWidget(CompositeWidget):
    min_length = 0
    max_length = 0
    count_uppercase = 0
    count_lowercase = 0
    count_digit = 0
    count_special = 0
    confirmation = True

    def __init__(self, name, value=None, **kwargs):
        # hint will be displayed with pwd1 widget
        hint = kwargs.pop('hint', None)
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.min_length = kwargs.get('min_length', 0)
        self.max_length = kwargs.get('max_length', 0)
        self.count_uppercase = kwargs.get('count_uppercase', 0)
        self.count_lowercase = kwargs.get('count_lowercase', 0)
        self.count_digit = kwargs.get('count_digit', 0)
        self.count_special = kwargs.get('count_special', 0)
        self.confirmation = kwargs.get('confirmation', True)
        confirmation_title = kwargs.get('confirmation_title') or _('Confirmation')
        self.strength_indicator = kwargs.get('strength_indicator', True)

        self.formats = kwargs.get('formats', ['sha1'])
        if not self.attrs.get('readonly'):
            self.add(PasswordWidget, name='pwd1', title='',
                    value='',
                    required=kwargs.get('required', False),
                    autocomplete='off',
                    hint=hint)
            if self.confirmation:
                self.add(PasswordWidget, name='pwd2', title=confirmation_title,
                        required=kwargs.get('required', False),
                        autocomplete='off')
        else:
            encoded_value = force_text(base64.encodebytes(force_bytes(json.dumps(value))))
            if value:
                fake_value = '*' * 8
            else:
                fake_value = ''
            self.add(HtmlWidget, 'hashed', title=htmltext(
                    '<input value="%s" readonly="readonly">'\
                    '<input type="hidden" name="%s$encoded" value="%s"></input>' % (
                        fake_value, self.name, encoded_value)))

    def add_media(self):
        get_response().add_javascript(['jquery.js', 'jquery.passstrength.js'])

    def render_content(self):
        if self.attrs.get('readonly') or not self.strength_indicator:
            return CompositeWidget.render_content(self)
        r = TemplateIO(html=True)
        r += CompositeWidget.render_content(self)
        r += htmltext('''<script>
$(function() {
  $('input[id="form_%(form_id)s"]').passStrengthify({
    levels: ["%(veryweak)s", "%(veryweak)s", "%(weak)s", "%(weak)s", "%(moderate)s", "%(good)s", "%(strong)s", "%(verystrong)s"],
    minimum: %(min_length)s,
    labels: {
      passwordStrength: "%(password_strength)s",
      tooShort: "%(tooshort)s"
    }
  });
});
</script>''') % {'form_id': self.get_widget('pwd1').get_name(),
                'min_length': self.min_length,
                'veryweak': _('Very weak'),
                'weak': _('Weak'),
                'moderate': _('Moderate'),
                'good': _('Good'),
                'strong': _('Strong'),
                'verystrong': _('Very strong'),
                'password_strength': _('Password strength:'),
                'tooshort': _('Too short')}
        return r.getvalue()

    def _parse(self, request):
        CompositeWidget._parse(self, request)
        if request.form.get('%s$encoded' % self.name):
            self.value = json_loads(base64.decodebytes(
                force_bytes(request.form.get('%s$encoded' % self.name))))
            return
        pwd1 = self.get('pwd1') or ''

        if not self.get_widget('pwd1'):
            # we are in read-only mode, stop here.
            return

        set_errors = []
        min_len = self.min_length
        if len(pwd1) < min_len:
            set_errors.append(_('Password is too short.  It must be at least %d characters.') % min_len)

        max_len = self.max_length
        if max_len and len(pwd1) > max_len:
            set_errors.append(_('Password is too long.  It must be at most %d characters.') % max_len)

        count = self.count_uppercase
        if len([x for x in pwd1 if x.isupper()]) < count:
            set_errors.append(
                ngettext('Password must contain an uppercase character.',
                    'Password must contain at least %(count)d uppercase characters.',
                    count) % {'count': count})

        count = self.count_lowercase
        if len([x for x in pwd1 if x.islower()]) < count:
            set_errors.append(
                ngettext('Password must contain a lowercase character.',
                    'Password must contain at least %(count)d lowercase characters.',
                    count) % {'count': count})

        count = self.count_digit
        if len([x for x in pwd1 if x.isdigit()]) < self.count_digit:
            set_errors.append(
                ngettext('Password must contain a digit.',
                    'Password must contain at least %(count)d digits.',
                    count) % {'count': count})

        count = self.count_special
        if len([x for x in pwd1 if not x.isalnum()]) < count:
            set_errors.append(
                ngettext('Password must contain a special character.',
                    'Password must contain at least %(count)d special characters.',
                    count) % {'count': count})

        if self.confirmation:
            pwd2 = self.get('pwd2') or ''
            if pwd1 != pwd2:
                self.get_widget('pwd2').set_error(_('Passwords do not match'))
                pwd1 = None

        if set_errors:
            self.get_widget('pwd1').set_error(' '.join(set_errors))
            pwd1 = None

        PASSWORD_FORMATS = {
            'cleartext': lambda x: force_str(x),
            'md5': lambda x: force_str(hashlib.md5(force_bytes(x)).hexdigest()),
            'sha1': lambda x: force_str(hashlib.sha1(force_bytes(x)).hexdigest()),
        }

        if pwd1:
            self.value = {}
            for fmt in self.formats:
                self.value[fmt] = PASSWORD_FORMATS[fmt](pwd1)
        else:
            self.value = None


class MapWidget(CompositeWidget):
    template_name = 'qommon/forms/widgets/map.html'

    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.add(HiddenWidget, 'latlng', value=value)
        widget = self.get_widget('latlng')
        self.readonly = kwargs.pop('readonly', False)
        self.map_attributes = {}
        self.map_attributes.update(get_publisher().get_map_attributes())
        self.sync_map_and_address_fields = get_publisher().has_site_option('sync-map-and-address-fields')
        for attribute in ('initial_zoom', 'min_zoom', 'max_zoom', 'init_with_geoloc'):
            if attribute in kwargs:
                self.map_attributes['data-' + attribute] = kwargs.pop(attribute)
        if kwargs.get('default_position'):
            self.map_attributes['data-def-lat'] = kwargs['default_position'].split(';')[0]
            self.map_attributes['data-def-lng'] = kwargs['default_position'].split(';')[1]

    def initial_position(self):
        if self.value:
            return {'lat': self.value.split(';')[0],
                    'lng': self.value.split(';')[1]}
        return None

    def add_media(self):
        get_response().add_javascript(['qommon.map.js'])

    def _parse(self, request):
        CompositeWidget._parse(self, request)
        self.value = self.get('latlng')
        if self.value:
            lat, lon = self.value.split(';')
            lat_lon = misc.normalize_geolocation({'lat': lat, 'lon': lon})
            self.value = '%s;%s' % (lat_lon['lat'], lat_lon['lon'])


class HiddenErrorWidget(HiddenWidget):
    def set_error(self, error):
        Widget.set_error(self, error)


class SingleSelectWidgetWithOther(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value=value, **kwargs)
        if 'title' in kwargs:
            del kwargs['title']
        options = kwargs.get('options')[:]
        if len(options[0]) == 1:
            options = [(x[0], x[0], x[0]) for x in options]
        elif len(options[0]) == 2:
            options = [(x[0], x[1], x[0]) for x in options]
        options.append(('__other', kwargs.pop('other_label', _('Other:')), '__other'))
        kwargs['options'] = options
        if value is None or value in [x[0] for x in options]:
            choice_value = value
            other_value = None
        else:
            choice_value = '__other'
            other_value = value
        self.add(SingleSelectWidget, 'choice', value=choice_value, **kwargs)
        self.add(StringWidget, 'other', value=other_value, size=35)

    def _parse(self, request):
        self.value = self.get('choice')
        if self.value == '__other':
            self.value = self.get('other')


class ComputedExpressionWidget(CompositeWidget):
    '''Widget that checks the entered value is a correct workflow
       expression.'''

    def __init__(self, name, value=None, *args, **kwargs):
        if not value:
            value = {}
        else:
            from wcs.workflows import WorkflowStatusItem
            value = WorkflowStatusItem.get_expression(value)

        value_placeholder = kwargs.pop('value_placeholder', None)
        CompositeWidget.__init__(self, name, value, **kwargs)

        options = [
            ('text', _('Text'), 'text'),
            ('template', _('Template'), 'template'),
            ('python', _('Python Expression'), 'python'),
        ]

        self.add(StringWidget, 'value_text', size=80,
                 value=value.get('value') if value.get('type') == 'text' else None,
                 placeholder=value_placeholder)

        self.add(StringWidget, 'value_template', size=80,
                 value=value.get('value') if value.get('type') == 'template' else None,
                 placeholder=value_placeholder)

        self.add(StringWidget, 'value_python', size=80,
                 value=value.get('value') if value.get('type') == 'python' else None,
                 placeholder=value_placeholder)

        self.add(SingleSelectWidget, 'type', options=options,
                 value=value.get('type'),
                 attrs={'data-dynamic-display-parent': 'true'})

    def render_content(self):
        return htmltext('''
           <style>
           span[data-name="%(name)s"].text::after { content: "%(text_label)s"; }
           span[data-name="%(name)s"].template::after { content: "%(template_label)s"; }
           span[data-name="%(name)s"].python::after { content: "%(python_label)s"; }
           </style>
           <span class="text"
                 data-name="%(name)s"
                 data-dynamic-display-value="text"
                 data-dynamic-display-child-of="%(name)s$type"
                 >%(value_text)s</span
          ><span class="template"
                 data-name="%(name)s"
                 data-dynamic-display-value="template"
                 data-dynamic-display-child-of="%(name)s$type"
                 >%(value_template)s</span
          ><span class="python"
                 data-name="%(name)s"
                 data-dynamic-display-value="python"
                 data-dynamic-display-child-of="%(name)s$type"
                 >%(value_python)s</span
          >%(type)s''') % {
                'name': self.name,
                'text_label': _('Text'),
                'template_label': _('Template'),
                'python_label': _('Python'),
                'value_text': self.get_widget('value_text').render_content(),
                'value_template': self.get_widget('value_template').render_content(),
                'value_python': self.get_widget('value_python').render_content(),
                'type': self.get_widget('type').render_content()
            }

    @classmethod
    def validate_template(cls, template):
        try:
            Template(template, raises=True)
        except TemplateError as e:
            raise ValidationError('%s' % e)

    @classmethod
    def validate(cls, expression):
        if not expression:
            return
        from wcs.workflows import WorkflowStatusItem
        expression = WorkflowStatusItem.get_expression(expression)
        if expression['type'] == 'python':
            if '{{' in expression['value']:
                raise ValidationError(_('invalid usage, Python expression cannot contain {{'))
            try:
                compile(expression['value'], '<string>', 'eval')
            except SyntaxError as e:
                raise ValidationError(_('syntax error in Python expression: %s') % e)
        else:
            cls.validate_template(expression['value'])

    def _parse(self, request):
        self.value = None
        if not self.get('type'):
            return
        value_type = self.get('type')
        value_content = self.get('value_%s' % value_type)
        if value_type == 'python' and value_content:
            self.value = '=' + (value_content or '')
        else:
            self.value = value_content
        if self.value:
            try:
                self.validate(self.value)
            except ValidationError as e:
                self.set_error(str(e))


class ConditionWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)

        if not value:
            value = {}

        options = []
        options.append(('django', _('Django Expression'), 'django'))
        if not kwargs.get('django_only'):
            options.append(('python', _('Python Expression'), 'python'))

        self.add(StringWidget, 'value_django', size=80,
                 value=value.get('value') if value.get('type') == 'django' else None)

        self.add(StringWidget, 'value_python', size=80,
                 value=value.get('value') if value.get('type') == 'python' else None)

        self.add(SingleSelectWidget, 'type', options=options,
                 value=value.get('type'),
                 attrs={'data-dynamic-display-parent': 'true'})

    @property
    def content_extra_attributes(self):
        validation_url = get_publisher().get_root_url() + 'api/validate-condition'
        return {'data-validation-url': validation_url}

    def _parse(self, request):
        self.value = None
        if not self.get('type') or self.get('type') == 'none':
            return

        self.value = {}
        self.value['type'] = self.get('type')

        if self.value['type']:
            self.value['value'] = self.get('value_%s' % self.value['type'])

        if self.value['value']:
            try:
                Condition(self.value).validate()
            except ValidationError as e:
                self.set_error(str(e))
        else:
            self.value = None

    def render_content(self):
        return htmltext('''
           <span class="django"
                 data-dynamic-display-value="django"
                 data-dynamic-display-child-of="%(name)s$type"
                 >%(value_django)s</span
          ><span class="python"
                 data-dynamic-display-value="python"
                 data-dynamic-display-child-of="%(name)s$type"
                 >%(value_python)s</span
          >%(type)s''') % {
                'name': self.name,
                'value_django': self.get_widget('value_django').render_content(),
                'value_python': self.get_widget('value_python').render_content(),
                'type': self.get_widget('type').render_content()
            }


class DjangoConditionWidget(StringWidget):
    def _parse(self, request):
        super(DjangoConditionWidget, self)._parse(request)
        if self.value:
            try:
                Condition({'type': 'django', 'value': self.value}).validate()
            except ValidationError as e:
                self.set_error(str(e))