wcs/wcs/qommon/form.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2010  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import base64
import cStringIO
import fnmatch
import mimetypes
import os
import re
import socket
import tempfile
import time
import random
import datetime
import itertools
import hashlib
import json

try:
    from PIL import Image
except ImportError:
    Image = None

from storage import atomic_write

try:
    from feedparser import _sanitizeHTML
except ImportError:
    _sanitizeHTML = None

try:
    import DNS
    DNS.ParseResolvConf()
except ImportError:
    DNS = None

try:
    import magic
except ImportError:
    magic = None

import quixote

from quixote import get_publisher, get_request, get_response, get_session
from quixote.http_request import Upload
from quixote.form import *
from quixote.html import htmltext, htmltag, htmlescape, TemplateIO
from quixote.util import randbytes

import misc
from strftime import strftime
from publisher import get_cfg
from wcs import file_validation
from . import ezt

QuixoteForm = Form

Widget.REQUIRED_ERROR = N_('required field')
get_error_orig = Widget.get_error

def get_i18n_error(self, request=None):
    error = get_error_orig(self, request)
    if error == Widget.REQUIRED_ERROR:
        return _(error)
    return error

def set_message(self, message):
    self.message = message

def get_message(self):
    if hasattr(self, 'message'):
        return self.message
    else:
        return ''

def render_message(self, message):
    if message:
        return htmltext('<div class="message">%s</div>') % message
    else:
        return ''

def is_prefilled(self):
    if hasattr(self, 'prefilled'):
        return self.prefilled
    else:
        return False

def render_title(self, title):
    if title:
        if self.required:
            title += htmltext('<span class="required">*</span>')
        return htmltext('<div class="title"><label for="form_%s">%s</label></div>') % (
                        self.name, title)
    else:
        return ''


def render(self):
    # quixote/form/widget.py, Widget::render
    r = TemplateIO(html=True)
    classnames = '%s widget' % self.__class__.__name__
    if hasattr(self, 'extra_css_class') and self.extra_css_class:
        classnames += ' ' + self.extra_css_class
    if self.get_error():
        classnames += ' widget-with-error'
    if self.get_message():
        classnames += ' widget-with-message'
    if self.is_required():
        classnames += ' widget-required'
    if self.is_prefilled():
        classnames += ' widget-prefilled'
    attributes = {}
    if hasattr(self, 'div_id') and self.div_id:
        attributes['data-valuecontainerid'] = 'form_%s' % self.name
    if hasattr(self, 'prefill_attributes') and self.prefill_attributes:
        for k, v in self.prefill_attributes.items():
            attributes['data-' + k] = v
    if hasattr(self, 'div_id') and self.div_id:
        attributes['id'] = self.div_id
    for attr in ('data-dynamic-display-child-of', 'data-dynamic-display-value'):
        if attr in self.attrs:
            attributes[attr] = self.attrs.pop(attr)
    attributes['class'] = classnames
    r += htmltext('<div %s>' % ' '.join(['%s="%s"' % x for x in attributes.items()]))
    r += self.render_title(self.get_title())
    classnames = 'content'
    if hasattr(self, 'content_extra_css_class') and self.content_extra_css_class:
        classnames += ' ' + self.content_extra_css_class
    r += htmltext('<div class="%s">' % classnames)
    r += self.render_error(self.get_error())
    r += self.render_content()
    r += self.render_hint(self.get_hint())
    r += self.render_message(self.get_message())
    r += htmltext('</div>')
    r += htmltext('</div>')
    if self.render_br:
        r += htmltext('<br class="%s" />') % classnames
    r += htmltext('\n')
    return r.getvalue()

Widget.get_error = get_i18n_error
Widget.render = render
Widget.cleanup = None
Widget.set_message = set_message
Widget.get_message = get_message
Widget.render_message = render_message
Widget.render_title = render_title
Widget.is_prefilled = is_prefilled

def string_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    return htmltag("input", xml_end=True, type=self.HTML_TYPE, name=self.name,
                   value=self.value, **attrs)
StringWidget.render_content = string_render_content


def file_render_content(self):
    attrs = {'id': 'form_' + str(self.name).split('$')[0]}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    return htmltag("input", xml_end=True, type=self.HTML_TYPE, name=self.name,
                   value=self.value, **attrs)
FileWidget.render_content = file_render_content


def text_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    return (htmltag("textarea", name=self.name, **attrs) +
        htmlescape(self.value or "") + htmltext("</textarea>"))
TextWidget.render_content = text_render_content


def submit_render_content(self):
    if self.name in ('cancel', 'previous', 'save-draft'):
        self.attrs['formnovalidate'] = 'formnovalidate'
    return super(self.__class__, self).render_content()
SubmitWidget.render_content = submit_render_content


def radiobuttons_render_content(self):
    tags = []
    for object, description, key in self.options:
        if self.is_selected(object):
            checked = 'checked'
        else:
            checked = None
        r = htmltag("input", xml_end=True,
                    type="radio",
                    name=self.name,
                    value=key,
                    checked=checked,
                    **self.attrs)
        tags.append(htmltext('<label>') + r + htmlescape(description) + htmltext('</label>'))
    return htmlescape(self.delim).join(tags)
RadiobuttonsWidget.render_content = radiobuttons_render_content


def checkbox_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    return htmltag("input", xml_end=True, type="checkbox", name=self.name,
                   value="yes", checked=self.value and "checked" or None,
                   **attrs)
CheckboxWidget.render_content = checkbox_render_content


def select_render_content(self):
    attrs = {'id': 'form_' + self.name}
    if self.required:
        attrs['aria-required'] = 'true'
    if self.attrs:
        attrs.update(self.attrs)
    tags = [htmltag("select", name=self.name, **attrs)]
    for object, description, key in self.options:
        if self.is_selected(object):
            selected = 'selected'
        else:
            selected = None
        if description is None:
            description = ""
        r = htmltag("option", value=key, selected=selected)
        tags.append(r + htmlescape(description) + htmltext('</option>'))
    tags.append(htmltext("</select>"))
    return htmltext("\n").join(tags)
SelectWidget.render_content = select_render_content


class Form(QuixoteForm):
    TOKEN_NOTICE = N_(
            "The form you have submitted is invalid.  Most "
            "likely it has been successfully submitted once "
            "already.  Please review the form data "
            "and submit the form again.")
    ERROR_NOTICE = N_("There were errors processing your form.  See below for details.")

    info = None
    captcha = None
    advanced_label = '+'

    def __init__(self, *args, **kwargs):
        if kwargs.get('advanced_label'):
            self.advanced_label = kwargs.pop('advanced_label')
        QuixoteForm.__init__(self, *args, **kwargs)
        self.global_error_messages = None

    def keep_referer(self):
        self.add(HiddenWidget, '__keep_referer',
                value = get_request().environ.get('HTTP_REFERER'))

    def referer(self):
        if self.get_widget('__keep_referer'):
            return self.get_widget('__keep_referer').parse()

    def add_captcha(self, hint=None):
        if not self.captcha and not (get_session().won_captcha or get_session().user):
            self.captcha = CaptchaWidget('captcha', hint=hint)

    def add(self, widget_class, name, *args, **kwargs):
        if kwargs and not kwargs.has_key('render_br'):
            kwargs['render_br'] = False
        advanced = False
        if kwargs and kwargs.get('advanced', False):
            advanced = True
            del kwargs['advanced']
        QuixoteForm.add(self, widget_class, name, *args, **kwargs)
        widget = self._names[name]
        widget.advanced = advanced
        return widget

    def remove(self, name):
        widget = self._names.get(name)
        del self._names[name]
        self.widgets.remove(widget)

    def get_all_widgets(self):
        l = QuixoteForm.get_all_widgets(self)
        if self.captcha:
            l.append(self.captcha)
        return l

    def add_global_errors(self, error_messages):
        self.global_error_messages = error_messages

    def _get_default_action(self):
        if get_request().get_header('x-popup') == 'true':
            # do not leave action empty for popups, as they get embedded into
            # another URI
            return get_request().get_path()
        return QuixoteForm._get_default_action(self)

    def render_button(self, button):
        r = TemplateIO(html=True)
        classnames = '%s widget %s-button' % (
                button.__class__.__name__, button.name)
        r += htmltext('<div class="%s">') % classnames
        r += htmltext('<div class="content">')
        r += button.render_content()
        r += htmltext('</div>')
        r += htmltext('</div>')
        r += htmltext('\n')
        return r.getvalue()

    def _render_submit_widgets(self):
        r = TemplateIO(html=True)
        if self.submit_widgets:
            r += htmltext('<div class="buttons submit">')
            for widget in self.submit_widgets:
                if widget.name == 'prefill':
                    continue
                r += self.render_button(widget)
            r += htmltext('</div>')
        return r.getvalue()

    def _render_info_notice(self):
        return htmltext('<div class="infonotice">%s</div>' % _(self.info))

    def _render_error_notice(self):
        errors = []
        if self.has_errors():
            errors.append(_(QuixoteForm._render_error_notice(self)))
        if self.global_error_messages:
            errors.extend(self.global_error_messages)
        t = TemplateIO(html=True)
        t += htmltext('<div class="errornotice">')
        for error in errors:
            t += htmltext('<p>%s</p>') % error
        t += htmltext('</div>')
        return t.getvalue()

    def _render_body(self):
        r = TemplateIO(html=True)
        if self.has_errors() or self.global_error_messages:
            r += self._render_error_notice()
        if self.info:
            r += self._render_info_notice()
        if self._names.has_key('prefill'):
            r += self._render_prefill_widgets()

        r += self._render_widgets()
        if self.captcha:
            r += self.captcha.render()
        r += self._render_submit_widgets()
        return r.getvalue()

    def _render_prefill_widgets(self):
        r = TemplateIO(html=True)
        if self.submit_widgets:
            r += htmltext('<div class="prefill buttons">')
            for widget in self.submit_widgets:
                if widget.name == 'prefill':
                    r += widget.render()
            r += htmltext('</div>')
        return r.getvalue()

    def _render_widgets(self):
        r = TemplateIO(html=True)
        advanced_widgets = []
        for widget in self.widgets:
            if not hasattr(widget, 'advanced') or not widget.advanced:
                r += widget.render()
            else:
                advanced_widgets.append(widget)
        if advanced_widgets:
            get_response().add_javascript(['jquery.js', 'qommon.forms.js'])
            r += htmltext('<fieldset class="form-plus">')
            r += htmltext('<legend>%s</legend>') % self.advanced_label
            for widget in advanced_widgets:
                r += widget.render()
            r += htmltext('</fieldset>')
        return r.getvalue()


class HtmlWidget(object):
    error = None

    def __init__(self, string, title = None, *args, **kwargs):
        self.string = string
        self.title = title

    def render(self):
        return self.render_content()

    def render_content(self):
        if self.title:
            return htmltext(self.title)
        return htmltext(self.string)

    def has_error(self, request):
        return False

    def parse(self, *args):
        pass

    def clear_error(self, request=None):
        pass

class CompositeWidget(quixote.form.CompositeWidget):
    def render_as_thead(self):
        r = TemplateIO(html=True)
        r += htmltext('<tr>\n')
        for widget in self.get_widgets():
            r += htmltext('<th>')
            r += widget.get_title()
            r += htmltext('</th>')
        r += htmltext('</tr>\n')
        return r.getvalue()

    def render_content_as_tr(self):
        r = TemplateIO(html=True)
        r += htmltext('<tr>\n')
        for widget in self.get_widgets():
            classnames = '%s widget' % widget.__class__.__name__
            if hasattr(self, 'extra_css_class') and widget.extra_css_class:
                classnames += ' ' + widget.extra_css_class
            r += htmltext('<td><div class="%s"><div class="content">' % classnames)
            r += widget.render_content()
            r += htmltext('</div></div></td>')
        r += htmltext('</tr>\n')
        return r.getvalue()


class StringWidget(quixote.form.StringWidget):
    def __init__(self, name, *args, **kwargs):
        self.validation_function = kwargs.pop('validation_function', None)
        super(StringWidget, self).__init__(name, *args, **kwargs)

    def _parse(self, request):
        quixote.form.StringWidget._parse(self, request)
        if self.value and self.validation_function:
            try:
                self.validation_function(self.value)
            except ValueError as e:
                self.error = str(e)


class TextWidget(quixote.form.TextWidget):
    def __init__(self, name, *args, **kwargs):
        self.validation_function = kwargs.pop('validation_function', None)
        super(TextWidget, self).__init__(name, *args, **kwargs)

    def _parse(self, request):
        quixote.form.TextWidget._parse(self, request)
        if self.value is not None:
            try:
                maxlength = int(self.attrs.get('maxlength', 0))
            except (TypeError, ValueError):
                maxlength = 0
            if maxlength and len(self.value) > maxlength:
                self.error = _('too many characters (limit is %d)') % maxlength
            if self.validation_function:
                try:
                    self.validation_function(self.value)
                except ValueError as e:
                    self.error = str(e)

class CheckboxWidget(quixote.form.CheckboxWidget):
    '''
    Widget just like CheckboxWidget but with an effective support for the
    required attribute, if required the checkbox will have to be checked.
    '''
    def _parse(self, request):
        self.value = self.name in request.form and not request.form[self.name] in (False, '', 'False')
        if self.required and not self.value:
            self.set_error(self.REQUIRED_ERROR)

    def set_value(self, value):
        if value in (None, False, '', 'False'):
            self.value = False
        else:
            self.value = True

class UploadedFile: #pylint: disable=C1001
    def __init__(self, directory, filename, upload):
        self.directory = directory
        self.base_filename = upload.base_filename
        self.content_type = upload.content_type
        content = upload.fp.read()
        self.size = len(content)
        # Find a good filename
        if filename:
            self.filename = filename
        elif self.base_filename:
            self.filename = self.base_filename
        else:
            t = datetime.datetime.now().isoformat()
            fd, path = tempfile.mkstemp(prefix=t, suffix='.upload',
                    dir=self.dir_path())
            os.close(fd)
            self.filename = os.path.basename(filename)
        file_path = self.build_file_path()
        if not os.path.exists(self.dir_path()):
            os.mkdir(self.dir_path())
        file(file_path, 'w').write(content)

    def dir_path(self):
        return os.path.join(get_publisher().app_dir, self.directory)

    def build_file_path(self):
        return os.path.join(get_publisher().app_dir, self.directory,
                self.filename)

    def get_file(self):
        return file(self.build_file_path())

    def build_response(self):
        response = get_response()
        response.content_type = self.content_type
        response.set_header('content-disposition',
                'attachment; filename="%s"' % self.base_filename)
        return self.get_file().read()


class UploadValidationError(Exception):
    pass


class UploadWidget(CompositeWidget):
    def __init__(self, name, value=None, directory=None, filename=None,
            validation=None, **kwargs):
        CompositeWidget.__init__(self, name, **kwargs)
        del kwargs['title']
        kwargs.pop('hint')
        self.value = value
        self.directory = directory or 'uploads'
        self.filename = filename
        self.validation = validation
        if value:
            self.add(CheckboxWidget, 'orexisting',
                            title = _('Use previous file.'), value=True)
            self.widgets[-1].render_br = False
            self.add(FileWidget, 'file', title = _('Or upload a new one'), **kwargs)
        else:
            self.add(FileWidget, 'file', **kwargs)

    def _parse(self, request):
        if self.value and self.get('orexisting'):
            pass
        elif self.get('file'):
            upload = self.get('file')
            self.value = UploadedFile(self.directory, self.filename, upload)
            if self.validation:
                try:
                    self.validation(upload)
                except UploadValidationError, e:
                    self.error = str(e)
        else:
            self.value = None


class NoUpload(object):
    no_file = True
    metadata = None

    def __init__(self, validation_url):
        self.metadata = file_validation.get_validation(validation_url)


class FileWithPreviewWidget(CompositeWidget):
    """Widget that proposes a File Upload widget but that stores the file
    ondisk so it has a "readonly" mode where the filename is shown."""

    extra_css_class = 'file-upload-widget'
    max_file_size = None
    file_type = None

    max_file_size_bytes = None  # will be filled automatically

    def __init__(self, name, value=None, **kwargs):
        from wcs import fields

        CompositeWidget.__init__(self, name, value, **kwargs)
        self.value = value
        self.document_type = kwargs.pop('document_type', None) or {}
        self.readonly = kwargs.get('readonly')
        self.max_file_size = kwargs.pop('max_file_size', None)
        self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True)
        if self.max_file_size:
            self.max_file_size_bytes = FileSizeWidget.parse_file_size(self.max_file_size)
        self.add(HiddenWidget, 'token')
        if not self.readonly:
            hint = ''
            if self.allow_portfolio_picking:
                root_url = get_publisher().get_root_url()
                if (file_validation.has_file_validation()
                        and get_request().user
                        and not self.readonly):
                    get_response().add_javascript(['fargo.js'])
                    params = (root_url,
                              _('Pick a file from the portfolio'),
                              _('Use file from the portfolio'))
                    hint += htmltext('<p class="use-file-from-fargo"><span '
                                     'data-src="%sfargo/pick" '
                                     'data-width="500" '
                                     'data-height="400" '
                                     'data-title="%s" '
                                     'rel="popup">%s</span></p>' % params)
            hint += self.hint or ''
            attrs = {'data-url': get_publisher().get_root_url() + 'tmp-upload'}
            self.file_type = kwargs.pop('file_type', None)
            if self.file_type:
                attrs['accept'] = ','.join(self.file_type)
            if self.max_file_size_bytes:
                # this could be used for client size validation of file size
                attrs['data-max-file-size'] = str(self.max_file_size_bytes)
            self.add(FileWidget, 'file', hint=hint, render_br=False, attrs=attrs)
        if self.document_type.get('metadata'):
            if self.readonly:
                self.add(HiddenWidget, 'validation_url')
            else:
                validations = file_validation.get_validations(self.document_type)
                if validations:
                    options = [('', _('Known documents'), '')]
                    options += [(v['url'], v['display'], v['url']) for v in validations]
                    for validation in validations:
                        for i, meta_field in enumerate(self.metadata):
                            # translate varname to f<self.id>$f<subwidget.id>
                            if meta_field['varname'] in validation:
                                value = validation.pop(meta_field['varname'])
                                validation['f%s' % i] = value
                    self.add(SingleSelectWidget, 'validation_url', options=options,
                             attrs={'data-validations': json.dumps(validations)})
            for i, meta_field in enumerate(self.metadata):
                field = fields.get_field_class_by_type(meta_field.get('type', 'string'))()
                field.id = i
                field.init_with_json(meta_field, include_id=False)
                if meta_field.get('varname'):
                    subvalue = getattr(value, 'metadata', {}).get(meta_field['varname'])
                else:
                    subvalue = None
                # required only if composite field is required
                field.required = field.required and self.required
                field.extra_css_class = 'subwidget'
                if self.readonly:
                    # preview
                    field.add_to_view_form(self, subvalue)
                else:
                    field.add_to_form(self, subvalue)
        if value:
            self.set_value(value)

    def set_value(self, value):
        self.value = value
        if hasattr(self.value, 'token'):
            self.get_widget('token').set_value(self.value.token)
            if not get_session().get_tempfile(self.value.token):
                # oops, it has a token but it's not in the session; this is
                # probably because it was restored from a draft file created
                # from an expired session.
                self.value.token = get_session().add_tempfile(self.value)
                self.get_widget('token').set_value(self.value.token)

    def render_content(self):
        get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'jquery.iframe-transport.js',
                                       'jquery.fileupload.js', 'qommon.fileupload.js'])

        temp = get_session().get_tempfile(self.get('token')) or {}

        r = TemplateIO(html=True)
        if self.get_widget('file'):
            r += self.get_widget('file').render()

        r += htmltext('<div class="fileprogress" style="display: none;">')
        r += htmltext(' <div class="bar">%s</div>' % _('Upload in progress...'))
        r += htmltext('</div>')
        r += htmltext('<div class="fileinfo"><span class="filename">%s</span>'
                      % temp.get('base_filename', ''))
        if not self.readonly:
            r += htmltext(' <a href="#" class="remove" title="%s">%s</a>'
                          % (_('Remove this file'), _('remove')))
        elif temp:
            filetype = mimetypes.guess_type(temp.get('orig_filename', ''))
            include_image = False
            if filetype and filetype[0] and filetype[0].startswith('image'):
                include_image = True
            if Image:
                image_content = get_session().get_tempfile_content(self.get('token'))
                try:
                    image = Image.open(image_content.fp)
                except Exception:
                    include_image = False
            if include_image:
                r += htmltext('<img alt="" src="tempfile?t=%s&thumbnail=1" />' %
                              self.get('token'))
        r += htmltext('</div>')

        for widget in self.get_widgets():
            if widget is self.get_widget('file'):
                continue
            r += widget.render()

        return r.getvalue()

    @property
    def metadata(self):
        return self.document_type.get('metadata', [])

    def _parse(self, request):
        self.value = None
        if self.get('token'):
            token = self.get('token')
        elif self.get('validation_url'):
            self.value = NoUpload(self.get('validation_url'))
            return
        elif self.get('file'):
            token = get_session().add_tempfile(self.get('file'))
            request.form[self.get_widget('token').get_name()] = token
        else:
            token = None

        session = get_session()
        if token and session.tempfiles and session.tempfiles.has_key(token):
            self.value = session.get_tempfile_content(token)

        if self.value is None:
            # there's no file, check all metadata field are empty too
            # if not file and required metadata field become required
            if (self.get_widget('file')
                    and not self.required
                    and any([self.get('f%s' % i) for i, meta_field in enumerate(self.metadata)])):
                self.get_widget('file').required = True
                self.get_widget('file').set_error(self.REQUIRED_ERROR)
                for i, meta_field in enumerate(self.metadata):
                    name = 'f%s' % i
                    required = meta_field.get('required', True)
                    if required:
                        widget = self.get_widget(name)
                        widget.required = True
                        if not self.get(name):
                            widget.set_error(self.REQUIRED_ERROR)
            return

        # There is some file, check all required metadata files have been filled
        for i, meta_field in enumerate(self.metadata):
            name = 'f%s' % i
            required = meta_field.get('required', True)
            if required:
                widget = self.get_widget(name)
                widget.required = True
                if not self.get(name):
                    widget.set_error(self.REQUIRED_ERROR)

        if self.metadata:
            self.value.metadata = {}
            for i, meta_field in enumerate(self.metadata):
                name = 'f%s' % i
                if 'varname' in meta_field:
                    self.value.metadata[meta_field['varname']] = self.get(name)

        # Don't trust the browser supplied MIME type, update the Upload object
        # with a MIME type created with magic (or based on the extension if the
        # module is missing).
        #
        # This also helps people uploading PDF files that were downloaded from
        # sites setting a wrong MIME type (like application/force-download) for
        # various reasons.
        if magic:
            magic_object = magic.open(magic.MIME)
            magic_object.load()
            filetype = magic_object.file(self.value.fp.name).split(';')[0]
            magic_object.close()
        else:
            filetype, encoding = mimetypes.guess_type(self.value.base_filename)

        if not filetype:
            filetype = 'application/octet-stream'

        self.value.content_type = filetype

        if self.max_file_size:
            # validate file size
            file_size = os.path.getsize(self.value.fp.name)
            if file_size > self.max_file_size_bytes:
                self.error = _('over file size limit (%s)') % self.max_file_size

        if self.file_type:
            # validate file type
            accepted_file_types = []
            for file_type in self.file_type:
                accepted_file_types.extend(file_type.split(','))

            valid_file_type = False
            for accepted_file_type in accepted_file_types:
                # fnmatch is used to handle generic mimetypes, like
                # image/*
                if fnmatch.fnmatch(self.value.content_type, accepted_file_type):
                    valid_file_type = True
                    break
            if not valid_file_type:
                self.error = _('invalid file type')

    def render_hint(self, hint):
        return ''


class PicklableUpload(Upload):
    def __getstate__(self):
        odict = self.__dict__.copy()
        if 'fp' in odict:
            del odict['fp']

        basedir = os.path.join(get_publisher().app_dir, 'uploads')
        if not os.path.exists(basedir):
            os.mkdir(basedir)
        if 'qfilename' in odict:
            filepath = os.path.join(basedir, self.qfilename)
        else:
            while True:
                filename = str(random.random())
                filepath = os.path.join(basedir, filename)
                if not os.path.exists(filepath):
                    break
            self.qfilename = filename

        if 'fp' in self.__dict__:
            self.fp.seek(0)
            atomic_write(filepath, self.fp, async=False)

        odict['qfilename'] = self.qfilename
        return odict

    def get_file_pointer(self):
        if 'fp' in self.__dict__ and self.__dict__.get('fp') is not None:
            return self.__dict__.get('fp')
        elif hasattr(self, 'qfilename'):
            basedir = os.path.join(get_publisher().app_dir, 'uploads')
            self.fp = file(os.path.join(basedir, self.qfilename))
            return self.fp
        return None

    def __setstate__(self, dict):
        self.__dict__.update(dict)
        if hasattr(self, 'data'):
            # backward compatibility with older w.c.s. version
            self.fp = cStringIO.StringIO(self.data)
            del self.data

    def get_file(self):
        # quack like UploadedFile
        return self.get_file_pointer()

    def get_content(self):
        if hasattr(self, 'qfilename'):
            filename = os.path.join(get_publisher().app_dir, 'uploads', self.qfilename)
            return file(filename).read()
        return None


class EmailWidget(StringWidget):
    HTML_TYPE = 'email'

    def __init__(self, *args, **kwargs):
        StringWidget.__init__(self, *args, **kwargs)
        if not 'size' in kwargs:
            self.attrs['size'] = '35'

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            # basic tests first
            if not '@' in self.value[1:-1]:
                self.error = _('must be a valid email address')
            elif self.value[0] != '"' and ' ' in self.value:
                self.error = _('must be a valid email address')
            elif self.value[0] != '"' and self.value.count('@') != 1:
                self.error = _('must be a valid email address')
            elif get_cfg('emails', {}).get('check_domain_with_dns', True):
                # testing for domain existence
                domain = self.value.split('@')[-1]
                if [x for x in domain.split('.') if not x]:
                    # empty parts in domain, ex: @example..net, or
                    # @.example.net
                    self.error = _('invalid address domain')
                    return
                if not type(domain) is unicode:
                    domain = unicode(domain, 'utf-8', 'ignore')
                domain = domain.encode('idna')
                # simply lookup host name; note it will fail if hostname
                # doesn't have an A entry
                try:
                    socket.gethostbyname(domain)
                except socket.error:
                    # will fail on lack of DNS module or MX lookup failure
                    try:
                        l = len(DNS.mxlookup(domain))
                    except:
                        l = 0
                    if l == 0:
                        self.error = _('invalid address domain')
                else:
                    # and then, localpart could be tested
                    pass


class WcsExtraStringWidget(StringWidget):
    field = None
    prefill = False

    def render_content(self):
        if self.field and self.field.validation and not 'pattern' in self.attrs:
            self.attrs['pattern'] = self.field.validation
        s = StringWidget.render_content(self)
        return s

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.field and self.field.validation and self.value is not None:
            if not re.match(self.field.validation, self.value):
                self.error = _('wrong format')


class DateWidget(StringWidget):
    '''StringWidget which checks the value entered is a correct date'''
    minimum_date = None
    maximum_date = None
    content_extra_css_class = 'date'

    def __init__(self, name, value=None, **kwargs):
        self.minimum_date = None

        if kwargs.get('minimum_date'):
            self.minimum_date = misc.get_as_datetime(kwargs.get('minimum_date')).timetuple()[:3]
            del kwargs['minimum_date']
        if kwargs.get('maximum_date'):
            self.maximum_date = misc.get_as_datetime(kwargs.get('maximum_date')).timetuple()[:3]
            del kwargs['maximum_date']
        if kwargs.get('minimum_is_future'):
            if kwargs.get('date_can_be_today'):
                self.minimum_date = time.localtime()
            else:
                self.minimum_date = (datetime.datetime.today() + datetime.timedelta(1)).timetuple()
        if kwargs.get('date_in_the_past'):
            if kwargs.get('date_can_be_today'):
                self.maximum_date = time.localtime()
            else:
                self.maximum_date = (datetime.datetime.today() - datetime.timedelta(1)).timetuple()

        if 'minimum_is_future' in kwargs:
            del kwargs['minimum_is_future']
        if 'date_in_the_past' in kwargs:
            del kwargs['date_in_the_past']
        if 'date_can_be_today' in kwargs:
            del kwargs['date_can_be_today']

        if isinstance(value, datetime.datetime) or isinstance(value, datetime.date):
            value = value.strftime(misc.date_format())

        StringWidget.__init__(self, name, value=value, **kwargs)
        self.attrs['size'] = '12'
        self.attrs['maxlength'] = '10'

    def parse(self, request=None):
        StringWidget.parse(self, request=request)
        if type(self.value) is unicode:
            self.value = self.value.encode(get_publisher().site_charset)
        return self.value

    @classmethod
    def get_format_string(cls):
        return misc.date_format()

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            for format_string in (self.get_format_string(),
                    misc.date_format(), '%Y-%m-%d'):
                try:
                    value = time.strptime(self.value, format_string)
                except ValueError:
                    continue
                self.value = strftime(self.get_format_string(), value)
                break
            else:
                self.error = _('invalid date')
                self.value = None
                return
            if value[0] < 1800 or value[0] > 2099:
                self.error = _('invalid date')
                self.value = None
            elif self.minimum_date and value[:3] < self.minimum_date[:3]:
                self.error = _('invalid date: date must be on or after %s') % strftime(
                        misc.date_format(), datetime.datetime(*self.minimum_date[:6]))
            elif self.maximum_date and value[:3] > self.maximum_date[:3]:
                format_string = misc.date_format()
                self.error = _('invalid date; date must be on or before %s') % strftime(
                        misc.date_format(), datetime.datetime(*self.maximum_date[:6]))

    @classmethod
    def prepare_javascript(cls):
        get_response().add_javascript([
            'jquery.js', 'bootstrap-datetimepicker.js', 'qommon.forms.js'])
        current_language = get_request().language
        if current_language != 'en' and current_language in get_publisher().supported_languages or []:
            get_response().add_javascript(['bootstrap-datetimepicker.%s.js' % current_language])

        get_response().add_css_include('datetimepicker.css')

    def render_content(self):
        self.attrs['id'] = 'form_%s' % self.name
        self.attrs['class'] = 'date-pick'

        if self.attrs.get('readonly'):
            return StringWidget.render_content(self)

        self.prepare_javascript()

        date_format = self.get_format_string().replace('%Y', 'yyyy').replace(
                '%m', 'mm').replace('%d', 'dd').replace('%H', 'hh').replace(
                '%M',  'ii').replace('%S', 'ss')

        self.attrs['data-date-format'] = date_format
        self.attrs['data-min-view'] = '0'
        self.attrs['data-start-view'] = '2'
        if not 'hh' in date_format:
            # if the date format doesn't contain the time, set widget not to go
            # into the time pages
            self.attrs['data-min-view'] = '2'
        if not self.value:
            # if there's no value we set the initial view to be the view of
            # decades, it's more appropriate to select a far away date.
            self.attrs['data-start-view'] = '4'

        if self.minimum_date:
            start_date = date_format.replace(
                    'yyyy', '%04d' % self.minimum_date[0]).replace(
                    'mm', '%02d' % self.minimum_date[1]).replace(
                    'dd', '%02d' % self.minimum_date[2]).replace(
                    'hh', '00').replace('ii', '00').replace('ss', '00')
            self.attrs['data-start-date'] = start_date

        if self.maximum_date:
            end_date = date_format.replace(
                    'yyyy', '%04d' % self.maximum_date[0]).replace(
                    'mm', '%02d' % self.maximum_date[1]).replace(
                    'dd', '%02d' % self.maximum_date[2]).replace(
                    'hh', '00').replace('ii', '00').replace('ss', '00')
            self.attrs['data-end-date'] = end_date

        return StringWidget.render_content(self)


class DateTimeWidget(DateWidget):
    '''StringWidget which checks the value entered is a correct date/time'''
    content_extra_css_class = 'date'

    def __init__(self, name, value=None, **kwargs):
        DateWidget.__init__(self, name, value=value, **kwargs)
        self.attrs['size'] = '16'
        self.attrs['maxlength'] = '16'

    def get_format_string(self):
        return misc.datetime_format()


class RegexStringWidget(StringWidget):
    '''StringWidget which checks the value entered is a correct regex'''
    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            try:
                re.compile(self.value)
            except:
                self.error = _('invalid regular expression')
                self.value = None


class CheckboxesWidget(CompositeWidget):
    readonly = False

    def __init__(self, name, value = None, elements = None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.element_names = {}

        if kwargs.has_key('title'):
            del kwargs['title']
        if kwargs.has_key('readonly'):
            del kwargs['readonly']
            self.readonly = True
        self.inline = kwargs.get('inline', True)
        self.max_choices = int(kwargs.get('max_choices', 0) or 0)

        for v in elements:
            if type(v) is tuple:
                title = v[1]
                key = v[0]
                if type(key) is int:
                    name = 'element%d' % v[0]
                elif type(key) in (str, htmltext):
                    name = str('element%s' % v[0])
                    key = str(key)
                else:
                    raise NotImplementedError()
            else:
                title = v
                key = v
                name = 'element%d' % len(self.element_names.keys())

            if value and key in value:
                checked = True
            else:
                checked = False
            self.add(CheckboxWidget, name, title = title, value = checked, **kwargs)
            self.element_names[name] = key

    def _parse(self, request):
        if self.readonly:
            return
        values = []
        for name in self.element_names:
            value = self.get(name)
            if value is True:
                values.append(self.element_names[name])
        self.value = values
        if self.required and not self.value:
            self.set_error(self.REQUIRED_ERROR)
        if self.value and self.max_choices and len(self.value) > self.max_choices:
            self.set_error(_('You must select at most %d answers.') % self.max_choices)

    def set_value(self, value):
        self.value = value
        for name in self.element_names:
            widget = self.get_widget(name)
            if self.element_names[name] in self.value:
                widget.set_value(True)
            else:
                widget.set_value(False)

    def has_error(self, request=None):
        return Widget.has_error(self, request=request)

    def render_content(self):
        r = TemplateIO(html=True)
        if self.inline:
            r += htmltext('<ul class="inline">')
        else:
            r += htmltext('<ul>')
        for widget in self.get_widgets():
            r += htmltext('<li><label>')
            if self.readonly:
                widget.attrs['disabled'] = 'disabled'
                if widget.value:
                    r += htmltext('<input type="hidden" name="%s" value="yes" >') % widget.name
                widget.name = widget.name + 'xx'
            r += widget.render_content()
            r += widget.title
            r += htmltext('</label>')
            r += htmltext('</li>')
        r += htmltext('</ul>')
        return r.getvalue()

class ValidatedStringWidget(StringWidget):
    '''StringWidget which checks the value entered is correct according to a regex'''
    regex = None

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.regex and self.value is not None:
            match = re.match(self.regex, self.value)
            if not match or not match.group() == self.value:
                self.error = _('wrong format')

class UrlWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a correct url starting with http or https'''
    regex = r'^https?://.+'

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('must start with http:// or https:// and have a domain name')


class VarnameWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a syntactically correct
       variable name.'''
    regex = r'^[a-zA-Z][a-zA-Z0-9_]*'

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('must only consist of letters, numbers, or underscore')


class FileSizeWidget(ValidatedStringWidget):
    '''StringWidget which checks the value entered is a syntactically correct
       file size.'''
    regex = r'^\s*([\d]+)\s*([MK]i?)?[oB]?\s*$'

    @classmethod
    def parse_file_size(cls, value):
        try:
            value, unit = re.search(cls.regex, value).groups()
        except AttributeError: #  None has no .groups()
            raise ValueError()
        coeffs = {
            'Mi': 2**20,
            'Ki': 2**10,
            'M':  10**6,
            'K':  10**3,
            None: 1,
        }
        return int(value) * coeffs.get(unit)

    def _parse(self, request):
        ValidatedStringWidget._parse(self, request)
        if self.error:
            self.error = _('invalid file size')


class CaptchaWidget(CompositeWidget):
    def __init__(self, name, value = None, mode = 'arithmetic-simple', *args, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.render_br = False
        if value:
            token = value
        else:
            token = get_session().create_captcha_token()
        hidden_input = htmltext('<input type="hidden" name="%s$token" value="%s"></input>')
        self.add(HtmlWidget, 'token', title = hidden_input % (self.name, token['token']))

        # create question, and fill token['answer']
        if mode == 'arithmetic' or mode == 'arithmetic-simple':
            a, b = random.randint(2, 9), random.randint(2, 9)
            while b == a:
                # don't get twice the same number
                b = random.randint(2, 9)
            if mode == 'arithmetic-simple':
                operator = random.choice([N_('plus'), N_('minus')])
            else:
                operator = random.choice([N_('times'), N_('plus'), N_('minus')])
            if operator == 'times':
                answer = a * b
            elif operator == 'plus':
                answer = a + b
            elif operator == 'minus':
                if b > a:
                    a, b = b, a
                answer = a - b
            self.question = _('What is the result of %(a)d %(op)s %(b)d?') % {
                    'a': a, 'b': b, 'op': _(operator)}
            self.hint = kwargs.get('hint')
            if self.hint is None:
                self.hint = _('Please answer this simple mathematical question as proof you are not a bot.')
        self.add(StringWidget, 'q', required=True, attrs={'required': 'required'})
        token['answer'] = str(answer)

    def _parse(self, request):
        v = {'answer': self.get('q'),
             'token': request.form.get('%s$token' % self.name) }
        token = get_session().get_captcha_token(v['token'])
        if v['answer'] and token and token['answer'] == v['answer'].strip():
            get_session().won_captcha = True
            self.value = v
        elif v['answer']:
            self.error = _('wrong answer')

    def get_title(self):
        return self.question

    def render_content(self):
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render_content()
        return r.getvalue()

class WidgetList(quixote.form.widget.WidgetList):
    def __init__(self, name, value=None,
                 element_type=StringWidget,
                 element_kwargs={},
                 add_element_label="Add row", **kwargs):

        if add_element_label == 'Add row':
            add_element_label = _('Add row')

        super(WidgetList, self).__init__(name, value=value,
                element_type=element_type,
                element_kwargs=element_kwargs,
                add_element_label=add_element_label, **kwargs)


    def render(self):
        get_response().add_javascript(['jquery.js', 'widget_list.js'])
        r = TemplateIO(html=True)
        r += self.render_title(self.get_title())
        r += self.render_error(self.get_error())
        add_element_widget = self.get_widget('add_element')
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            r += widget.render()
        r += add_element_widget.render()
        r += self.render_hint(self.get_hint())
        return r.getvalue()

class WidgetDict(quixote.form.widget.WidgetDict):
    # Fix the title and hint setting
    # FIXME: to be fixed in Quixote upstream : title and hint parameters should be removed
    def __init__(self, name, value=None, title='', hint='',
                 element_key_type=StringWidget,
                 element_value_type=StringWidget,
                 element_key_kwargs={},
                 element_value_kwargs={},
                 add_element_label='Add row', **kwargs):

        if add_element_label == 'Add row':
            add_element_label = _('Add row')

        quixote.form.widget.WidgetDict.__init__(self, name, value,
                element_key_type=element_key_type,
                element_value_type=element_value_type,
                element_key_kwargs=element_key_kwargs,
                element_value_kwargs=element_value_kwargs,
                add_element_label=add_element_label,
                **kwargs)
        if title:
            self.title = title
        if hint:
            self.hint = hint

class TagsWidget(StringWidget):
    def __init__(self, name, value = None, known_tags = None, **kwargs):
        StringWidget.__init__(self, name, value, **kwargs)
        self.known_tags = known_tags

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value is not None:
            self.value = [x.strip() for x in self.value.split(',') if x.strip()]

    def render_content(self):
        r = TemplateIO(html=True)
        get_response().add_javascript(['jquery.js', 'jquery.autocomplete.js'])
        get_response().add_css_include('../js/jquery.autocomplete.css')
        id = 'tags-%s' % randbytes(8)
        if self.value:
            value = ', '.join(self.value) + ', '
        else:
            value = ''
        r += htmltag('input', xml_end = True,
                id = id,
                type = self.HTML_TYPE,
                name = self.name,
                value = value,
                **self.attrs)
        if self.known_tags:
            known_tags = ', '.join([repr(x) for x in sorted(self.known_tags)])
        else:
            known_tags = ''
        r += htmltext('''<script type="text/javascript">
$("#%s").autocompleteArray([
    %s
    ],{mode:'multiple', multipleSeparator:', '});
</script>''' % (id, known_tags))
        return r.getvalue()

class WysiwygTextWidget(TextWidget):
    def _parse(self, request):
        TextWidget._parse(self, request)
        if self.value:
            if _sanitizeHTML:
                self.value = _sanitizeHTML(self.value, get_request().charset, 'text/html')
            if self.value.startswith('<br />'):
                self.value = self.value[6:]
            if self.value.endswith('<br />'):
                self.value = self.value[:-6]

    def render_content(self):
        get_response().add_javascript(['jquery.js', 'ckeditor/ckeditor.js',
            'qommon.wysiwyg.js', 'ckeditor/adapters/jquery.js'])

        attrs = self.attrs.copy()

        return (htmltag('textarea', name=self.name, **attrs) +
                htmlescape(self.value or '') +
                htmltext("</textarea>"))


class TableWidget(CompositeWidget):
    readonly = False

    def __init__(self, name, value = None, rows = None, columns = None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.rows = rows
        self.columns = columns

        if kwargs.has_key('title'):
            del kwargs['title']

        if kwargs.has_key('readonly') and kwargs.get('readonly'):
            self.readonly = True

        for i, row in enumerate(rows):
            for j, column in enumerate(columns):
                widget = self.add_widget(kwargs, i, j)
                widget = self.get_widget('c-%s-%s' % (i, j))
                if value and self.readonly:
                    if not get_request().form:
                        get_request().form = {}
                    get_request().form[widget.name] = value[i][j]
                    widget.set_value(value[i][j])

    def add_widget(self, kwargs, i, j):
        widget_kwargs = {}
        if kwargs.has_key('readonly') and kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(StringWidget, 'c-%s-%s' % (i, j), **widget_kwargs)

    def render_content(self):
        r = TemplateIO(html=True)
        r += htmltext('<table><thead><tr><td></td>')
        for column in self.columns:
            r += htmltext('<th><span>%s</span></th>') % column
        r += htmltext('</tr></thead><tbody>')
        for i, row in enumerate(self.rows):
            r += htmltext('<tr><th>%s</th>') % row
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                r += htmltext('<td>')
                r += widget.render_content()
                r += htmltext('</td>')
            r += htmltext('</tr>')
        r += htmltext('</tbody></table>')
        return r.getvalue()

    def parse(self, request=None):
        CompositeWidget.parse(self, request=request)
        if request is None:
            request = get_request()
        if (request.form or request.get_method() == 'POST') and self.required:
            if not self.value:
                self.set_error(self.REQUIRED_ERROR)
            else:
                for row in self.value:
                    for column in row:
                        if column:
                            break
                    else:
                        continue
                    break
                else:
                    self.set_error(self.REQUIRED_ERROR)
        return self.value

    def _parse(self, request):
        if self.readonly:
            return
        table = []
        for i, row in enumerate(self.rows):
            row = []
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                row.append(widget.parse())
            table.append(row)
        self.value = table

    def set_value(self, value):
        self.value = value
        if not value:
            return
        for i, row in enumerate(self.rows):
            for j, column in enumerate(self.columns):
                widget = self.get_widget('c-%s-%s' % (i, j))
                widget.set_value(value[i][j])


class SingleSelectTableWidget(TableWidget):
    def add_widget(self, kwargs, i, j):
        widget_kwargs = {'options': kwargs.get('options')}
        if kwargs.has_key('readonly') and kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(SingleSelectWidget, 'c-%s-%s' % (i, j), **widget_kwargs)


class CheckboxesTableWidget(TableWidget):
    def add_widget(self, kwargs, i, j):
        widget_kwargs = {'options': kwargs.get('options')}
        if kwargs.has_key('readonly') and kwargs.get('readonly'):
            widget_kwargs['readonly'] = 'readonly'
        return self.add(CheckboxWidget, 'c-%s-%s' % (i, j), **widget_kwargs)


class SingleSelectHintWidget(SingleSelectWidget):

    def separate_hint(self):
        return (self.hint and len(self.hint) > 80)

    def render_content(self):
        attrs = {'id': 'form_' + self.name}
        if self.attrs:
            attrs.update(self.attrs)
        tags = [htmltag('select', name=self.name, **attrs)]
        options = self.options[:]
        if not self.separate_hint() and self.hint:
            r = htmltag('option', value='', selected=None)
            tags.append(r + htmlescape(self.hint) + htmltext('</option>'))
            if self.options[0][0] is None:
                # hint has been put as first element, skip the default empty
                # value.
                options = self.options[1:]
        for object, description, key in options:
            if self.is_selected(object):
                selected = 'selected'
            else:
                selected = None
            if description is None:
                description = ''
            r = htmltag('option', value=key, selected=selected)
            tags.append(r + htmlescape(description) + htmltext('</option>'))
        tags.append(htmltext('</select>'))
        return htmltext('\n').join(tags)

    def get_hint(self):
        if self.separate_hint():
            return SingleSelectWidget.get_hint(self)
        return None


class WidgetListAsTable(WidgetList):
    def render_content(self):
        r = TemplateIO(html=True)
        add_element_widget = self.get_widget('add_element')
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            if not hasattr(widget, 'render_content_as_tr'):
                r += widget.render()
        r += htmltext('<table>')
        r += self.get_widgets()[1].render_as_thead()
        for widget in self.get_widgets():
            if widget is add_element_widget:
                continue
            if hasattr(widget, 'render_content_as_tr'):
                r += widget.render_content_as_tr()
        r += htmltext('</table>')
        if not self.readonly:
            r += add_element_widget.render()
        return r.getvalue()

    def render(self):
        return render(self)


class TableRowWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        for i, column in enumerate(self.columns):
            self.add(StringWidget, name='col%s'%i, title=column, **kwargs)

class TableListRowsWidget(WidgetListAsTable):
    readonly = False

    def create_row_class(self, columns):
        class klass(TableRowWidget): pass
        setattr(klass, 'columns', columns)
        return klass

    def add_element(self, value=None):
        name = "element%d" % len(self.element_names)
        self.add(self.table_row_class, name, value=value, **self.widget_kwargs)
        self.element_names.append(name)
        self.get_widget('added_elements').value = len(self.element_names)

    def __init__(self, name, value=None, columns=None, min_rows=5, **kwargs):
        self.table_row_class = self.create_row_class(columns)
        self.widget_kwargs = {}
        if kwargs.has_key('readonly'):
            del kwargs['readonly']
            self.readonly = True
            self.widget_kwargs['readonly'] = 'readonly'
        WidgetListAsTable.__init__(self, name, value,
                element_type=self.table_row_class,
                element_kwargs=self.widget_kwargs, **kwargs)
        self.columns = columns

        while len(self.element_names) < min_rows:
            self.add_element()

        self.set_value(value)

    def parse(self, request=None):
        WidgetListAsTable.parse(self, request=request)
        if request is None:
            request = get_request()
        add_element_pushed = self.get_widget('add_element').parse()
        if (request.form or request.get_method() == 'POST') and self.required:
            if not self.value and not add_element_pushed:
                self.set_error(self.REQUIRED_ERROR)
            for row in self.value:
                for column in row:
                    if column:
                        break
                else:
                    continue
                break
            else:
                if not add_element_pushed:
                    self.set_error(self.REQUIRED_ERROR)
        return self.value

    def _parse(self, request):
        if self.readonly:
            return
        table = []
        for i, row_name in enumerate(self.element_names):
            row = []
            row_widget = self.get_widget(row_name)
            notnull = False
            for j, column in enumerate(self.columns):
                widget = row_widget.get_widget('col%s'%j)
                row.append(widget.parse())
                if row[-1]:
                    notnull = True
            if notnull:
                table.append(row)
        self.value = table

    def set_value(self, value):
        self.value = value
        if not value:
            return
        while len(self.element_names) < len(value):
            self.add_element()
        for i, row_name in enumerate(self.element_names):
            widget_row = self.get_widget(row_name)
            for j, column in enumerate(self.columns):
                widget = widget_row.get_widget('col%s'%j)
                try:
                    widget.set_value(value[i][j])
                    if not get_request().form.get(widget.get_name()):
                        get_request().form[widget.get_name()] = value[i][j]
                except IndexError:
                    pass

class RankedItemsWidget(CompositeWidget):
    readonly = False

    def __init__(self, name, value=None, elements=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.element_names = {}

        if kwargs.has_key('title'):
            del kwargs['title']
        if kwargs.has_key('readonly'):
            if kwargs['readonly']:
                self.readonly = True
            del kwargs['readonly']
        if kwargs.has_key('required'):
            if kwargs['required']:
                self.required = True
            del kwargs['required']

        self.randomize_items = False
        if kwargs.has_key('randomize_items'):
            if kwargs['randomize_items']:
                self.randomize_items = True
            del kwargs['randomize_items']

        for v in elements:
            if type(v) is tuple:
                title = v[1]
                key = v[0]
                if type(key) is int:
                    name = 'element%d' % v[0]
                elif type(key) in (str, htmltext):
                    name = str('element%s' % v[0])
                    key = str(key)
                else:
                    raise NotImplementedError()
            else:
                title = v
                key = v
                name = 'element%d' % len(self.element_names.keys())

            if value:
                position = value.get(key)
            else:
                position = None
            self.add(IntWidget, name, title=title, value=position, size=5,
                     required=False, **kwargs)
            self.element_names[name] = key

        if self.randomize_items:
            random.shuffle(self.widgets)

        if self.readonly:
            def cmp_w(x, y):
                if x.value is None and y.value is None:
                    return 0
                if x.value is None:
                    return 1
                if y.value is None:
                    return -1
                return cmp(x.value, y.value)
            self.widgets.sort(cmp_w)

            if value:
                # in readonly mode, we mark all fields as already parsed, to
                # avoid getting them reinitialized on a has_error() call
                for name in self.element_names:
                    self.get_widget(name)._parsed = True
                self._parsed = True

    def _parse(self, request):
        values = {}
        for name in self.element_names:
            value = self.get(name)
            if value is not None:
                values[self.element_names[name]] = value
            if value is not None and type(value) is not int:
                self.get_widget(name).set_error(IntWidget.TYPE_ERROR)
        self.value = values or None

    def set_value(self, value):
        self.value = value
        if value:
            for name in self.element_names:
                self.get_widget(name).set_value(self.value.get(self.element_names[name]))

    def render_content(self):
        r = TemplateIO(html=True)
        r += htmltext('<ul>')
        for widget in self.get_widgets():
            if widget.has_error():
                r += htmltext('<li class="error"><label>')
            else:
                r += htmltext('<li><label>')
            if self.readonly:
                widget.attrs['disabled'] = 'disabled'
                if widget.value:
                    r += htmltext('<input type="hidden" name="%s" value="%s" >') % (
                            widget.name, widget.value)
                widget.name = widget.name + 'xx'
            r += widget.render_content()
            r += widget.title
            r += htmltext('</label>')
            r += htmltext('</li>')
        r += htmltext('</ul>')
        return r.getvalue()


class JsonpSingleSelectWidget(Widget):
    def __init__(self, name, value=None, url=None, **kwargs):
        Widget.__init__(self, name, value=value, **kwargs)
        self.url = url

    def render_content(self):
        if self.value is None:
            value = None
        else:
            value = htmlescape(self.value)
        r = TemplateIO(html=True)

        attrs = {'id': 'form_' + self.name}
        r += htmltag('input', xml_end=True, type="hidden", name=self.name, value=value, **attrs)

        attrs = {'id': 'form_' + self.name + '_display'}
        if value and get_session().jsonp_display_values:
            key = '%s_%s' % (self.url, value)
            if key in get_session().jsonp_display_values:
                attrs['value'] = get_session().jsonp_display_values.get(key)
        r += htmltag('input', xml_end=True, type="hidden", name=self.name + '_display', **attrs)
        initial_display_value = attrs.get('value')

        get_response().add_javascript(['jquery.js', 'select2/select2.min.js'])
        get_response().add_css_include('../js/select2/select2.css')

        # init select2 widget
        allowclear = ""
        if not self.required:
            allowclear = "placeholder: '...', allowClear: true,"

        r += htmltext("""
<script id="script_%(id)s">
$(document).ready(function() {
$("#form_%(id)s").select2({
    minimumInputLength: 1,
    width: '20em',
    %(allowclear)s
    formatNoMatches: function () { return "%(nomatches)s"; },
    formatInputTooShort: function (input, min) { return "%(tooshort)s"; },
    formatLoadMore: function (pageNumber) { return "%(loadmore)s"; },
    formatSearching: function () { return "%(searching)s"; },
    ajax: {
""" % {'id': self.name,
                'allowclear': allowclear,
                'nomatches': _('No matches found'),
                'tooshort': _('Please enter more characters'),
                'loadmore': _('Loading more results...'),
                'searching': _('Searching...')})

        if '[' in self.url:
            vars = get_publisher().substitutions.get_context_variables()
            # skip variables that were not set (None)
            vars = dict((x, y) for x, y in vars.items() if y is not None)
            url = misc.get_variadic_url(self.url, vars, encode_query=False)
        else:
            url = self.url

        if not '[var_' in url:
            # if the url is not parametric, set the url directly
            r += htmltext("""url: "%(url)s",""" % {'url': url})
        else:
            # otherwise keep that blank for now (see below)
            r += htmltext("""url: null,""")

        # setting up the select2 widget continues here
        r += htmltext("""
        dataType: 'jsonp',
        data: function (term, page) {
            return {
                q: term,
                page_limit: 10
            };
        },
        results: function (data, page) { // parse the results into the format expected by Select2.
            var more = (page * 10) < data.total; // whether or not there are more results available
            // since we are using custom formatting functions we do not need to alter remote JSON data
            return {results: data.data, more: more};
        }
    },
    formatResult: function(result) { return result.text; }
});""")

        if '[var_' in url:
            # if this is a parametric url, store template url and hook to the
            # appropriate onchange event to give the url to select2
            r += htmltext("""
$("#form_%(id)s").data('select2').opts.wcs_base_url = '%(url)s';
""" % {'id': self.name, 'url': url})
            variables = re.findall(r'\[(var_.+?)\]', url)
            r += htmltext("""
function url_replace_%(id)s() {
    var url = $("#form_%(id)s").data('select2').opts.wcs_base_url;""" % {'id': self.name})
            for variable in variables:
                r += htmltext("""
    selector = '#' + $('#%(variable)s').data('valuecontainerid');
    url = url.replace('[%(variable)s]', $(selector).val());""" % {'variable': variable})
            r += htmltext("""
    $("#form_%(id)s").data('select2').opts.ajax.url = url;
    $("#form_%(id)s").data('select2').clear();
}
""" % {'id': self.name} )
            for variable in variables:
                r += htmltext("""
$('#%(variable)s').change(url_replace_%(id)s);
$('#%(variable)s').change();
""" % {'id': self.name, 'variable': variable})

        # finish setting up select2, update the _display hidden field with the
        # selected text
        r += htmltext("""

$("#form_%(id)s").change(function() {
    var text = $("#form_%(id)s").data('select2').selection.find('span').text();
    $('#form_%(id)s_display').val(text);
});

""" % {'id': self.name})

        if initial_display_value:
            r += htmltext("""
$("#form_%(id)s").select2("data", {id: "%(value)s", text: "%(text)s"});
""" % {'id': self.name, 'value': self.value, 'text': initial_display_value})

        r += htmltext("""});</script>""")

        return r.getvalue()

    def parse(self, request=None):
        if request and request.form.get(self.name) and request.form.get(self.name + '_display'):
            # store text value associated to the jsonp value
            if not get_session().jsonp_display_values:
                get_session().jsonp_display_values = {}
            value = request.form.get(self.name)
            display_value = request.form.get(self.name + '_display')
            get_session().jsonp_display_values['%s_%s' % (self.url, value)] = display_value

        return Widget.parse(self, request=request)

class AutocompleteStringWidget(WcsExtraStringWidget):
    url = None

    def __init__(self, *args, **kwargs):
        WcsExtraStringWidget.__init__(self, *args, **kwargs)
        if kwargs.get('url'):
            self.url = kwargs.get('url')

    def render_content(self):
        get_response().add_javascript(['jquery.js', 'jquery-ui.js'])

        if self.url and '[' in self.url:
            vars = get_publisher().substitutions.get_context_variables()
            # skip variables that were not set (None)
            vars = dict((x, y) for x, y in vars.items() if y is not None)
            url = misc.get_variadic_url(self.url, vars, encode_query=False)
        else:
            url = self.url

        r = TemplateIO(html=True)
        r += WcsExtraStringWidget.render_content(self)
        if not url:
            # there's no autocomplete URL, get out now.
            return r.getvalue()

        r += htmltext("""
<script id="script_%(id)s">
$(function() {
  $("#form_%(id)s").autocomplete({
    source: function( request, response ) {
      $.ajax({
        url: $("#form_%(id)s").data('uiAutocomplete').options.url,
        dataType: "jsonp",
        data: {
          q: request.term
        },
        success: function( data ) {
          response( $.map(data.data, function(item) {
            return {label: item.text, value: item.label};
           }));
        }
      });
    },
    minLength: 2,
    open: function() {
      $(this).removeClass("ui-corner-all").addClass("ui-corner-top");
    },
    close: function() {
      $(this).removeClass("ui-corner-top").addClass("ui-corner-all");
    }
  });
""" % {'id': self.name})

        if not '[var_' in url:
            r += htmltext("""
$("#form_%(id)s").data('uiAutocomplete').options.url = '%(url)s';
""" % {'id': self.name, 'url': url})

        if '[var_' in url:
            # if this is a parametric url, store template url and hook to the
            # appropriate onchange event to give the url to autocomplete
            r += htmltext("""
$("#form_%(id)s").data('uiAutocomplete').options.wcs_base_url = '%(url)s';
""" % {'id': self.name, 'url': url})
            variables = re.findall(r'\[(var_.+?)\]', url)
            r += htmltext("""
function url_replace_%(id)s() {
    var url = $("#form_%(id)s").data('uiAutocomplete').options.wcs_base_url;""" % {'id': self.name})
            for variable in variables:
                r += htmltext("""
    selector = '#' + $('#%(variable)s').data('valuecontainerid');
    url = url.replace('[%(variable)s]', $(selector).val());""" % {'variable': variable})
            r += htmltext("""
    $("#form_%(id)s").data('uiAutocomplete').options.url = url;
    if ($("form_%(id)s").val() != $("form_%(id)s").attr('value'))
        $("#form_%(id)s").val('');
}
""" % {'id': self.name} )
            for variable in variables:
                r += htmltext("""
$('#%(variable)s').change(url_replace_%(id)s);
$('#%(variable)s').change();
""" % {'id': self.name, 'variable': variable})

        r += htmltext("""
});
</script>""")

        return r.getvalue()


class ColourWidget(SingleSelectWidget):

    def __init__(self, *args, **kwargs):
        colours = ['%s%s%s' % x for x in itertools.product(('00', '66', '99', 'FF'), repeat=3)]
        SelectWidget.__init__(self, options=colours, *args, **kwargs)
        self.attrs['class'] = 'colour-picker'

    def render_content(self):
        get_response().add_javascript(['jquery.js', 'jquery.colourpicker.js'])
        return SingleSelectWidget.render_content(self)

    def _parse(self, request):
        SingleSelectWidget._parse(self, request)
        if self.value is not None:
            self.value = self.value.upper()


class PasswordEntryWidget(CompositeWidget):
    min_length = 0
    max_length = 0
    count_uppercase = 0
    count_lowercase = 0
    count_digit = 0
    count_special = 0
    confirmation = True

    def __init__(self, name, value=None, **kwargs):
        # hint will be displayed with pwd1 widget
        hint = kwargs.pop('hint', None)
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.min_length = kwargs.get('min_length', 0)
        self.max_length = kwargs.get('max_length', 0)
        self.count_uppercase = kwargs.get('count_uppercase', 0)
        self.count_lowercase = kwargs.get('count_lowercase', 0)
        self.count_digit = kwargs.get('count_digit', 0)
        self.count_special = kwargs.get('count_special', 0)
        self.confirmation = kwargs.get('confirmation', True)
        confirmation_title = kwargs.get('confirmation_title') or _('Confirmation')
        self.strength_indicator = kwargs.get('strength_indicator', True)

        self.formats = kwargs.get('formats', ['sha1'])
        if not self.attrs.get('readonly'):
            self.add(PasswordWidget, name='pwd1', title='',
                    value='',
                    required=kwargs.get('required', False),
                    autocomplete='off',
                    hint=hint)
            if self.confirmation:
                self.add(PasswordWidget, name='pwd2', title=confirmation_title,
                        required=kwargs.get('required', False),
                        autocomplete='off')
        else:
            encoded_value = base64.encodestring(json.dumps(value))
            if value:
                fake_value = '*' * 8
            else:
                fake_value = ''
            self.add(HtmlWidget, 'hashed', title=htmltext(
                    '<input value="%s" readonly="readonly">'\
                    '<input type="hidden" name="%s$encoded" value="%s"></input>' % (
                        fake_value, self.name, encoded_value)))

    def render_content(self):
        if self.attrs.get('readonly') or not self.strength_indicator:
            return CompositeWidget.render_content(self)
        get_response().add_javascript(['jquery.js', 'jquery.passstrength.js'])
        r = TemplateIO(html=True)
        r += CompositeWidget.render_content(self)
        r += htmltext('''<script>
$(function() {
  $('input[id="form_%(form_id)s"]').passStrengthify({
    levels: ["%(veryweak)s", "%(veryweak)s", "%(weak)s", "%(weak)s", "%(moderate)s", "%(good)s", "%(strong)s", "%(verystrong)s"],
    minimum: %(min_length)s,
    labels: {
      passwordStrength: "%(password_strength)s",
      tooShort: "%(tooshort)s"
    }
  });
});
</script>''') % {'form_id': self.get_widget('pwd1').get_name(),
                'min_length': self.min_length,
                'veryweak': _('Very weak'),
                'weak': _('Weak'),
                'moderate': _('Moderate'),
                'good': _('Good'),
                'strong': _('Strong'),
                'verystrong': _('Very strong'),
                'password_strength': _('Password strength:'),
                'tooshort': _('Too short')}
        return r.getvalue()

    def _parse(self, request):
        CompositeWidget._parse(self, request)
        if request.form.get('%s$encoded' % self.name):
            self.value = json.loads(base64.decodestring(
                request.form.get('%s$encoded' % self.name)))
            return
        pwd1 = self.get('pwd1') or ''

        if not self.get_widget('pwd1'):
            # we are in read-only mode, stop here.
            return

        set_errors = []
        min_len = self.min_length
        if len(pwd1) < min_len:
            set_errors.append(_('Password is too short.  It must be at least %d characters.') % min_len)

        max_len = self.max_length
        if max_len and len(pwd1) > max_len:
            set_errors.append(_('Password is too long.  It must be at most %d characters.') % max_len)

        count = self.count_uppercase
        if len(filter(lambda c: c.isupper(), pwd1)) < count:
            set_errors.append(
                ngettext('Password must contain an uppercase character.',
                    'Password must contain at least %(count)d uppercase characters.',
                    count) % {'count': count})

        count = self.count_lowercase
        if len(filter(lambda c: c.islower(), pwd1)) < count:
            set_errors.append(
                ngettext('Password must contain a lowercase character.',
                    'Password must contain at least %(count)d lowercase characters.',
                    count) % {'count': count})

        count = self.count_digit
        if len(filter(lambda c: c.isdigit(), pwd1)) < self.count_digit:
            set_errors.append(
                ngettext('Password must contain a digit.',
                    'Password must contain at least %(count)d digits.',
                    count) % {'count': count})

        count = self.count_special
        if len(filter(lambda c: not c.isalnum(), pwd1)) < count:
            set_errors.append(
                ngettext('Password must contain a special character.',
                    'Password must contain at least %(count)d special characters.',
                    count) % {'count': count})

        if self.confirmation:
            pwd2 = self.get('pwd2') or ''
            if pwd1 != pwd2:
                self.get_widget('pwd2').set_error(_('Passwords do not match'))
                pwd1 = None

        if set_errors:
            self.get_widget('pwd1').set_error(' '.join(set_errors))
            pwd1 = None

        PASSWORD_FORMATS = {
            'cleartext': lambda x: x,
            'md5': lambda x: hashlib.md5(x).hexdigest(),
            'sha1': lambda x: hashlib.sha1(x).hexdigest(),
        }

        if pwd1:
            self.value = {}
            for fmt in self.formats:
                self.value[fmt] = PASSWORD_FORMATS[fmt](pwd1)
        else:
            self.value = None


class MapWidget(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value, **kwargs)
        self.add(HiddenWidget, 'latlng', value=value)
        widget = self.get_widget('latlng')
        if value and get_request().form and not get_request().form.get(widget.name):
            get_request().form[widget.name] = value
        self.readonly = kwargs.pop('readonly', False)
        self.kwargs = kwargs

    def render_content(self):
        get_response().add_javascript(['qommon.map.js'])
        r = TemplateIO(html=True)
        for widget in self.get_widgets():
            r += widget.render()
        attrs = {
            'class': 'qommon-map',
            'id': 'map-%s' % self.name,
        }
        if self.value:
            attrs['data-init-lat'], attrs['data-init-lng'] = self.value.split(';')
        if self.readonly:
            attrs['data-readonly'] = 'true'
        for attribute in ('initial_zoom', 'min_zoom', 'max_zoom'):
            if attribute in self.kwargs and self.kwargs.get(attribute) is not None:
                attrs['data-%s' % attribute] = self.kwargs.get(attribute)
        default_position = self.kwargs.get('default_position')
        if not default_position:
            default_position = get_publisher().get_site_option('default_position')
        if not default_position:
            default_position = '50.84;4.36'
        attrs['data-def-lat'], attrs['data-def-lng'] = default_position.split(';')
        if self.kwargs.get('init_with_geoloc'):
            attrs['data-init-with-geoloc'] = 1
        r += htmltext('<div %s></div>' % ' '.join(['%s="%s"' % x for x in attrs.items()]))
        return r.getvalue()

    def _parse(self, request):
        CompositeWidget._parse(self, request)
        self.value = self.get('latlng')


class HiddenErrorWidget(HiddenWidget):
    def set_error(self, error):
        Widget.set_error(self, error)


class SingleSelectWidgetWithOther(CompositeWidget):
    def __init__(self, name, value=None, **kwargs):
        CompositeWidget.__init__(self, name, value=value, **kwargs)
        options = kwargs.get('options')[:]
        if len(options[0]) == 1:
            options = [(x[0], x[0], x[0]) for x in options]
        elif len(options[0]) == 2:
            options = [(x[0], x[1], x[0]) for x in options]
        options.append(('__other', _('Other:'), '__other'))
        kwargs['options'] = options
        if value is None or value in [x[0] for x in options]:
            choice_value = value
            other_value = None
        else:
            choice_value = '__other'
            other_value = value
        self.add(SingleSelectWidget, 'choice', value=choice_value, **kwargs)
        self.add(StringWidget, 'other', value=other_value, size=35)

    def _parse(self, request):
        self.value = self.get('choice')
        if self.value == '__other':
            self.value = self.get('other')


class ComputedExpressionWidget(StringWidget):
    '''StringWidget that checks the entered value is a correct workflow
       expression.'''

    @classmethod
    def validate_ezt(cls, template):
        processor = ezt.Template(compress_whitespace=False)
        try:
            processor.parse(template or '')
        except ezt.EZTException as e:
            parts = []
            parts.append({
                ezt.ArgCountSyntaxError: _('wrong number of arguments'),
                ezt.UnknownReference: _('unknown reference'),
                ezt.NeedSequenceError: _('sequence required'),
                ezt.UnclosedBlocksError: _('unclosed block'),
                ezt.UnmatchedEndError: _('unmatched [end]'),
                ezt.BaseUnavailableError: _('unavailable base location'),
                ezt.BadFormatConstantError: _('bad format constant'),
                ezt.UnknownFormatConstantError: _('unknown format constant'),
                }.get(e.__class__))
            if e.line:
                parts.append(_('at line %(line)d and column %(column)d') % {
                    'line': e.line+1,
                    'column': e.column+1})
            raise ValueError(_('error in template (%s)') % ' '.join(parts))

    def _parse(self, request):
        StringWidget._parse(self, request)
        if self.value:
            if self.value.startswith('='):
                # python expression
                try:
                    compile(self.value[1:], '<string>', 'eval')
                except SyntaxError as e:
                    self.set_error(_('syntax error (%s)') % e)
            else:
                # ezt expression
                try:
                    self.validate_ezt(self.value)
                except ValueError as e:
                    self.set_error(str(e))