wcs/tests/admin_pages/test_form.py

# -*- coding: utf-8 -*-

import datetime
import io
import os
import re
import tarfile
import time
import xml.etree.ElementTree as ET

import mock
import pytest
from utilities import clean_temporary_pub, create_temporary_pub, get_app, login
from webtest import Upload

from wcs import fields
from wcs.carddef import CardDef
from wcs.categories import Category
from wcs.data_sources import NamedDataSource
from wcs.formdef import FormDef
from wcs.qommon.errors import ConnectionError
from wcs.qommon.http_request import HTTPRequest
from wcs.workflows import Workflow
from wcs.wscalls import NamedWsCall

from .test_all import create_role, create_superuser


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates'], indirect=True)


@pytest.fixture
def pub(request):
    pub = create_temporary_pub(
        sql_mode=bool('sql' in request.param), templates_mode=bool('templates' in request.param)
    )

    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
    pub.set_app_dir(req)
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    return pub


def teardown_module(module):
    clean_temporary_pub()


def test_forms(pub):
    create_superuser(pub)
    pub.role_class.wipe()
    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/')
    assert 'You first have to define roles.' in resp.text
    assert 'New Form' not in resp.text


def test_forms_new(pub):
    create_superuser(pub)
    app = login(get_app(pub))
    create_role(pub)

    FormDef.wipe()
    # create a new form
    resp = app.get('/backoffice/forms/')
    assert 'New Form' in resp.text
    resp = resp.click('New Form')
    resp.forms[0]['name'] = 'form title'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert '<h2>form title' in resp.text

    # makes sure the data has been correctly saved
    formdef = FormDef.get(1)
    assert formdef.name == 'form title'
    assert formdef.url_name == 'form-title'
    assert formdef.fields == []
    assert formdef.disabled is True


def test_forms_new_popup(pub):
    FormDef.wipe()
    create_superuser(pub)
    app = login(get_app(pub))
    create_role(pub)

    # create a new form
    resp = app.get('/backoffice/forms/')
    assert 'New Form' in resp.text
    resp = resp.click('New Form', extra_environ={'HTTP_X_POPUP': 'true'})
    assert 'popup-content' in resp.text
    resp.forms[0]['name'] = 'form title'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert '<h2>form title' in resp.text

    # makes sure the data has been correctly saved
    formdef = FormDef.get(1)
    assert formdef.name == 'form title'
    assert formdef.url_name == 'form-title'
    assert formdef.fields == []
    assert formdef.disabled is True


def assert_option_display(resp, label, value):
    option_line = re.findall('%s.*%s' % (label, value), resp.text, re.DOTALL)
    assert option_line
    assert not '</li>' in option_line


def test_forms_edit(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    # try changing an option

    # confirmation page
    assert_option_display(resp, 'Confirmation Page', 'Enabled')
    resp = resp.click('Confirmation Page')
    assert resp.forms[0]['confirmation'].checked
    resp.forms[0]['confirmation'].checked = False
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Confirmation Page', 'Disabled')
    assert FormDef.get(1).confirmation is False

    # try cancel button
    resp = resp.click('Confirmation Page')
    assert resp.forms[0]['confirmation'].checked is False
    resp.forms[0]['confirmation'].checked = True
    resp = resp.forms[0].submit('cancel')
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Confirmation Page', 'Disabled')
    assert FormDef.get(1).confirmation is False

    # Limit to one form
    assert_option_display(resp, 'Limit to one form', 'Disabled')
    resp = resp.click('Limit to one form')
    assert resp.forms[0]['only_allow_one'].checked is False
    resp.forms[0]['only_allow_one'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Limit to one form', 'Enabled')
    assert FormDef.get(1).only_allow_one is True

    # Misc management
    assert_option_display(resp, 'Management', 'Default')
    resp = resp.click('Management', href='options/management')
    assert resp.forms[0]['include_download_all_button'].checked is False
    resp.forms[0]['include_download_all_button'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Management', 'Custom')
    assert FormDef.get(1).include_download_all_button is True

    # Tracking code
    assert_option_display(resp, 'Tracking Code', 'Disabled')
    resp = resp.click('Tracking Code')
    assert resp.forms[0]['enable_tracking_codes'].checked is False
    resp.forms[0]['enable_tracking_codes'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Tracking Code', 'Enabled')
    assert FormDef.get(1).enable_tracking_codes is True

    resp = resp.click('Tracking Code')
    assert resp.forms[0]['drafts_lifespan'].value == ''
    resp.forms[0]['drafts_lifespan'].value = 'xxx'
    resp = resp.forms[0].submit()
    assert 'Lifespan must be between 2 and 100 days.' in resp
    resp.forms[0]['drafts_lifespan'].value = '120'
    resp = resp.forms[0].submit()
    assert 'Lifespan must be between 2 and 100 days.' in resp
    resp.forms[0]['drafts_lifespan'].value = '5'
    resp = resp.forms[0].submit().follow()
    assert FormDef.get(1).drafts_lifespan == '5'

    # CAPTCHA
    assert_option_display(resp, 'CAPTCHA for anonymous users', 'Disabled')
    resp = resp.click('CAPTCHA for anonymous users')
    assert resp.forms[0]['has_captcha'].checked is False
    resp.forms[0]['has_captcha'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'CAPTCHA for anonymous users', 'Enabled')
    assert FormDef.get(1).has_captcha is True

    # Appearance
    assert_option_display(resp, 'Appearance', 'Standard')
    resp = resp.click('Appearance')
    assert resp.forms[0]['appearance_keywords'].value == ''
    resp.forms[0]['appearance_keywords'] = 'foobar'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Appearance', 'foobar')
    assert FormDef.get(1).appearance_keywords == 'foobar'

    # Publication
    assert_option_display(resp, 'Online Status', 'Active')
    resp = resp.click('Online Status')
    assert resp.forms[0]['disabled'].checked is False
    resp.forms[0]['disabled'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Online Status', 'Disabled')
    assert FormDef.get(1).disabled is True

    resp = resp.click('Online Status')
    assert resp.forms[0]['disabled'].checked is True
    resp.forms[0]['disabled_redirection'] = 'http://www.example.net'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Online Status', 'Redirected')
    assert FormDef.get(1).disabled is True
    assert FormDef.get(1).disabled_redirection == 'http://www.example.net'

    resp = resp.click('Online Status')
    resp.forms[0]['disabled'].checked = False
    resp.forms[0]['expiration_date$date'] = '2000-01-01'  # this is past(tm)
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Online Status', 'Inactive by date')

    # enable geolocation
    resp = resp.click('Geolocation')
    resp.forms[0]['geoloc_label'] = 'Foobar'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Geolocation', 'Enabled')
    assert FormDef.get(formdef.id).geolocations == {'base': 'Foobar'}

    # and disable it
    resp = resp.click('Geolocation')
    resp.forms[0]['geoloc_label'] = ''
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Geolocation', 'Disabled')
    assert FormDef.get(formdef.id).geolocations is None


def test_form_title_change(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('change title')
    assert resp.form['name'].value == 'form title'
    assert 'data-slug-sync' in resp.text
    assert not 'change-nevertheless' in resp.text
    resp.form['name'] = 'new title'
    resp = resp.form.submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    formdef = FormDef.get(formdef.id)
    assert formdef.name == 'new title'
    assert formdef.url_name == 'form-title'
    assert formdef.internal_identifier == 'new-title'

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('change title')
    assert not 'data-slug-sync' in resp.text
    assert not 'change-nevertheless' in resp.text

    formdef.data_class()().store()
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('change title')
    assert 'change-nevertheless' in resp.text

    formdef2 = FormDef()
    formdef2.name = 'other title'
    formdef2.fields = []
    formdef2.store()

    resp = app.get('/backoffice/forms/%s/' % formdef2.id)
    resp = resp.click('change title')
    assert resp.form['name'].value == 'other title'
    resp.form['url_name'] = formdef.url_name
    resp = resp.form.submit()
    assert 'This identifier is already used.' in resp.text

    resp.form['url_name'] = 'foobar'
    resp = resp.form.submit().follow()
    assert FormDef.get(formdef2.id).url_name == 'foobar'


def test_forms_edit_publication_date(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/options/online_status')
    resp.form['publication_date$date'] = '2020-01-01'
    resp = resp.form.submit()
    assert FormDef.get(formdef.id).publication_date == '2020-01-01 00:00'

    resp = app.get('/backoffice/forms/1/options/online_status')
    assert resp.form['publication_date$date'].value == '2020-01-01'
    resp.form['publication_date$time'] = '12:00'
    resp = resp.form.submit()
    assert FormDef.get(formdef.id).publication_date == '2020-01-01 12:00'

    resp = app.get('/backoffice/forms/1/options/online_status')
    assert resp.form['publication_date$date'].value == '2020-01-01'
    assert resp.form['publication_date$time'].value == '12:00'

    formdef.publication_date = None
    formdef.store()

    resp = app.get('/backoffice/forms/1/options/online_status')
    resp.form['publication_date$time'] = '12:00'
    resp = resp.form.submit()
    assert 'invalid value' in resp


def test_form_category(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert_option_display(resp, 'Category', 'None')

    Category.wipe()
    cat = Category(name='Foo')
    cat.store()
    cat = Category(name='Bar')
    cat.store()
    resp = app.get('/backoffice/forms/1/')
    assert 'Category' in resp.text
    assert_option_display(resp, 'Category', 'None')


def test_form_category_select(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    Category.wipe()
    cat = Category(name='Foo')
    cat.store()
    cat = Category(name='Bar')
    cat.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='category')
    resp = resp.forms[0].submit('cancel')
    assert FormDef.get(formdef.id).category_id is None

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='category')
    resp.forms[0]['category_id'] = cat.id
    resp = resp.forms[0].submit('submit')
    assert FormDef.get(formdef.id).category_id == cat.id


def test_form_workflow(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert_option_display(resp, 'Workflow', 'Default')

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    workflow.store()
    workflow = Workflow(name='Workflow Two')
    workflow.store()

    resp = app.get('/backoffice/forms/1/')
    assert_option_display(resp, 'Workflow', 'Default')


def test_form_workflow_change(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    workflow.store()
    workflow = Workflow(name='Workflow Two')
    workflow.possible_status = Workflow.get_default_workflow().possible_status[:]
    workflow.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow', index=1)
    resp = resp.forms[0].submit('cancel')
    assert FormDef.get(formdef.id).workflow_id is None

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow', index=1)
    assert 'Workflow One' not in resp.text  # this workflow doesn't have any status
    resp.forms[0]['workflow_id'] = workflow.id
    resp = resp.forms[0].submit('submit')
    assert FormDef.get(formdef.id).workflow_id == workflow.id


def test_form_workflow_link(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    workflow.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert '/backoffice/workflows/_default/' in resp.text

    formdef.workflow = workflow
    formdef.store()

    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert '/backoffice/workflows/%s/' % workflow.id in resp.text

    # check workflow link is not displayed if user has no access right
    pub.cfg['admin-permissions'] = {'workflows': ['x']}  # block access
    pub.write_cfg()
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert '/backoffice/workflows/%s/' % workflow.id not in resp.text


def test_form_workflow_remapping(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()
    formdata1 = data_class()
    formdata1.status = 'wf-new'
    formdata1.store()

    formdata2 = data_class()
    formdata2.status = 'draft'
    formdata2.store()

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    workflow.store()
    workflow = Workflow(name='Workflow Two')
    # create it with a single status
    workflow.possible_status = [Workflow.get_default_workflow().possible_status[-1]]
    workflow.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow', index=1)
    resp.forms[0]['workflow_id'] = workflow.id
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/workflow-status-remapping?new=2'
    resp = resp.follow()
    for status in Workflow.get_default_workflow().possible_status:
        assert resp.forms[0]['mapping-%s' % status.id]
        # there's only one possible new status
        assert len(resp.forms[0]['mapping-%s' % status.id].options) == 1
    assert data_class.get(formdata1.id).status == 'wf-new'
    assert data_class.get(formdata2.id).status == 'draft'
    resp = resp.forms[0].submit()
    assert data_class.get(formdata1.id).status == 'wf-finished'
    assert data_class.get(formdata2.id).status == 'draft'

    # change to another workflow, with no mapping change
    workflow2 = workflow
    workflow = Workflow(name='Workflow Three')
    workflow.possible_status = Workflow.get_default_workflow().possible_status[-2:][:]
    workflow.store()

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow', index=1)
    resp.forms[0]['workflow_id'] = workflow.id
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/workflow-status-remapping?new=3'
    resp = resp.follow()
    for status in workflow2.possible_status:
        assert resp.forms[0]['mapping-%s' % status.id]
        # there are two status
        assert len(resp.forms[0]['mapping-%s' % status.id].options) == 2
    resp = resp.forms[0].submit()
    assert data_class.get(formdata1.id).status == 'wf-finished'
    assert data_class.get(formdata2.id).status == 'draft'


def test_form_submitter_roles(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href=re.compile('^roles$'))
    resp.form['roles$element0'] = 'logged-users'
    assert not 'required_authentication_contexts' in resp.text
    resp = resp.form.submit()
    assert FormDef.get(formdef.id).roles == ['logged-users']

    # add auth contexts support
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'auth-contexts', 'fedict')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href=re.compile('^roles$'))
    assert 'required_authentication_contexts' in resp.text
    resp.form['required_authentication_contexts$element0'].checked = True
    resp = resp.form.submit()
    resp = resp.follow()
    assert FormDef.get(formdef.id).required_authentication_contexts == ['fedict']

    # check internal roles are not advertised
    role2 = pub.role_class(name='internal')
    role2.internal = True
    role2.store()

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href=re.compile('^roles$'))
    assert len(resp.form['roles$element0'].options) == 3  # None, Logged users, foobar
    with pytest.raises(ValueError):
        resp.form['roles$element0'] = str(role2.id)


def test_form_workflow_role(pub):
    create_superuser(pub)
    role = create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='role/_receiver')
    resp = resp.forms[0].submit('cancel')

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='role/_receiver')
    resp.forms[0]['role_id'] = role.id
    resp = resp.forms[0].submit('submit')
    assert FormDef.get(1).workflow_roles == {'_receiver': '1'}

    # check it doesn't fail if a second role with the same name exists
    role = pub.role_class(name='foobar')
    role.store()
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='role/_receiver')


def test_form_workflow_options(pub):
    create_superuser(pub)
    create_role(pub)

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_options = {'2*1*body': 'xxx'}
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert '"workflow-options"' in resp.text


def test_form_workflow_variables(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    # check it's not visible
    assert '"workflow-variables"' not in resp
    # check it doesn't crash anyway
    resp = app.get('/backoffice/forms/1/workflow-variables', status=404)

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    from wcs.workflows import WorkflowVariablesFieldsFormDef

    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
    workflow.variables_formdef.fields.append(
        fields.StringField(id='1', varname='test', label='Test', type='string')
    )
    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert '"workflow-variables"' in resp.text

    # visit the variables page
    resp = resp.click(href='workflow-variables')

    # and set a value
    resp.forms[0]['f1'] = 'foobar'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'

    # check the value has been correctly saved
    assert FormDef.get(formdef.id).workflow_options == {'test': 'foobar'}

    # go back to the variables page, also check value
    resp = resp.follow()
    resp = resp.click(href='workflow-variables')
    assert resp.forms[0]['f1'].value == 'foobar'
    resp.forms[0]['f1'] = 'barbaz'
    resp = resp.forms[0].submit('cancel')
    assert resp.location == 'http://example.net/backoffice/forms/1/'

    # check with a date field
    workflow.variables_formdef.fields.append(
        fields.DateField(id='2', varname='test2', label='Test2', type='date')
    )
    workflow.store()

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow-variables')
    resp.form['f2'] = '2016-06-17'
    resp = resp.form.submit()
    assert time.strftime('%d %m %y', FormDef.get(formdef.id).workflow_options.get('test2')) == '17 06 16'


def test_form_workflow_table_variables(pub):
    create_superuser(pub)
    create_role(pub)

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    from wcs.workflows import WorkflowVariablesFieldsFormDef

    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
    workflow.variables_formdef.fields.append(
        fields.TableRowsField(id='1', varname='test', label='Test2', type='tablerows', columns=['a'])
    )
    workflow.variables_formdef.fields.append(
        fields.StringField(id='2', varname='test2', label='Test', type='string')
    )
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert '"workflow-variables"' in resp.text

    # visit the variables page
    resp = resp.click(href='workflow-variables')

    # and set a value
    resp.form['f1$element0$col0'] = 'foobar'
    resp.form['f2'] = 'foobar'
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/'

    # check the value has been correctly saved
    assert FormDef.get(formdef.id).workflow_options == {'test': [['foobar']], 'test2': 'foobar'}

    # go back to the variables page, also check value
    resp = resp.follow()
    resp = resp.click(href='workflow-variables')
    assert resp.form['f1$element0$col0'].value == 'foobar'
    assert resp.form['f2'].value == 'foobar'


def test_form_roles(pub):
    create_superuser(pub)
    role = create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click('User Roles')
    resp = resp.forms[0].submit('cancel')

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('User Roles')
    resp.forms[0]['roles$element0'].value = role.id
    resp = resp.forms[0].submit('submit')
    assert FormDef.get(1).roles == [role.id]


def test_form_always_advertise(pub):
    create_superuser(pub)
    role = create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    # Display to unlogged users
    formdef.roles = [role.id]
    formdef.store()
    resp = app.get('/backoffice/forms/1/')
    assert_option_display(resp, 'Display to unlogged users', 'Disabled')
    resp = resp.click('Display to unlogged users')
    assert resp.forms[0]['always_advertise'].checked is False
    resp.forms[0]['always_advertise'].checked = True
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Display to unlogged users', 'Enabled')
    assert FormDef.get(1).always_advertise is True


def test_form_templates(pub):
    create_superuser(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', varname='test', label='Test', type='string')]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {'1': 'hello'}
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    assert_option_display(resp, 'Templates', 'None')
    assert resp.pyquery('[href="options/templates"]').attr.rel == ''  # no popup
    resp = resp.click('Templates')
    resp.form['digest_template'] = 'X{{form_var_test}}Y'
    resp = resp.form.submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Templates', 'Custom')
    formdef = FormDef.get(formdef.id)
    assert formdef.digest_template == 'X{{form_var_test}}Y'
    assert formdef.lateral_template is None
    assert formdef.submission_lateral_template is None

    assert 'Existing forms will be updated in the background.' in resp.text
    # afterjobs are actually run synchronously during tests; we don't have
    # to wait to check the digest has been updated:
    assert formdef.data_class().get(formdata.id).digest == 'XhelloY'

    resp = app.get('/backoffice/forms/1/options/templates')
    resp.form['lateral_template'] = 'X{{form_var_test}}Y'
    resp.form['submission_lateral_template'] = 'X{{form_var_test}}YZ'
    resp = resp.form.submit().follow()
    assert_option_display(resp, 'Templates', 'Custom')
    formdef = FormDef.get(formdef.id)
    assert formdef.digest_template == 'X{{form_var_test}}Y'
    assert formdef.lateral_template == 'X{{form_var_test}}Y'
    assert formdef.submission_lateral_template == 'X{{form_var_test}}YZ'
    assert 'Existing forms will be updated in the background.' not in resp.text


def test_form_delete(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click(href='delete')
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/'
    resp = resp.follow()
    assert FormDef.count() == 0


def test_form_delete_with_data(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click(href='delete')
    if pub.is_using_postgresql():
        assert 'Deletion is not possible' in resp

    formdata.status = 'draft'
    formdata.store()
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click(href='delete')
    assert 'Deletion is not possible' not in resp

    formdata.status = 'wf-rejected'
    formdata.store()
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click(href='delete')
    assert 'Deletion is not possible' not in resp


def test_form_duplicate(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click(href='duplicate')
    assert resp.location == 'http://example.net/backoffice/forms/2/'
    resp = resp.follow()
    assert FormDef.count() == 2
    assert FormDef.get(2).name == 'form title (copy)'

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='duplicate')
    assert resp.location == 'http://example.net/backoffice/forms/3/'
    resp = resp.follow()
    assert FormDef.count() == 3
    assert FormDef.get(3).name == 'form title (copy 2)'


def test_form_export(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click(href='export')
    xml_export = resp.text

    fd = io.StringIO(xml_export)
    formdef2 = FormDef.import_from_xml(fd)
    assert formdef2.name == 'form title'


def test_form_import(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()
    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))

    FormDef.wipe()
    assert FormDef.count() == 0

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.forms[0].submit()
    assert FormDef.count() == 1

    # import the same formdef a second time, make sure url name and internal
    # identifier are not reused
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.forms[0].submit()
    assert FormDef.count() == 2
    assert FormDef.get(1).url_name == 'form-title'
    assert FormDef.get(2).url_name == 'form-title-1'
    assert FormDef.get(1).internal_identifier == 'form-title'
    assert FormDef.get(2).internal_identifier == 'form-title-1'

    # import a formdef with an url name that doesn't match its title,
    # it should be kept intact.
    formdef.url_name = 'xxx-other-form-title'
    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))

    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.forms[0].submit()
    assert FormDef.get(3).url_name == 'xxx-other-form-title'
    assert FormDef.get(3).internal_identifier == 'form-title-2'

    # import an invalid file
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.form['file'] = Upload('formdef.wcs', b'garbage')
    resp = resp.form.submit()
    assert 'Invalid File' in resp.text

    # xml with duplicate id, fix it
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.StringField(id='42', label='1st field', type='string'),
        fields.StringField(id='42', label='2nd field', type='string'),
    ]
    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))
    FormDef.wipe()
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.form['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.form.submit()
    resp = resp.follow()
    assert 'form contained errors and has been automatically fixed' in resp.text
    assert FormDef.count() == 1
    assert FormDef.get(1).fields[0].id == '1'
    assert FormDef.get(1).fields[1].id == '2'


def test_form_import_from_url(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()
    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))

    FormDef.wipe()
    assert FormDef.count() == 0

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp = resp.form.submit()
    assert 'You have to enter a file or a URL' in resp

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        urlopen.side_effect = ConnectionError('...')
        resp.form['url'] = 'http://remote.invalid/test.wcs'
        resp = resp.form.submit()
        assert 'Error loading form' in resp
        urlopen.side_effect = lambda *args: io.StringIO(formdef_xml.decode())
        resp.form['url'] = 'http://remote.invalid/test.wcs'
        resp = resp.form.submit()
    assert FormDef.count() == 1


def test_form_qrcode(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click(href='qrcode')
    assert '<div id="qrcode">' in resp.text
    resp = resp.click('Download')
    assert resp.content_type == 'image/png'


def test_form_description(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    assert_option_display(resp, 'Description', 'None')

    resp = resp.click('Description')
    resp.forms[0]['description'].value = '<p>Hello World</p>'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    resp = resp.follow()
    assert_option_display(resp, 'Description', 'On')


def test_form_enable_from_fields_page(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.disabled = True
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'This form is currently disabled.' in resp
    resp = resp.click('Enable').follow()
    assert resp.request.path == '/backoffice/forms/1/fields/'
    assert 'This form is currently disabled.' not in resp


def test_form_new_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'There are not yet any fields for this form' in resp.text

    resp.forms[0]['label'] = 'foobar'
    resp.forms[0]['type'] = 'string'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    resp = resp.follow()
    assert 'foobar' in resp.text
    assert 'Use drag and drop' in resp.text

    assert len(FormDef.get(1).fields) == 1
    assert FormDef.get(1).fields[0].key == 'string'
    assert FormDef.get(1).fields[0].label == 'foobar'

    # add a title too
    resp.forms[0]['label'] = 'baz'
    resp.forms[0]['type'] = 'title'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    resp = resp.follow()

    # check it's in the preview
    resp = app.get('/backoffice/forms/1/')
    assert '<h3 data-field-id="1">baz</h3>' in resp.text


def test_form_field_without_label(pub):
    create_superuser(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.CommentField(id='1', label=None, type='comment')]
    formdef.store()

    app = login(get_app(pub))
    app.get('/backoffice/forms/1/fields/', status=200)  # ok, no error


def test_form_delete_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text
    assert 'Use drag and drop' in resp.text
    assert 'Also remove all fields from the page' not in resp.text

    resp = resp.click(href='1/delete')
    assert 'You are about to remove the "1st field" field.' in resp.text
    assert 'Warning:' not in resp.text
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 0


def test_form_delete_field_existing_data(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.StringField(id='1', label='1st field', type='string'),
        fields.CommentField(id='2', label='comment field', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {'1': 'hello'}
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    resp = resp.click(href='1/delete')
    assert 'You are about to remove the "1st field" field.' in resp.text
    assert 'Warning:' in resp.text
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_2'
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 1

    # check non-data fields do not show this warning
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    resp = resp.click(href='2/delete')
    assert 'You are about to remove the "comment field" field.' in resp.text
    assert 'Warning:' not in resp.text
    resp = resp.forms[0].submit()
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 0


def test_form_delete_page_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(id='1', label='page 1', type='page'),
        fields.StringField(id='2', label='field 1 1', type='string'),
        fields.StringField(id='3', label='field 1 2', type='string'),
        fields.PageField(id='4', label='page 2', type='page'),
        fields.PageField(id='5', label='page 3', type='page'),
        fields.StringField(id='6', label='field 3 1', type='string'),
        fields.StringField(id='7', label='field 3 2', type='string'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub))

    # delete fields from the page
    resp = app.get('/backoffice/forms/1/fields/1/delete')
    assert 'You are about to remove the "page 1" page.' in resp.text
    assert 'Also remove all fields from the page' in resp.text
    resp.forms[0]['delete_fields'] = True
    resp = resp.forms[0].submit()
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 4

    # empty page
    resp = app.get('/backoffice/forms/1/fields/4/delete')
    assert 'You are about to remove the "page 2" page.' in resp.text
    assert 'Also remove all fields from the page' not in resp.text
    resp = resp.forms[0].submit()
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 3

    # keep fields
    resp = app.get('/backoffice/forms/1/fields/5/delete')
    assert 'You are about to remove the "page 3" page.' in resp.text
    assert 'Also remove all fields from the page' in resp.text
    resp.forms[0]['delete_fields'] = False
    resp = resp.forms[0].submit()
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 2


def test_form_duplicate_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click(href='1/duplicate')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_2'
    resp = resp.follow()
    assert len(FormDef.get(1).fields) == 2
    assert FormDef.get(1).fields[0].label == '1st field'
    assert FormDef.get(1).fields[1].label == '1st field'


def test_form_duplicate_file_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')

    # add a first field
    resp.forms[0]['label'] = 'foobar'
    resp.forms[0]['type'] = 'file'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/%s/fields/' % formdef.id
    resp = resp.follow()
    assert 'foobar' in resp.text

    resp = resp.click(href='%s/duplicate' % FormDef.get(formdef.id).fields[0].id)
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_2'
    resp = resp.follow()


def test_form_edit_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == '1st field'
    resp.forms[0]['label'] = 'changed field'
    resp.forms[0]['required'] = False
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'

    assert FormDef.get(1).fields[0].label == 'changed field'
    assert FormDef.get(1).fields[0].required is False


def test_form_edit_field_advanced(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == '1st field'
    assert '<legend>Additional parameters</legend>' in resp.text
    assert '>Prefill</label>' in resp.text
    # check the "prefill" field is under additional parameters
    assert resp.text.index('<legend>Additional parameters</legend>') < resp.text.index('>Prefill</label>')

    # complete the "prefill" field
    resp.forms[0]['prefill$type'] = 'String / Template'
    resp.forms[0]['prefill$value_string'] = 'test'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'
    resp = resp.follow()

    assert FormDef.get(formdef.id).fields[0].prefill == {'type': 'string', 'value': 'test', 'locked': False}

    # do the same with 'data sources' field
    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == '1st field'
    assert '<legend>Additional parameters</legend>' in resp.text
    assert '>Data Source</label>' in resp.text
    # check the "data source" field is under additional parameters
    assert resp.text.index('<legend>Additional parameters</legend>') < resp.text.index('>Data Source</label>')

    # start filling the "data source" field
    resp.forms[0]['data_source$type'] = 'json'
    resp.forms[0]['data_source$value'] = 'http://example.net'
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()

    # it should now appear before the additional parameters section
    resp = resp.click('Edit', href='1/')
    assert resp.text.index('<legend>Additional parameters</legend>') > resp.text.index('>Data Source</label>')

    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.forms[0]['label'].value == '1st field'
    resp.forms[0]['prefill$type'] = 'User Field'
    resp.forms[0]['prefill$value_user'] = 'Email (builtin)'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    resp = resp.follow()
    assert (
        "&quot;1st field&quot; is not an email field. Are you sure you want to prefill it with user's email?"
        in resp.text
    )

    formdef.fields += [fields.EmailField(id='2', label='2nd field')]
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/2/')
    assert resp.forms[0]['label'].value == '2nd field'
    resp.forms[0]['prefill$type'] = 'User Field'
    resp.forms[0]['prefill$value_string'] = 'email'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_2'
    resp = resp.follow()
    assert "Are you sure you want to prefill" not in resp.text


def test_form_prefill_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'] = 'String / Template'
    resp.form['prefill$value_string'] = 'test'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].prefill == {'type': 'string', 'value': 'test', 'locked': False}

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'] = 'Python Expression'
    resp.form['prefill$value_formula'] = 'True'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].prefill == {'type': 'formula', 'value': 'True', 'locked': False}

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'] = 'String / Template'
    resp.form['prefill$value_string'] = '{{form_var_toto}}'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].prefill == {
        'type': 'string',
        'value': '{{form_var_toto}}',
        'locked': False,
    }

    # check error handling
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'] = 'Python Expression'
    resp.form['prefill$value_formula'] = ':'
    resp = resp.form.submit('submit')
    assert 'invalid expression: unexpected EOF while parsing' in resp.text

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'] = 'String / Template'
    resp.form['prefill$value_string'] = '{% if %}'
    resp = resp.form.submit('submit')
    assert 'syntax error in Django template: Unexpected end of expression' in resp.text


def test_form_edit_string_field_validation(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='1st field', type='string')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click('Edit', href='1/')
    resp.form['validation$type'] = 'Regular Expression'
    resp.form['validation$value_regex'] = r'\d+'
    resp.form['validation$error_message'] = 'Foo Error'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].validation == {
        'type': 'regex',
        'value': r'\d+',
        'error_message': 'Foo Error',
    }

    resp = resp.click('Edit', href='1/')
    resp.form['validation$type'] = 'None'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].validation is None

    resp = resp.click('Edit', href='1/')
    resp.form['validation$type'] = 'Django Condition'
    resp.form['validation$value_django'] = 'value|decimal < 20'
    resp.form['validation$error_message'] = 'Bar Error'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].validation == {
        'type': 'django',
        'value': 'value|decimal < 20',
        'error_message': 'Bar Error',
    }

    resp = resp.click('Edit', href='1/')
    resp.form['validation$type'] = 'Django Condition'
    resp.form['validation$value_django'] = '{{ value|decimal < 20 }}'
    resp = resp.form.submit('submit')
    assert 'syntax error' in resp.text


def test_form_edit_item_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.ItemField(id='1', label='1st field', type='item')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == '1st field'
    resp.forms[0]['label'] = 'changed field'
    resp.forms[0]['required'] = False
    resp = resp.forms[0].submit('items$add_element')
    # this adds a second field
    assert 'items$element0' in resp.form.fields
    assert 'items$element1' in resp.form.fields
    # but don't fill anything
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'
    resp = resp.follow()

    assert FormDef.get(1).fields[0].label == 'changed field'
    assert FormDef.get(1).fields[0].required is False
    assert FormDef.get(1).fields[0].items is None

    # edit and fill with one item
    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == 'changed field'
    resp.forms[0]['items$element0'] = 'XXX'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'
    assert FormDef.get(1).fields[0].items == ['XXX']


def test_form_edit_item_field_data_source(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.ItemField(id='1', label='1st field', type='item')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.form['data_source$type'].options == [
        (u'None', True, u'None'),
        (u'json', False, u'JSON URL'),
        (u'jsonp', False, u'JSONP URL'),
        (u'python', False, u'Python Expression'),
    ]
    resp = resp.form.submit('submit').follow()

    data_source = NamedDataSource(name='Foobar')
    data_source.data_source = {'type': 'formula', 'value': '[]'}
    data_source.store()

    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.form['data_source$type'].options == [
        (u'None', True, u'None'),
        (u'foobar', False, u'Foobar'),
        (u'json', False, u'JSON URL'),
        (u'jsonp', False, u'JSONP URL'),
        (u'python', False, u'Python Expression'),
    ]
    resp.form['data_mode'].value = 'data-source'
    resp.form['data_source$type'].value = 'foobar'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].data_source == {'type': 'foobar'}

    carddef = CardDef()
    carddef.name = 'Baz'
    carddef.store()

    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.form['data_source$type'].options == [
        (u'None', False, u'None'),
        (u'foobar', True, u'Foobar'),
        (u'json', False, u'JSON URL'),
        (u'jsonp', False, u'JSONP URL'),
        (u'python', False, u'Python Expression'),
    ]

    carddef.digest_template = 'plop'
    carddef.store()

    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.form['data_source$type'].options == [
        (u'None', False, u'None'),
        (u'carddef:%s' % carddef.url_name, False, u'Baz'),
        (u'foobar', True, u'Foobar'),
        (u'json', False, u'JSON URL'),
        (u'jsonp', False, u'JSONP URL'),
        (u'python', False, u'Python Expression'),
    ]

    resp.form['data_source$type'].value = 'carddef:%s' % carddef.url_name
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].data_source == {'type': 'carddef:%s' % carddef.url_name}

    # set json source then back to none
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['data_source$type'].value = 'json'
    resp.form['data_source$value'].value = 'http://whatever'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(formdef.id).fields[0].data_source == {'type': 'json', 'value': 'http://whatever'}

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['data_source$type'].value = 'None'
    resp = resp.form.submit('submit').follow()
    resp = app.get('/backoffice/forms/1/')

    # change configuration for items
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['data_mode'].value = 'simple-list'
    resp.form['items$element0'] = 'XXX'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(1).fields[0].data_source is None
    assert FormDef.get(1).fields[0].items == ['XXX']


def test_form_edit_item_field_geojson_data_source(pub, http_requests):
    NamedDataSource.wipe()
    create_superuser(pub)
    create_role(pub)

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'geojson',
        'value': 'http://remote.example.net/geojson',
    }
    data_source.id_property = 'id'
    data_source.label_template_property = '{{ text }}'
    data_source.cache_duration = '5'
    data_source.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.ItemField(id='1', label='1st field', type='item')]
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['display_mode'] = 'map'
    assert resp.pyquery('option[value=foobar][data-type=geojson]')
    resp.form['data_mode'] = 'data-source'
    resp.form['data_source$type'] = 'foobar'
    resp.form['min_zoom'] = 'Wide area'
    resp.form['max_zoom'] = 'Small road'
    resp = resp.form.submit('submit').follow()
    formdef = FormDef.get(formdef.id)
    assert formdef.fields[0].data_source == {'type': 'foobar'}
    assert formdef.fields[0].min_zoom == '9'

    resp = app.get('/backoffice/forms/1/fields/1/')
    assert resp.form['min_zoom'].value == 'Wide area'


def test_form_edit_items_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.ItemsField(id='1', label='1st field', type='items')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert '1st field' in resp.text

    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == '1st field'
    assert resp.forms[0]['min_choices'].value == '0'
    assert resp.forms[0]['max_choices'].value == '0'
    resp.forms[0]['label'] = 'changed field'
    resp.forms[0]['required'] = False
    resp = resp.forms[0].submit('items$add_element')
    # this adds a second field
    assert 'items$element0' in resp.form.fields
    assert 'items$element1' in resp.form.fields
    # but don't fill anything
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'
    resp = resp.follow()

    assert FormDef.get(1).fields[0].label == 'changed field'
    assert FormDef.get(1).fields[0].required is False
    assert FormDef.get(1).fields[0].items is None
    assert FormDef.get(1).fields[0].min_choices == 0
    assert FormDef.get(1).fields[0].max_choices == 0

    # edit and fill with one item
    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['label'].value == 'changed field'
    resp.forms[0]['items$element0'] = 'XXX'
    resp.forms[0]['min_choices'] = 2
    resp.forms[0]['max_choices'] = 5
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/#itemId_1'
    assert FormDef.get(1).fields[0].items == ['XXX']
    assert FormDef.get(1).fields[0].min_choices == 2
    assert FormDef.get(1).fields[0].max_choices == 5

    # check prefilling is only possible with Python
    resp = resp.follow()
    resp = resp.click('Edit', href='1/')
    assert resp.forms[0]['prefill$type'].options == [
        (u'None', True, u'None'),
        (u'Python Expression', False, u'Python Expression'),
    ]

    # change configuration for datasource
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['data_mode'].value = 'data-source'
    resp.form['data_source$type'].value = 'json'
    resp.form['data_source$value'].value = 'http://whatever'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(1).fields[0].data_source == {'type': 'json', 'value': 'http://whatever'}

    # change configuration for items
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['data_mode'].value = 'simple-list'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(1).fields[0].data_source is None
    assert FormDef.get(1).fields[0].items == ['XXX']


def test_form_edit_page_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'There are not yet any fields for this form' in resp.text

    resp.forms[0]['label'] = 'foobar'
    resp.forms[0]['type'] = 'page'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    resp = resp.follow()
    assert 'Page #1' in resp.text
    assert 'foobar' in resp.text
    assert 'Use drag and drop' in resp.text

    assert len(FormDef.get(1).fields) == 1
    assert FormDef.get(1).fields[0].key == 'page'
    assert FormDef.get(1).fields[0].label == 'foobar'

    resp = resp.click('Edit', href='1/')
    resp.form['post_conditions$element0$condition$type'] = 'python'
    resp.form['post_conditions$element0$condition$value_python'] = 'foo'
    resp.form['post_conditions$element0$error_message'] = 'bar'
    resp = resp.form.submit('post_conditions$add_element')
    resp.form['post_conditions$element1$condition$type'] = 'python'
    resp.form['post_conditions$element1$condition$value_python'] = 'foo2'
    resp = resp.form.submit('submit')
    assert 'Both condition and error message are required.' in resp.text
    resp.form['post_conditions$element1$error_message'] = 'bar2'
    resp = resp.form.submit('submit').follow()

    assert FormDef.get(1).fields[0].post_conditions == [
        {'condition': {'type': 'python', 'value': 'foo'}, 'error_message': 'bar'},
        {'condition': {'type': 'python', 'value': 'foo2'}, 'error_message': 'bar2'},
    ]

    resp = resp.click('Edit', href='1/')
    resp.form['post_conditions$element1$condition$type'] = 'django'
    resp.form['post_conditions$element1$condition$value_django'] = 'foo3'
    resp = resp.form.submit('submit').follow()
    assert FormDef.get(1).fields[0].post_conditions == [
        {'condition': {'type': 'python', 'value': 'foo'}, 'error_message': 'bar'},
        {'condition': {'type': 'django', 'value': 'foo3'}, 'error_message': 'bar2'},
    ]


def test_form_edit_comment_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.CommentField(id='1', label='a comment field', type='comment')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert 'a comment field' in resp.text
    assert 'WysiwygTextWidget' in resp.text

    # legacy, double line breaks will be converted to paragraphs
    formdef.fields = [fields.CommentField(id='1', type='comment', label='a comment field\n\na second line')]
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert 'WysiwygTextWidget' in resp.text
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].label == '<p>a comment field</p>\n<p>a second line</p>'

    # starting with a <
    formdef.fields = [
        fields.CommentField(id='1', type='comment', label='<strong>a comment field\n\na second line</strong>')
    ]
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert 'WysiwygTextWidget' in resp.text

    # legacy, ezt syntax in a non-html field will be presented as a textarea
    formdef.fields = [fields.CommentField(id='1', type='comment', label='[if-any toto]hello world[end]')]
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert 'WysiwygTextWidget' not in resp.text

    # check a new field is created with label as HTML, enclosing label in <p>
    resp = app.get('/backoffice/forms/1/fields/')
    resp.forms[0]['label'] = 'foobar'
    resp.forms[0]['type'] = 'comment'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    assert FormDef.get(formdef.id).fields[-1].label == '<p>foobar</p>'

    # unless label is already given as HTML
    resp = app.get('/backoffice/forms/1/fields/')
    resp.forms[0]['label'] = '<div>blah</div>'
    resp.forms[0]['type'] = 'comment'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/forms/1/fields/'
    assert FormDef.get(formdef.id).fields[-1].label == '<div>blah</div>'


def test_form_comment_field_textwidget_validation(pub):
    create_superuser(pub)
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    # legacy, ezt syntax in a non-html field will be presented as a textarea
    formdef.fields = [fields.CommentField(id='1', type='comment', label='[if-any toto]hello world[end]')]
    formdef.store()
    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/fields/1/')

    # bad {% %} Django template syntax
    assert 'WysiwygTextWidget' not in resp.text
    resp.form.fields['label'][0].value = '{% if cond %}no endif provided'
    resp = resp.form.submit('submit')
    assert 'syntax error in Django template: Unclosed tag on line 1' in resp.text

    # bad {{ }} Django template syntax
    assert 'WysiwygTextWidget' not in resp.text
    resp.form.fields['label'][0].value = '{{0+0}}'
    resp = resp.form.submit('submit')
    assert 'syntax error in Django template: Could not parse' in resp.text

    # bad EZT syntax
    assert 'WysiwygTextWidget' not in resp.text
    resp.form.fields['label'][0].value = '[end]'
    resp = resp.form.submit('submit')
    assert 'syntax error in ezt template: unmatched [end]' in resp.text

    # good syntax
    assert 'WysiwygTextWidget' not in resp.text
    resp.form.fields['label'][0].value = '{{variable}}'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].label == '{{variable}}'


def test_form_comment_field_wysiwygtextwidget_validation(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.CommentField(id='1', label='a comment field', type='comment')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/fields/1/')
    assert 'a comment field' in resp.text

    # bad {% %} Django template syntax
    assert 'WysiwygTextWidget' in resp.text
    resp.form.fields['label'][0].value = '{% if cond %}no endif provided'
    resp = resp.form.submit('submit')
    assert 'syntax error in Django template: Unclosed tag on line 1' in resp.text

    # bad {{ }} Django template syntax
    assert 'WysiwygTextWidget' in resp.text
    resp.form.fields['label'][0].value = '{{0+0}}'
    resp = resp.form.submit('submit')
    assert 'syntax error in Django template: Could not parse' in resp.text

    # bad EZT syntax
    assert 'WysiwygTextWidget' in resp.text
    resp.form.fields['label'][0].value = '[end]'
    resp = resp.form.submit('submit')
    assert 'syntax error in ezt template: unmatched [end]' in resp.text

    # good syntax
    assert 'WysiwygTextWidget' in resp.text
    resp.form.fields['label'][0].value = '{{variable}}'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].label == '{{variable}}'


def test_form_edit_map_field(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.MapField(id='1', label='a field', type='map')]
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/forms/1/fields/1/')
    resp = resp.form.submit('submit')
    assert resp.location

    # min
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['min_zoom'] = 'Wide area'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].min_zoom == '9'

    # max
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['max_zoom'] = 'Small road'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].max_zoom == '16'

    # both
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['min_zoom'] = 'Wide area'
    resp.form['max_zoom'] = 'Small road'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].min_zoom == '9'
    assert FormDef.get(formdef.id).fields[0].max_zoom == '16'

    # inverted
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['min_zoom'] = 'Small road'
    resp.form['max_zoom'] = 'Wide area'
    resp = resp.form.submit('submit')
    assert 'widget-with-error' in resp.text

    # initial out of range
    formdef.store()
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['initial_zoom'] = 'Whole world'
    resp.form['min_zoom'] = 'Wide area'
    resp.form['max_zoom'] = 'Small road'
    resp = resp.form.submit('submit')
    assert 'widget-with-error' in resp.text

    # prefill fields
    resp = app.get('/backoffice/forms/1/fields/1/')
    resp.form['prefill$type'].value = 'Geolocation'
    resp.form['prefill$value_geolocation'].value = 'Position'
    resp = resp.form.submit('submit')
    assert FormDef.get(formdef.id).fields[0].prefill == {
        'type': 'geolocation',
        'value': 'position',
        'locked': False,
    }


def test_form_edit_field_warnings(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.StringField(id='%d' % i, label='field %d' % i, type='string') for i in range(1, 10)
    ]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'more than 500 fields' not in resp.text
    assert 'first field should be of type "page"' not in resp.text

    formdef.fields.append(fields.PageField(id='1000', label='page', type='page'))
    formdef.store()
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'more than 500 fields' not in resp.text
    assert 'first field should be of type "page"' in resp.text

    formdef.fields.extend(
        [fields.StringField(id='%d' % i, label='field %d' % i, type='string') for i in range(10, 510)]
    )
    formdef.store()
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='fields/')
    assert 'more than 500 fields' in resp.text
    assert 'first field should be of type "page"' in resp.text

    FormDef.wipe()


def test_form_limit_display_to_page(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', type='string', varname='foobar'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2', type='string', varname='baz'),
    ]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/fields/')
    assert '{{form_var_foobar}}' in resp.text
    assert '2nd page' in resp.text
    resp = resp.click('Limit display to this page', index=0)
    hidden_fields = ''.join(re.findall('display:none.*', resp.text))
    assert 'Display all pages' in resp.text
    assert '{{form_var_foobar}}' not in hidden_fields
    assert '{{form_var_baz}}' in hidden_fields


def test_form_fields_reorder(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', type='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2', type='string'),
    ]
    formdef.store()

    app = login(get_app(pub))

    # missing element in params: do nothing
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=0;3;1;2;' % formdef.id)
    assert resp.json == {'success': 'ko'}
    # missing order in params: do nothing
    resp = app.get('/backoffice/forms/%s/fields/update_order?element=0' % formdef.id)
    assert resp.json == {'success': 'ko'}

    resp = app.get('/backoffice/forms/%s/fields/update_order?order=0;3;1;2;&element=3' % formdef.id)
    assert resp.json == {'success': 'ok'}
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['0', '3', '1', '2']

    # unknown id: ignored
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=0;1;2;3;4;&element=3' % formdef.id)
    assert resp.json == {'success': 'ok'}
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['0', '1', '2', '3']
    # missing id: do nothing
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=0;3;1;&element=3' % formdef.id)
    assert resp.json == {'success': 'ko'}
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['0', '1', '2', '3']

    # move a page
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=2;0;1;3;&element=2' % formdef.id)
    assert resp.json == {
        'success': 'ok',
        'additional-action': {
            'message': 'Also move the fields of the page',
            'url': 'move_page_fields?fields=3&page=2',
        },
    }
    # reset
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=0;1;2;3;&element=2' % formdef.id)
    assert resp.json == {'success': 'ok'}
    # move the first page
    resp = app.get('/backoffice/forms/%s/fields/update_order?order=1;2;3;0;&element=0' % formdef.id)
    assert resp.json == {
        'success': 'ok',
        'additional-action': {
            'message': 'Also move the fields of the page',
            'url': 'move_page_fields?fields=1&page=0',
        },
    }


def test_form_move_page_fields(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', type='string'),
        fields.StringField(id='3', label='string 2', type='string'),
    ]
    formdef.store()

    app = login(get_app(pub))
    # missing element in params: do nothing
    app.get('/backoffice/forms/%s/fields/move_page_fields?fields=3' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['2', '0', '1', '3']
    # missing order in params: do nothing
    app.get('/backoffice/forms/%s/fields/move_page_fields?page=2' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['2', '0', '1', '3']

    # unknown id: do nothing
    app.get('/backoffice/forms/%s/fields/move_page_fields?fields=4&page=2' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['2', '0', '1', '3']

    # move the fields of the page
    app.get('/backoffice/forms/%s/fields/move_page_fields?fields=3&page=2' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['2', '3', '0', '1']

    # move the new first page
    app.get('/backoffice/forms/%s/fields/update_order?order=3;0;1;2&element=2' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['3', '0', '1', '2']
    # and the fields
    app.get('/backoffice/forms/%s/fields/move_page_fields?fields=3&page=2' % formdef.id)
    formdef = FormDef.get(formdef.id)
    assert [x.id for x in formdef.fields] == ['0', '1', '2', '3']


def test_form_legacy_int_id(pub):
    create_superuser(pub)
    create_role(pub)

    Category.wipe()
    cat = Category(name='Foo')
    cat.store()

    Workflow.wipe()
    workflow = Workflow(name='Workflow One')
    # create it with a single status
    workflow.possible_status = [Workflow.get_default_workflow().possible_status[-1]]
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []

    role = pub.role_class(name='ZAB')  # Z to get sorted last
    role.store()

    # set attributes using integers
    formdef.category_id = int(cat.id)
    formdef.workflow_id = int(workflow.id)
    formdef.workflow_roles = {'_receiver': int(role.id)}
    formdef.roles = ['logged-users', int(role.id)]

    formdef.store()

    formdef = FormDef.get(formdef.id)  # will run migrate

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')

    resp = resp.click(href='category')
    assert resp.forms[0]['category_id'].value

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='workflow', index=1)
    assert resp.forms[0]['workflow_id'].value

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('User Roles')
    assert resp.forms[0]['roles$element0'].value == 'logged-users'
    assert resp.forms[0]['roles$element1'].value == role.id

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('Recipient')
    assert resp.forms[0]['role_id'].value == role.id


def test_form_anonymise(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()
    for i in range(10):
        formdata = data_class()
        formdata.just_created()
        if i < 3:
            formdata.status = 'wf-new'
        else:
            formdata.status = 'wf-rejected'
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='anonymise')
    resp = resp.form.submit('cancel')
    assert len([x for x in formdef.data_class().select() if x.anonymised]) == 0
    assert resp.location == 'http://example.net/backoffice/forms/1/'

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='anonymise')
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    assert len([x for x in formdef.data_class().select() if x.anonymised]) == 0

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='anonymise')
    resp.form['before_request_date'].value = (
        datetime.datetime.today() + datetime.timedelta(days=1)
    ).strftime('%Y-%m-%d')
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/backoffice/forms/1/'
    assert len([x for x in formdef.data_class().select() if x.anonymised]) == 7

    data_class.wipe()
    for i in range(110):
        formdata = data_class()
        formdata.just_created()
        if i < 3:
            formdata.status = 'wf-rejected'
        else:
            formdata.status = 'wf-finished'
        formdata.store()

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='anonymise')
    resp.form['before_request_date'].value = (
        datetime.datetime.today() + datetime.timedelta(days=1)
    ).strftime('%Y-%m-%d')
    resp.form['endpoints$elementfinished'].checked = False
    resp = resp.form.submit('submit')
    assert '?job=' in resp.location
    resp = resp.follow()
    assert 'job-status' in resp.text
    assert 'completed' in resp.text
    assert len([x for x in formdef.data_class().select() if x.anonymised]) == 3


def test_form_public_url(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click('Display public URL')
    assert 'http://example.net/form-title/' in resp.text


def test_form_archive(pub):
    create_superuser(pub)
    create_role(pub)

    if pub.is_using_postgresql():
        # this doesn't exist in SQL
        pytest.skip('no archive in SQL mode')
        return

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = []
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()
    for i in range(10):
        formdata = data_class()
        formdata.just_created()
        if i < 3:
            formdata.status = 'wf-new'
        else:
            formdata.status = 'wf-rejected'
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='archive')
    resp = resp.form.submit('cancel')
    assert data_class.count() == 10
    assert resp.location == 'http://example.net/backoffice/forms/1/'

    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='archive')
    resp = resp.form.submit('submit')
    assert resp.content_type == 'application/x-wcs-archive'
    tf = tarfile.open(fileobj=io.BytesIO(resp.body))
    assert 'formdef' in [x.name for x in tf.getmembers()]
    assert len(tf.getmembers()) == 8  # 7 formdata + 1 formdef

    # second archive, it shouldn't get anything (but the formdef)
    resp = app.get('/backoffice/forms/1/')
    resp = resp.click(href='archive')
    resp = resp.form.submit('submit')
    assert resp.content_type == 'application/x-wcs-archive'
    tf = tarfile.open(fileobj=io.BytesIO(resp.body))
    assert 'formdef' in [x.name for x in tf.getmembers()]
    assert len(tf.getmembers()) == 1  # 0 formdata + 1 formdef


def test_form_overwrite(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form test'
    formdef.table_name = 'xxx'
    formdef.fields = [
        fields.StringField(id='1', label='1st field', type='string'),
        fields.StringField(id='2', label='2nd field', type='string'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'foo', '2': 'bar'}
    formdata.just_created()
    formdata.store()

    formdef_id = formdef.id
    formdef.fields[0].label = '1st modified field'
    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef_id)
    resp = resp.click(href='overwrite')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.forms[0].submit()
    assert 'Overwrite - Summary of changes' in resp.text
    resp = resp.forms[0].submit()
    assert FormDef.get(formdef_id).fields[0].label == '1st modified field'
    resp = resp.follow()
    assert 'The form has been successfully overwritten.' in resp.text

    # check with added/removed field
    new_formdef = FormDef()
    new_formdef.name = 'form test overwrite'
    new_formdef.fields = [
        fields.StringField(id='2', label='2nd field', type='string'),
        fields.StringField(id='3', label='3rd field', type='string'),
    ]
    new_formdef_xml = ET.tostring(new_formdef.export_to_xml(include_id=True))

    # and no data within
    formdef.data_class().wipe()
    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef_id)
    resp = resp.click(href='overwrite')
    resp.forms[0]['file'] = Upload('formdef.wcs', new_formdef_xml)
    resp = resp.forms[0].submit()
    assert FormDef.get(formdef_id).fields[0].id == '2'
    assert FormDef.get(formdef_id).fields[0].label == '2nd field'
    assert FormDef.get(formdef_id).fields[1].id == '3'
    assert FormDef.get(formdef_id).fields[1].label == '3rd field'

    # and data within
    formdef.store()
    formdata.data = {'1': 'foo', '2': 'bar'}
    formdata.just_created()
    formdata.store()

    resp = app.get('/backoffice/forms/%s/' % formdef_id)
    resp = resp.click(href='overwrite')
    resp.forms[0]['file'] = Upload('formdef.wcs', new_formdef_xml)
    resp = resp.forms[0].submit()
    assert 'The form removes or changes fields' in resp.text
    assert resp.forms[0]['force'].checked is False
    resp = resp.forms[0].submit()  # without checkbox (back to same form)
    resp.forms[0]['force'].checked = True
    resp = resp.forms[0].submit()

    assert FormDef.get(formdef_id).fields[0].id == '2'
    assert FormDef.get(formdef_id).fields[0].label == '2nd field'
    assert FormDef.get(formdef_id).fields[1].id == '3'
    assert FormDef.get(formdef_id).fields[1].label == '3rd field'

    # check with a field of different type
    formdef = FormDef.get(formdef_id)
    formdata = formdef.data_class()()
    formdata.data = {'1': 'foo', '2': 'bar', '3': 'baz'}
    formdata.just_created()
    formdata.store()

    new_formdef = FormDef()
    new_formdef.name = 'form test overwrite'
    new_formdef.fields = [
        fields.StringField(id='2', label='2nd field', type='string'),
        fields.DateField(id='3', label='3rd field, date', type='date'),
    ]  # (string -> date)
    new_formdef_xml = ET.tostring(new_formdef.export_to_xml(include_id=True))

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef_id)
    resp = resp.click(href='overwrite', index=0)
    resp.forms[0]['file'] = Upload('formdef.wcs', new_formdef_xml)
    resp = resp.forms[0].submit()
    assert 'The form removes or changes fields' in resp.text
    resp.forms[0]['force'].checked = True
    resp = resp.forms[0].submit()
    assert FormDef.get(formdef_id).fields[1].id == '3'
    assert FormDef.get(formdef_id).fields[1].label == '3rd field, date'
    assert FormDef.get(formdef_id).fields[1].type == 'date'

    # check we kept stable references
    assert FormDef.get(formdef_id).url_name == 'form-test'
    assert FormDef.get(formdef_id).table_name == 'xxx'

    # check existing data
    data = FormDef.get(formdef_id).data_class().get(formdata.id).data
    assert data.get('2') == 'bar'
    if pub.is_using_postgresql():
        # in SQL, check data with different type has been removed
        assert data.get('3') is None


def test_form_export_import_export_overwrite(pub):
    create_superuser(pub)
    create_role(pub)

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.table_name = 'xxx'
    formdef.fields = [  # unordered id
        fields.StringField(id='1', label='field 1', type='string'),
        fields.DateField(id='12', label='field 2', type='date'),
        fields.ItemField(id='4', label='field 3', type='item'),
    ]
    formdef.store()

    # add data
    formdata = formdef.data_class()()
    formdata.data = {'1': 'foo'}
    formdata.just_created()
    formdata.store()

    formdef_xml = ET.tostring(formdef.export_to_xml(include_id=True))

    assert FormDef.count() == 1
    assert formdef.url_name == 'form-title'

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/')
    resp = resp.click(href='import')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef_xml)
    resp = resp.forms[0].submit()
    assert FormDef.count() == 2

    formdef2 = FormDef.get(2)
    assert formdef2.url_name == 'form-title-1'
    # fields are imported with original ids
    for i, field in enumerate(formdef.fields):
        field2 = formdef2.fields[i]
        assert (field.id, field.label, field.type) == (field2.id, field2.label, field2.type)

    # modify imported formdef, then overwrite original formdef with it
    formdef2.fields.insert(2, fields.StringField(id='2', label='field 4', type='string'))
    formdef2.fields.insert(3, fields.DateField(id='3', label='field 5', type='date'))
    formdef2.fields.append(fields.ItemField(id='5', label='field 6', type='item'))
    formdef2.store()
    formdef2_xml = ET.tostring(formdef2.export_to_xml(include_id=True))
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click(href='overwrite')
    resp.forms[0]['file'] = Upload('formdef.wcs', formdef2_xml)
    resp = resp.forms[0].submit()
    assert 'Overwrite - Summary of changes' in resp.text
    resp = resp.forms[0].submit()
    formdef_overwrited = FormDef.get(formdef.id)
    for i, field in enumerate(formdef2.fields):
        field_ow = formdef_overwrited.fields[i]
        assert (field.id, field.label, field.type) == (field_ow.id, field_ow.label, field_ow.type)


def test_form_comment_with_error_in_wscall(http_requests, pub):
    create_superuser(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall(name='xxx')
    wscall.description = 'description'
    wscall.request = {
        'url': 'http://remote.example.net/404',
        'request_signature_key': 'xxx',
        'qs_data': {'a': 'b'},
        'method': 'POST',
        'post_data': {'c': 'd'},
    }
    wscall.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.CommentField(id='1', label='x [webservice.xxx.foobar] x', type='comment')]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert 'x [webservice.xxx.foobar] x' in resp.text