wcs/tests/form_pages/test_all.py

import hashlib
import io
import json
import os
import re
import time
import urllib.parse
import xml.etree.ElementTree as ET
import zipfile
from unittest import mock

import pytest
from django.utils.encoding import force_bytes, force_text
from webtest import Hidden, Radio, Upload

from wcs import fields
from wcs.carddef import CardDef
from wcs.categories import Category
from wcs.data_sources import NamedDataSource
from wcs.formdef import FormDef
from wcs.forms.root import PublicFormStatusPage
from wcs.qommon import force_str
from wcs.qommon.emails import docutils
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.qommon.misc import ConnectionError
from wcs.roles import logged_users_role
from wcs.tracking_code import TrackingCode
from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
from wcs.wf.create_formdata import CreateFormdataWorkflowStatusItem, Mapping
from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
from wcs.wf.jump import JumpWorkflowStatusItem
from wcs.wf.redirect_to_url import RedirectToUrlWorkflowStatusItem
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
from wcs.wf.resubmit import ResubmitWorkflowStatusItem
from wcs.wf.wscall import JournalWsCallErrorPart
from wcs.workflows import (
    ChoiceWorkflowStatusItem,
    CommentableWorkflowStatusItem,
    DisplayMessageWorkflowStatusItem,
    EditableWorkflowStatusItem,
    JumpOnSubmitWorkflowStatusItem,
    Workflow,
    WorkflowBackofficeFieldsFormDef,
    WorkflowVariablesFieldsFormDef,
)

from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login


def assert_equal_zip(stream1, stream2):
    with zipfile.ZipFile(stream1) as z1, zipfile.ZipFile(stream2) as z2:
        assert set(z1.namelist()) == set(z2.namelist())
        for name in z1.namelist():
            if name == 'styles.xml':
                continue
            if name in ['content.xml', 'meta.xml']:
                t1, t2 = ET.tostring(ET.XML(z1.read(name))), ET.tostring(ET.XML(z2.read(name)))
                try:
                    # >= python 3.8: tostring preserves attribute order; use canonicalize to sort them
                    t1, t2 = ET.canonicalize(t1), ET.canonicalize(t2)
                except AttributeError:
                    pass
            else:
                t1, t2 = z1.read(name), z2.read(name)
            assert t1 == t2, 'file "%s" differs' % name


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates', 'pickle-lazy'], indirect=True)


@pytest.fixture
def pub(request, emails):
    pub = create_temporary_pub(
        sql_mode=bool('sql' in request.param),
        templates_mode=bool('templates' in request.param),
        lazy_mode=bool('lazy' in request.param),
    )
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    if Category.count() == 0:
        cat = Category(name='foobar')
        cat.store()

    return pub


@pytest.fixture
def error_email(pub):
    pub.cfg['debug'] = {'error_email': 'errors@localhost.invalid'}
    pub.write_cfg()
    pub.set_config()


def teardown_module(module):
    clean_temporary_pub()


def create_formdef():
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()
    return formdef


def create_user(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.name = 'User Name'
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()
    return user


def create_user_and_admin(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()

    admin = pub.user_class()
    admin.email = 'admin@localhost'
    admin.is_admin = True
    admin.store()
    account = PasswordAccount(id='admin')
    account.set_password('admin')
    account.user_id = admin.id
    account.store()
    return user, admin


def test_home(pub):
    create_formdef()
    home = get_app(pub).get('/')
    assert 'category-misc' in home.text
    assert '<a class="" href="test/">test</a>' in home.text


def test_home_with_user_forms(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.category_id = '1'
    wf = Workflow(name='status')
    wf.add_status('Status1', 'st1')
    wf.store()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-st1'
    formdata.data = {}
    formdata.store()
    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="/test/%s/"' % formdata.id in resp
    assert 'Draft' in resp
    assert '<a href="/test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s/' % draft.id, status=302)
    resp = resp.follow(status=200)

    # disable formdef: formdatas are still visible and accessible, drafts are not
    formdef.disabled = True
    formdef.store()
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="/test/%s/"' % formdata.id in resp
    assert 'Draft' not in resp
    assert '<a href="test/%s"' % draft.id not in resp
    resp = app.get('/test/%s/' % formdata.id)
    resp = app.get('/test/%s/' % draft.id, status=302)
    resp = resp.follow(status=403)


def test_home_category(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'category-foobar' in home.text
    assert 'category-misc' not in home.text
    assert '<a class="" href="foobar/test/">test</a>' in home.text


def test_home_two_categories(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'foobar'
    formdef2.fields = []
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text  # 1st formdef
    assert 'category-misc' in resp.text  # 2nd formdef, fake category
    assert '<a class="" href="foobar/test/">test</a>' in resp.text

    cat2 = Category(name='barfoo')
    cat2.store()
    formdef2.category_id = cat2.id
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text  # 1st formdef
    assert 'category-barfoo' in resp.text  # 2nd formdef
    assert 'category-misc' not in resp.text  # no more "misc" category


def test_home_keywords(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.keywords = 'hello, world'
    formdef.store()
    home = get_app(pub).get('/')
    assert home.html.find('div', {'data-keywords': 'hello world'}) or home.html.find(
        'div', {'data-keywords': 'world hello'}
    )
    assert home.html.find('li', {'data-keywords': 'hello world'}) or home.html.find(
        'li', {'data-keywords': 'world hello'}
    )


def test_home_formdef_description(pub):
    formdef = create_formdef()
    formdef.description = 'HELLO WORLD'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'HELLO WORLD' in home.text
    assert '<a class="" href="test/">test</a>' in home.text


def test_home_disabled(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' not in home.text

    # check access is denied
    get_app(pub).get('/test/', status=403)


def test_home_disabled_with_redirect(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.disabled_redirection = 'http://example.org'
    formdef.store()
    resp = get_app(pub).get('/')
    assert '<a class="redirection" href="test/">test</a>' in resp.text
    resp = resp.click('test')
    assert resp.location == 'http://example.org'


def test_home_inaccessible(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.store()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://example.net/login/?next=http%3A%2F%2Fexample.net%2F'


def test_home_always_advertise(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.always_advertise = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' in home.text
    assert '<a href="test/">test</a><span> (authentication required)</span>' in home.text


def test_home_redirect(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/'


def test_home_redirect_var(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/[site_lang]/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/en/'


def test_category_page(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    resp = get_app(pub).get('/foobar/')
    assert '<h2>foobar</h2>' in resp.text
    assert '<a class="" href="test/">test</a>' in resp.text


def test_category_page_redirect(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/'


def test_category_page_redirect_var(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/[site_lang]/[category_slug]/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/en/foobar/'


def test_form_access(pub):
    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    # check a formdef protected by a role cannot be accessed
    formdef.roles = [role.id]
    formdef.store()
    # an unlogged user will ge ta redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # while a logged-in user will get a 403
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=403)

    # unless the user has the right role
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check admin has access, even without specific roles
    user = create_user(pub)
    user.roles = []
    user.is_admin = True
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check special "logged users" role
    formdef.roles = [logged_users_role().id]
    formdef.store()
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)
    resp = get_app(pub).get('/test/', status=302)  # redirect to login

    # check "receiver" can also access the formdef
    formdef = create_formdef()
    formdef.roles = [-2]
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)


def test_form_access_auth_context(pub):
    create_user(pub)

    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'auth-contexts', 'fedict')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    formdef.required_authentication_contexts = ['fedict']
    formdef.roles = [logged_users_role().id]
    formdef.store()

    # an unlogged user will get a redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # a user logged in with a simple username/password tuple will get a page
    # to relogin with a stronger auth
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' in resp.text

    for session in pub.session_manager.values():
        session.saml_authn_context = 'urn:oasis:names:tc:SAML:2.0:ac:classes:SmartcardPKI'
        session.store()
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' not in resp.text
    assert resp.form


def test_form_cancelurl(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()

    # path
    resp = get_app(pub).get('/test/?cancelurl=/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/plop/'

    # full URL
    resp = get_app(pub).get('/test/?cancelurl=http://example.net/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/plop/'

    # remote site
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.add_section('api-secrets')
    pub.site_options.set('api-secrets', 'example.org', 'xyz')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/?cancelurl=http://example.org/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.org/plop/'

    pub.site_options.remove_section('api-secrets')
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.set('options', 'relatable-hosts', 'example.com')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.set('options', 'relatable-hosts', 'example.com, example.org')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/?cancelurl=http://example.org/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.org/plop/'


def test_form_submit(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert 'None' not in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text


def test_form_submit_no_confirmation(pub):
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1


def test_form_string_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert next_page.pyquery('div.error').text() == 'required field'
    next_page.forms[0]['f0'] = 'foobar'
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': 'foobar'}


def test_form_items_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='List of items',
            type='items',
            required=True,
            varname='foo',
            items=['Foo', 'Bar', 'Three', 'Four', 'Five', 'Six'],
        )
    ]
    formdef.store()
    formdef.data_class().wipe()

    page = get_app(pub).get('/test/')
    assert 'List of items' in page.text
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert next_page.pyquery('div.error').text() == 'required field'
    next_page.forms[0]['f0$element0'].checked = True
    next_page.forms[0]['f0$element1'].checked = True
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar']
    assert data.data['0_display'] == 'Foo, Bar'

    formdef.fields[0].min_choices = 2
    formdef.fields[0].max_choices = 5
    formdef.store()

    page = get_app(pub).get('/test/')
    page.forms[0]['f0$element0'].checked = True
    page = page.forms[0].submit('submit')
    assert page.pyquery('div.error').text() == 'You must select at least 2 answers.'
    page.forms[0]['f0$element1'].checked = True
    page.forms[0]['f0$element2'].checked = True
    page.forms[0]['f0$element3'].checked = True
    page.forms[0]['f0$element4'].checked = True
    page.forms[0]['f0$element5'].checked = True
    page = page.forms[0].submit('submit')
    assert page.pyquery('div.error').text() == 'You must select at most 5 answers.'
    page.forms[0]['f0$element5'].checked = False
    page = next_page.forms[0].submit('submit').follow()
    assert 'The form has been recorded' in page.text


def test_form_string_with_invalid_xml_chars(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0'] = 'hello\x0b\x0cworld'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    data = formdef.data_class().select()[0]
    assert data.data == {'0': 'helloworld'}


def test_form_submit_handling_role_info(pub):
    role = pub.role_class(name='xxx')
    role.details = 'Managing service'
    role.store()
    formdef = create_formdef()
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp
    assert 'Your case is handled by' in resp
    assert 'Managing service' in resp

    formdata = formdef.data_class().select()[0]
    formdata.jump_status('rejected')
    formdata.store()
    resp = resp.test_app.get(resp.request.url)
    assert 'Your case has been handled by' in resp


def assert_current_page(resp, page_label):
    for li_tag in resp.html.findAll('li'):
        if 'current' in li_tag.attrs['class']:
            assert li_tag.find_all('span')[-1].text == page_label


def test_form_multi_page(pub):
    for initial_condition in (None, 'True'):
        formdef = create_formdef()
        formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2'),
        ]
        formdef.fields[0].condition = {'type': 'python', 'value': initial_condition}
        formdef.store()
        page = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        page.forms[0]['f1'] = 'foo'
        assert page.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
        next_page = page.forms[0].submit('submit')
        assert_current_page(next_page, '2nd page')
        assert next_page.forms[0]['previous']
        next_page.forms[0]['f3'] = 'bar'
        next_page = next_page.forms[0].submit('submit')
        assert_current_page(next_page, 'Validating')
        assert 'Check values then click submit.' in next_page.text
        next_page = next_page.forms[0].submit('submit')
        assert next_page.status_int == 302
        next_page = next_page.follow()
        assert 'The form has been recorded' in next_page.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        data = formdef.data_class().get(data_id)
        assert data.data == {'1': 'foo', '3': 'bar'}


def test_form_multi_page_title_and_subtitle_as_template(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.TitleField(id='4', label='<i>title of second page {{ form_var_foo }}</i>', type='title'),
        fields.SubtitleField(
            id='5', label='<i>subtitle of second page {{ form_var_foo }}</i>', type='subtitle'
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '35 < 42'
    resp = resp.form.submit('submit')
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    expected_label = '&lt;i&gt;title of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="title "><h3>%s</h3></div>' % expected_label in resp.text
    expected_label = '&lt;i&gt;subtitle of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="subtitle "><h4>%s</h4></div>' % expected_label in resp.text


def test_form_multi_page_condition(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']


def test_form_multi_page_condition_select(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar']
        ),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'var_foo == "Foo"'}
        ),
        fields.PageField(
            id='3', label='3rd page', type='page', condition={'type': 'python', 'value': 'var_foo == "Bar"'}
        ),
        fields.StringField(id='4', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert '2nd page' not in resp.text
    assert '3rd page' not in resp.text
    assert resp.forms[0]['f1'].value == 'Foo'  # preset
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert '3rd page' not in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' not in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')


def test_form_multi_page_condition_select_new_varname(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar']
        ),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "Foo"'},
        ),
        fields.PageField(
            id='3',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "Bar"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert '2nd page' not in resp.text
    assert '3rd page' not in resp.text
    resp.forms[0]['f1'] = 'Foo'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert '3rd page' not in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' not in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')


def test_form_multi_page_condition_checkbox(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'].checked = True
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']
    resp.forms[0]['f1'].checked = False
    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_json_check(pub):
    # make sure the json export has no value for fields from hidden pages
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2', varname='st2'),
        fields.PageField(
            id='4',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "True"'},
        ),
        fields.StringField(id='5', label='string 3', varname='st3'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit')  # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit')  # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit()

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None


def test_form_multi_page_condition_no_confirmation_json_check(pub):
    # same as above but without the confirmation page.
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2', varname='st2'),
        fields.PageField(
            id='4',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "True"'},
        ),
        fields.StringField(id='5', label='string 3', varname='st3'),
        fields.PageField(id='6', label='4th page', type='page'),
        fields.CommentField(id='7', label='Check values then click submit.', type='comment'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit')  # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit')  # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit('submit')

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None


def test_form_multi_page_condition_data_source(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    # add the named data source, empty
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_data_source_with_form_variable(pub):
    # this tries to recreate #8272 which is about a json datasource being
    # used in a page condition and taking a value from the given page to
    # filter its content.  It is emulated here with a Python datasource
    # being empty if a field was not set.
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='xxx', required=False),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    # add the named data source, related to a field on the first page
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': 'form_var_xxx and [form_var_xxx] or []'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'HELLO'
    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_on_first_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    # should be on second page already
    with pytest.raises(AssertionError):
        resp.form.get('previous')
    resp.form['f3'] = 'foo'
    assert_current_page(resp, '2nd page')
    resp = resp.form.submit('submit')  # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')  # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('previous')  # -> 2nd page
    assert_current_page(resp, '2nd page')
    assert resp.form['f3']
    with pytest.raises(AssertionError):
        resp.form.get('previous')


def test_form_multi_page_condition_on_first_and_next(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
        fields.StringField(id='1', label='string', varname='val1'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'vars().get("form_var_val1") == "foo"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert resp.form['f3']
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo', '3': 'bar'}

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'xxx'
    resp = resp.form.submit('submit')
    with pytest.raises(AssertionError):
        assert resp.form['f3']
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data.get('1') == 'xxx'
    assert data.data.get('3') is None


def test_form_multi_page_condition_no_visible_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    get_app(pub).get('/test/', status=404)


def test_form_multi_page_many_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(
            id='0', label='2nd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
    ]

    formdef.store()
    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        get_app(pub).get('/test/')
        call_count = invalidate_cache.call_count

    for i in range(30):
        formdef.fields.append(
            fields.PageField(
                id=str(i + 2),
                label='page %s' % (i + 2),
                type='page',
                condition={'type': 'python', 'value': 'True'},
            )
        )
    formdef.store()

    # check the cache doesn't get invalidated for every page
    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        get_app(pub).get('/test/')
        assert invalidate_cache.call_count <= call_count


def test_form_multi_page_condition_stored_values(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "toto"'},
        ),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert 'bar' in resp.text
    resp = resp.form.submit('previous')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert 'bar' not in resp.text
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None

    # same without validation page
    formdef.confirmation = False
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None


def test_form_multi_page_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
        fields.StringField(id='5', label='string 3', varname='bar'),
    ]

    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foo'
    assert formdef.data_class().select()[0].data['5'] == 'bar'

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'False'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'global-errors' in resp.text
    assert 'You shall not pass.' in resp.text

    formdef.fields[4].post_conditions = [
        {
            'condition': {'type': 'python', 'value': 'form_var_foo == "foo"'},
            'error_message': 'You shall not pass.',
        },
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

    # check a post-condition raising an exception, they should always fail.
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': '1/0'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    # check a post-condition referring to a field on the same page
    formdef.fields[4].post_conditions = [
        {
            'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'},
            'error_message': 'You shall not pass.',
        },
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'foo'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text


def test_form_multi_page_conditions_and_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0',
            label='1st page',
            type='page',
            post_conditions=[
                {
                    'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'},
                    'error_message': 'You shall not pass.',
                }
            ],
        ),
        fields.StringField(id='1', label='string', varname='bar'),
        fields.PageField(id='3', label='2nd page', type='page'),
    ]

    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text

    # add a conditional page, this will cause pages to be evaluated first
    # (and would trigger #25197)
    formdef.fields.append(
        fields.PageField(id='4', label='3rd page', type='page', condition={'type': 'python', 'value': 'True'})
    )
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text


def test_form_multi_page_page_name_as_title(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2  # in steps and in main body

    # add a comment that will not be displayed and should therefore not be
    # considered.
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.CommentField(id='5', label='bla bla bla', type='comment'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2  # in steps and in main body


def test_form_submit_with_user(pub, emails):
    create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text
    # check the user received a copy by email
    assert emails.get('New form (test)')
    assert emails.get('New form (test)')['email_rcpt'] == ['foo@localhost']


def test_form_submit_with_just_disabled_user(pub, emails):
    user = create_user(pub)
    formdef = create_formdef()
    app = login(get_app(pub), username='foo', password='foo')
    formdef.data_class().wipe()
    resp = app.get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    user.is_active = False
    user.store()
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].user_id is None


def test_form_titles(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.SubtitleField(id='5', label='subtitle of 1st page', type='subtitle'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.TitleField(id='6', label='title of second page', type='title'),
        fields.StringField(id='3', label='string 2', required=False),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert '<h3 data-field-id="0">1st page/h3>' not in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp.form['f3'] = 'foo'
    resp = resp.form.submit('submit')  # -> validation page
    assert '<h3>1st page</h3>' in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<div class="title "><h3>1st page</h3></div>' not in resp.text
    assert '<div class="subtitle "><h4>subtitle of 1st page</h4></div>' in resp.text
    assert '<div class="title "><h3>title of second page</h3></div>' in resp.text


def test_form_summary_empty_pages(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='toto'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_toto == "foo"'},
        ),
        fields.TitleField(id='6', label='title in second page', type='title'),
        fields.StringField(id='3', label='string'),
        fields.PageField(id='4', label='3rd page', type='page'),
        fields.StringField(id='5', label='string'),
        fields.PageField(id='7', label='4th page', type='page'),
        fields.CommentField(id='8', label='Bla bla bla', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')
    formdata_id = resp.location.split('/')[-2]
    resp = resp.follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('previous')  # -> 1st page
    resp.form['f1'] = 'baz'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' not in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    # change condition to have second page never displayed
    formdef.fields[2].condition['value'] = False
    formdef.store()
    formdata = formdef.data_class().get(formdata_id)
    resp = app.get(formdata.get_url())
    # it was filled by user, it should still appear (conditions should not be
    # replayed)
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text


def test_form_display_locations(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string1', display_locations=[]),
        fields.StringField(id='2', label='string2', display_locations=['validation']),
        fields.StringField(id='3', label='string3', display_locations=['summary']),
        fields.CommentField(
            id='4', label='Bla bla bla', type='comment', display_locations=['validation', 'summary']
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'plop1'
    resp.form['f2'] = 'plop2'
    resp.form['f3'] = 'plop3'
    resp = resp.form.submit('submit')  # -> validation
    pq = resp.pyquery.remove_namespaces()
    assert pq('div[style="display: none;"] [name=f1]')
    assert not pq('div[style="display: none;"] [name=f2]')
    assert pq('div[style="display: none;"] [name=f3]')
    assert 'Bla bla bla' in resp.text

    resp = resp.form.submit('submit').follow()  # -> submit
    assert formdef.data_class().select()[0].data['1'] == 'plop1'
    assert formdef.data_class().select()[0].data['2'] == 'plop2'
    assert formdef.data_class().select()[0].data['3'] == 'plop3'
    assert 'plop1' not in resp.text
    assert 'plop2' not in resp.text
    assert 'plop3' in resp.text
    assert 'Bla bla bla' in resp.text


def test_multipage_form_display_locations(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string1', display_locations=[]),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(id='3', label='Bla bla bla', type='comment', display_locations=['validation']),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'plop1'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit')  # -> validation

    pq = resp.pyquery.remove_namespaces()
    assert '<h3>1st page</h3>' not in resp.text  # page 1 title not displayed
    assert pq('div[style="display: none;"] [name=f1]')  # but page 1 field included, hidden
    assert '<h3>2nd page</h3>' in resp.text  # page 2 title
    assert 'Bla bla bla' in resp.text  # and page 2 comment field


def test_form_visit_existing(pub):
    user = create_user(pub)
    formdef = create_formdef()
    login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.store()

    formdata_user = formdef.data_class()()
    formdata_user.user_id = user.id
    formdata_user.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = get_app(pub).get('/test/%s/' % formdata_user.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % formdata_user.id)
    assert 'The form has been recorded on' in resp


def test_form_no_tracking_code(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert '<h3>Tracking code</h3>' not in resp.text


def test_form_no_tracking_code_variable(pub):
    create_user(pub)
    FormDef.wipe()
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(id='3', label='<p>xxx{{form_tracking_code|default:""}}yyy</p>', type='comment'),
    ]
    formdef.store()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp.form['f1'] = 'foo'
    app.post('/test/autosave', params=resp.form.submit_fields())
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')
    assert 'xxxyyy' in resp.text
    resp = resp.form.submit('submit')
    assert_current_page(resp, 'Validating')
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo'}
    assert data.tracking_code is None


def get_displayed_tracking_code(resp):
    tracking_code = None
    for a_tag in resp.html.findAll('a'):
        if 'code/' in a_tag['href']:
            tracking_code = a_tag.text
            break
    return tracking_code


def test_form_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()

    # check anonymous user can't get to it from the URL
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('http://example.net/test/%s/' % formdata_id)
    assert resp.location.startswith('http://example.net/login')

    # or logged users that didn't enter the code:
    create_user(pub)
    login(get_app(pub), username='foo', password='foo').get(
        'http://example.net/test/%s/' % formdata_id, status=403
    )

    # check we can also get to it as a logged user
    pub.session_manager.session_class.wipe()
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()

    # go back as anonymous
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1  # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

    # check we can also use it with lowercase letters.
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()


def test_form_tracking_code_rate_limit(pub, freezer):
    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'rate-limit', '2/2s')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    # twice
    freezer.move_to('2018-12-01T00:00:00')
    get_app(pub).get('/code/ABC/load', status=404)
    get_app(pub).get('/code/ABC/load', status=404)
    # and out
    get_app(pub).get('/code/ABC/load', status=403)
    get_app(pub).get('/code/ABC/load', status=403)
    # wait two second
    freezer.move_to('2018-12-01T00:00:02')
    # and ok again
    get_app(pub).get('/code/ABC/load', status=404)


def test_form_tracking_code_as_user(pub, nocache):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1  # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert str(formdef.data_class().select()[0].user_id) == str(user.id)
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

    # and check we can also get back to it as anonymous
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter

    # and check a bot is not allowed to get it
    app = get_app(pub)
    resp = app.get('/code/%s/load' % tracking_code, headers={'User-agent': 'Googlebot'}, status=403)


def test_form_tracking_code_prefill(pub, nocache):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.enable_tracking_codes = True
    formdef.store()

    # first time
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'You already started to fill this form.' not in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')

    # second time, invitation to load an existing draft
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'You already started to fill this form.' in resp.text


def test_form_empty_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # check we get a 404 if we use the tracking code before it gets any data
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow(status=404)


def test_form_tracking_code_email(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'), fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    app.post('/test/autosave', params=resp.form.submit_fields())

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp = resp.forms[0].submit()
    assert emails.get('Tracking Code reminder')
    assert tracking_code in emails.get('Tracking Code reminder')['payload']
    assert resp.location == 'http://example.net/test/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'barfoo'


def test_form_tracking_code_email_antibot(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'), fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    app.post('/test/autosave', params=resp.form.submit_fields())

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp.forms[0]['validation'].checked = True  # stupit bot will do that
    resp = resp.forms[0].submit()
    assert not emails.count()


def test_form_tracking_code_remove_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '1'
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # visit page, check there's no remove draft button
    resp = app.get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' not in resp.text

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    resp = resp.forms[0].submit()  # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0


def test_form_tracking_code_remove_empty_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('previous')
    app.post('/test/autosave', params=resp.form.submit_fields())
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '0'

    # make draft empty
    formdata = formdef.data_class().select()[0]
    formdata.data = {}
    formdata.store()
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    assert resp.forms[1]['f0'].value == ''

    resp = resp.forms[0].submit()  # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0


def test_form_discard_draft(pub, nocache):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    formdef.data_class().wipe()

    # anonymous user, no tracking code (-> no draft)
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == []
    assert 'Cancel' in resp.text
    assert 'Discard' not in resp.text
    resp = resp.form.submit('cancel')

    # anonymous user, tracking code (-> draft)
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, no tracking code
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, tracking code
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # anonymous user, tracking code, recalled
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    tracking_code = get_displayed_tracking_code(resp)

    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()
    assert resp.forms[1]['f0'].value == 'foobar'
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']

    # logged-in user, no tracking code, recalled
    formdef.data_class().wipe()
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.click('Continue with draft').follow()
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']


def test_form_invalid_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    # create a secondary formdef, to always have the tracking code form
    # displayed on homepage
    formdef2 = FormDef()
    formdef2.name = 'test2'
    formdef2.fields = []
    formdef2.enable_tracking_codes = True
    formdef2.store()

    resp = get_app(pub).get('/')

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.store()

    # check we can go back to it
    formdef.data_class().wipe()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata.id
    resp = resp.follow()

    # check we get a not found error message on non-existent code
    fake_code = TrackingCode().get_new_id()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = fake_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % fake_code
    resp = resp.follow(status=404)

    # check we also get an error if tracking code access is disabled after the
    # fact
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow(status=404)


def test_form_tracking_code_as_variable(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(type='comment', id='3', label='!{{ form_tracking_code }}!'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foobar'
    resp = resp.form.submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    assert '!%s!' % tracking_code in resp.text


def test_form_draft_with_file(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert '<span class="filename">test.txt</span>' in resp.text

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert resp.click('test.txt').follow().text == 'foobar'


def test_form_draft_with_file_direct_validation(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert 'test2.txt' in resp.text

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert resp.click('test2.txt').follow().text == 'foobar2'


def test_form_draft_with_date(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='date', type='date')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0'] = '2012-02-12'
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert '2012-02-12' in resp.text

    # check submitted form keeps the date
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert '2012-02-12' in resp.text


@pytest.mark.parametrize('field_type', ['string', 'item'])
@pytest.mark.parametrize('logged_in', ['anonymous', 'logged-in'])
def test_form_draft_from_prefill(pub, field_type, logged_in):
    create_user(pub)

    formdef = create_formdef()
    if field_type == 'string':
        formdef.fields = [fields.StringField(id='0', label='string')]
    else:
        formdef.fields = [fields.ItemField(id='0', label='item', type='item', items=['foo', 'bar', 'hello'])]
    formdef.enable_tracking_codes = True
    formdef.store()
    formdef.data_class().wipe()

    app = get_app(pub)
    if logged_in == 'logged-in':
        login(app, username='foo', password='foo')

    # no draft
    app.get('/test/')
    assert formdef.data_class().count() == 0
    formdef.data_class().wipe()

    # draft created if there's been some prefilled fields
    formdef.fields[0].prefill = {'type': 'string', 'value': '{{request.GET.test|default:""}}'}
    formdef.store()
    app.get('/test/?test=hello')
    assert formdef.data_class().count() == 1
    formdef.data_class().wipe()

    # unless the call was made from an application
    app.get('/test/?test=hello', headers={'User-agent': 'python-requests/0'})
    assert formdef.data_class().count() == 0

    # or a bot
    app.get('/test/?test=hello', headers={'User-agent': 'Googlebot'})
    assert formdef.data_class().count() == 0

    # check there's no leftover draft after submission
    for with_tracking_code in (False, True):
        formdef.enable_tracking_codes = with_tracking_code
        formdef.store()
        formdef.data_class().wipe()
        resp = app.get('/test/?test=hello')
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().count() == 1


@pytest.mark.parametrize('tracking_code', [True, False])
def test_form_direct_draft_access(pub, tracking_code):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = tracking_code
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.status = 'draft'
    formdata.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    formdata.user_id = user.id
    formdata.store()
    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/test/?mt=')

    formdata.user_id = 1000
    formdata.store()
    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=403)


def form_password_field_submit(app, password):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [fields.PasswordField(id='0', label='password', formats=['sha1', 'md5', 'cleartext'])]
    formdef.store()
    page = app.get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert [x.text for x in next_page.pyquery('div.error')] == ['required field'] * 2
    next_page.forms[0]['f0$pwd1'] = password
    next_page.forms[0]['f0$pwd2'] = password
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {
        '0': {
            'sha1': force_str(hashlib.sha1(force_bytes(password)).hexdigest()),
            'md5': force_str(hashlib.md5(force_bytes(password)).hexdigest()),
            'cleartext': force_str(password),
        }
    }


def test_form_password_field_submit(pub):
    create_user(pub)
    form_password_field_submit(get_app(pub), 'foobar')
    form_password_field_submit(get_app(pub), force_str('•	83003706'))
    form_password_field_submit(login(get_app(pub), username='foo', password='foo'), 'foobar\u00eb')


def test_form_multi_page_formdef_count_condition(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_objects.count > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text

    # add a formdata this will make the second page appear.
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should NOT go straight to validation
    assert 'Check values then click submit.' not in resp.text


def test_form_multi_page_post_edit(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    next_page.forms[0]['f3'] = 'barXYZ'
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text

    data_id = formdef.data_class().select()[0].id

    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    # check there's no new "phantom" history entry
    assert len(formdef.data_class().get(data_id).evolution) == 1
    assert resp.forms[0]['f1'].value == 'foo'
    resp.forms[0]['f1'] = 'foo2'

    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1'].value == 'foo2'
    resp = resp.forms[0].submit('submit')
    assert 'Save Changes' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo2' in resp.text  # modified value is there
    assert 'barXYZ' in resp.text  # unchanged value is still there
    assert len(formdef.data_class().get(data_id).evolution) == 2  # new history entry
    assert formdef.data_class().get(data_id).evolution[-1].who == '_submitter'
    assert formdef.data_class().get(data_id).evolution[-1].status is None

    # modify workflow to jump to another status after the edition
    st2 = workflow.add_status('Status2', 'st2')
    editable.status = st2.id
    workflow.store()

    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo2'
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    assert (
        formdef.data_class().get(data_id).data['1'] == 'foo2'
    )  # check foo3 has not been overwritten in database
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo3' in resp.text  # modified value is there
    assert 'barXYZ' in resp.text  # unchanged value is still there
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st2.id
    assert len(formdef.data_class().get(data_id).evolution) == 3  # single new history entry
    assert formdef.data_class().get(data_id).evolution[-1].who == '_submitter'
    assert formdef.data_class().get(data_id).evolution[-1].status == 'wf-%s' % st2.id

    # jump to a nonexistent status == do not jump, but add a LoggedError
    if pub.is_using_postgresql():
        pub.loggederror_class.wipe()
        assert pub.loggederror_class.count() == 0
    editable.status = 'deleted_status_id'
    workflow.store()
    # go back to st1
    formdata = formdef.data_class().get(data_id)
    formdata.status = 'wf-%s' % st1.id
    formdata.store()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    if pub.is_using_postgresql():
        assert pub.loggederror_class.count() == 1
        logged_error = pub.loggederror_class.select()[0]
        assert logged_error.formdata_id == str(formdata.id)
        assert logged_error.formdef_id == formdef.id
        assert logged_error.workflow_id == workflow.id
        assert logged_error.status_id == st1.id
        assert logged_error.status_item_id == editable.id
        assert logged_error.occurences_count == 1

    # do it again: increment logged_error.occurences_count
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    if pub.is_using_postgresql():
        assert pub.loggederror_class.count() == 1
        logged_error = pub.loggederror_class.select()[0]
        assert logged_error.occurences_count == 2


def test_form_edit_autocomplete_list(pub):
    create_user(pub)

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = get_app(pub)
    login(app, username='foo', password='foo')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert 'data-select2-url=' in resp.text
        # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
        resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
        resp.form['f0'].force_value('1')
        resp.form.fields['f0_display'].force_value('hello')
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

        resp = resp.follow()
        url = resp.request.url
        resp = resp.form.submit('button_editable')
        assert 'wfedit' in resp.location
        resp = resp.follow()
        assert 'data-value="1"' in resp
        assert 'data-initial-display-value="hello"' in resp

        # relogin
        app = get_app(pub)
        login(app, username='foo', password='foo')
        resp = app.get(url)
        resp = resp.form.submit('button_editable')
        resp = resp.follow()
        assert 'data-value="1"' in resp
        assert 'data-initial-display-value="hello"' in resp


def test_form_count_dispatching(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.condition = {'type': 'python', 'value': 'form_objects.count_status_st2 < 1'}
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    workflow.add_status('Status2', 'st2')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text  # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 0
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text  # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 1


def test_preview_form(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = []
    formdef.disabled = True
    formdef.store()

    # check the preview page is not accessible to regular users
    get_app(pub).get('/preview/test/', status=403)

    # check it's accessible to admins
    user.is_admin = True
    user.store()
    page = login(get_app(pub), username='foo', password='foo').get('/preview/test/')

    # check no formdata gets stored
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 200
    assert 'This was only a preview: form was not actually submitted.' in next_page.text
    assert len([x for x in formdef.data_class().select() if not x.is_draft()]) == 0


def test_form_item_data_source_field_submit(pub):
    def submit_item_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0'] = '1'
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux')]),
    }
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un', 'more': 'foo'}, {'id': '2', 'text': 'deux', 'more': 'bar'}])
    assert submit_item_data_source_field(ds) == {
        '0': '1',
        '0_display': 'un',
        '0_structured': {'id': '1', 'text': 'un', 'more': 'foo'},
    }

    # numeric identifiers
    ds['value'] = repr([{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    # json source
    ds = {
        'type': 'json',
        'value': 'http://www.example.net/plop',
    }

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    # numeric identifiers
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}


def test_form_items_data_source_field_submit(pub):
    def submit_items_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemsField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0$element1'].checked = True
        resp.forms[0]['f0$element3'].checked = True
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux'), ('3', 'trois')]),
    }
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [{'id': '1', 'text': 'un'}, {'id': '3', 'text': 'trois'}],
    }

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}, {'id': '3', 'text': 'trois'}])
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [{'id': '1', 'text': 'un'}, {'id': '3', 'text': 'trois'}],
    }

    ds['value'] = repr(
        [
            {'id': '1', 'text': 'un', 'more': 'foo'},
            {'id': '2', 'text': 'deux', 'more': 'bar'},
            {'id': '3', 'text': 'trois', 'more': 'baz'},
        ]
    )
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [
            {'id': '1', 'more': 'foo', 'text': 'un'},
            {'id': '3', 'more': 'baz', 'text': 'trois'},
        ],
    }


def test_form_page_string_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text


def test_form_page_profile_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'


def test_form_page_profile_first_name_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields = [
        fields.StringField(
            id='_first_name', label='first name', type='string', extra_css_class='autocomplete-given-name'
        ),
        fields.StringField(
            id='_city', label='city', type='string', extra_css_class='autocomplete-address-level2'
        ),
        fields.StringField(id='_plop', label='plop', type='string', extra_css_class='xxx'),
    ]
    user_formdef.store()
    user.form_data = {'_first_name': 'plop', '_city': 'mytown'}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': '_first_name'}),
        fields.StringField(id='1', label='string', prefill={'type': 'user', 'value': '_city'}),
        fields.StringField(id='2', label='string', prefill={'type': 'user', 'value': '_plop'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''
    assert resp.forms[0]['f0'].attrs['autocomplete'] == 'given-name'  # html5
    assert resp.forms[0]['f1'].value == ''
    assert resp.forms[0]['f1'].attrs['autocomplete'] == 'address-level2'  # html5

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'plop'
    assert resp.forms[0]['f1'].value == 'mytown'


def test_form_page_formula_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': repr('HELLO WORLD')})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text


def test_form_page_template_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(
            id='0', label='string', prefill={'type': 'string', 'value': '{{session_user_display_name}}'}
        )
    ]
    formdef.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert resp.form['f0'].value == 'User Name'
    assert 'widget-prefilled' in resp.text

    # erroneous prefill
    formdef.fields = [
        fields.StringField(
            id='0',
            label='string',
            prefill={'type': 'string', 'value': '{{session_user_display_name|unknown}}'},
        )
    ]
    formdef.store()

    resp = app.get('/test/')
    assert resp.form['f0'].value == ''
    # still marked with a css class, in case of live changes.
    assert 'widget-prefilled' in resp.text


def test_form_page_session_var_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'session_var_foo'})
    ]
    formdef.store()

    # check it's empty if it doesn't exist
    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    # check it's not set if it's not whitelisted
    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check it works
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        fd.write(
            '''[options]
query_string_allowed_vars = foo,bar
'''
        )

    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello'

    # check it survives a login
    resp = get_app(pub).get('/?session_var_foo=hello2')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('Login')
    resp = resp.follow()
    resp.forms[0]['username'] = 'foo'
    resp.forms[0]['password'] = 'foo'
    resp = resp.forms[0].submit()
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello2'

    # check repeated options are ignored
    resp = get_app(pub).get('/?session_var_foo=hello&session_var_foo=hello2')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check extra query string parameters are not lost
    resp = get_app(pub).get('/?session_var_foo=hello&foo=bar')
    assert urllib.parse.urlparse(resp.location).path == '/'
    assert urllib.parse.urlparse(resp.location).query == 'foo=bar'

    os.unlink(os.path.join(pub.app_dir, 'site-options.cfg'))


def test_form_page_template_list_prefill(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='1',
            type='item',
            label='item',
            varname='item',
            required=True,
            items=['Foo', 'Bar'],
            prefill={'type': 'string', 'value': 'Foo'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == 'Foo'

    formdef.fields[0].prefill['value'] = 'Bar'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == 'Bar'

    formdef.fields[0].prefill['value'] = 'Baz'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'invalid value selected' in resp.text

    formdef.fields[0].prefill['value'] = '{{plop|default:""}}'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'invalid value selected' not in resp.text


def test_form_page_query_string_list_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='item',
            varname='item',
            required=False,
            data_source={'type': 'foobar'},
            prefill={'type': 'string', 'value': '{{request.GET.preselect}}'},
        )
    ]
    formdef.store()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un'},
                {'id': '2', 'text': 'deux'},
                {'id': '3', 'text': 'trois'},
                {'id': '4', 'text': 'quatre'},
            ]
        ),
    }
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == '1'

    resp = get_app(pub).get('/test/?preselect=2')
    assert resp.form['f1'].value == '2'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert 'deux' in resp.text


def test_form_page_profile_prefill_list(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['', 'bar@localhost', 'foo@localhost'],
            required=False,
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    # invalid value
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['', 'bar@localhost'],
            required=False,
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'invalid value selected' in resp.text
    assert resp.forms[0]['f0'].value == ''


def test_form_page_formula_prefill_items_field(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='items',
            items=['foo', 'bar', 'baz'],
            prefill={'type': 'formula', 'value': '["foo", "baz"]'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0$element0'].checked
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked
    assert 'widget-prefilled' in resp.text

    resp.form['f0$element0'].checked = False
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert '>foo<' not in resp.text
    assert '>bar<' not in resp.text
    assert '>baz<' in resp.text

    # check with remote json
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='items',
            data_source=ds,
            display_disabled_items=True,
            prefill={'type': 'formula', 'value': '["2"]'},
        )
    ]

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not resp.form['f0$element1'].checked
        assert resp.form['f0$element2'].checked


def test_form_page_changing_prefill(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3', label='string 2', prefill={'type': 'string', 'value': '{{ form_var_foo }} World'}
        ),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('previous')  # back to 1st page
    assert resp.form['f1'].value == 'Hello'
    resp.form['f1'] = 'Goodbye Cruel'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Goodbye Cruel World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    resp.form['f3'].value = 'Changed value'
    resp = resp.form.submit('previous')  # back to 1st page
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Changed value'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Changed value'


def test_form_page_changing_prefill_draft(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3', label='string 2', prefill={'type': 'string', 'value': '{{ form_var_foo }} World'}
        ),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    tracking_code = get_displayed_tracking_code(resp)

    # start with a new session and restore draft using the tracking code
    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()
    assert_current_page(resp, '3rd page')
    resp = resp.forms[1].submit('previous')  # back to 2nd page
    assert resp.forms[1]['f3'].value == 'Hello World'
    resp = resp.forms[1].submit('previous')  # back to 1st page
    assert resp.forms[1]['f1'].value == 'Hello'
    resp.forms[1]['f1'] = 'Goodbye Cruel'
    resp = resp.forms[1].submit('submit')  # -> 2nd page
    assert resp.forms[1]['f3'].value == 'Goodbye Cruel World'


def test_prefill_query_parameter(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(
            id='1',
            label='str',
            varname='foo',
            required=False,
            prefill={'type': 'string', 'value': '{{request.GET.prefill}}'},
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/?prefill=Hello')
    assert resp.form['f1'].value == 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp = resp.form.submit('previous')  # back to 1st page
    # check it has not be reset to the empty string (as there's no request.GET
    # anymore)
    assert resp.form['f1'].value == 'Hello'


def test_form_captcha(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.store()

    # test authenticated users are not presented with a captcha
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' not in resp.text

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text

    session_id = list(app.cookies.values())[0].strip('"')
    session = pub.session_class.get(session_id)
    resp.form['captcha$q'] = session.get_captcha_token(resp.forms[0]['captcha$token'].value)['answer']
    resp = resp.form.submit('submit')
    assert resp.status_code == 302  # redirect when formdata is created

    # and check it gets it only once
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' not in resp.text


def test_form_captcha_and_no_validation_page(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.confirmation = False
    formdef.store()

    # test authenticated users are not stopped on a confirmation page
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert resp.status_code == 302  # redirect when formdata is created

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text


def test_form_table_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableField(
            id='0',
            label='table',
            type='table',
            rows=[force_str('à'), 'b'],
            columns=['c', 'd', force_str('e')],
            required=False,
        )
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': [['', '', ''], ['', '', '']]}
    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableField(
            id='0', label='table', type='table', rows=['a', 'b'], columns=['c', 'd', 'e'], required=True
        )
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' not in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$c-0-0'] = 'a'
    resp.form['f0$c-1-0'] = 'b'
    resp.form['f0$c-0-1'] = 'c'
    resp.form['f0$c-1-1'] = 'd'
    resp.form['f0$c-0-2'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'c', 'e'], ['b', 'd', '']]}

    # check table is present in received email (via form_details).
    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$c-0-0'] = 'àà'  # would trigger column length bug (#23072)
    resp.form['f0$c-1-0'] = 'bb'
    resp.form['f0$c-0-1'] = 'cc'
    resp.form['f0$c-1-1'] = 'dd'
    resp.form['f0$c-0-2'] = 'ee'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    # check rst2html didn't fail
    assert b'ee' in emails.get('New form (test)')['msg'].get_payload()[1].get_payload(decode=True)


def test_form_table_rows_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=False)
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': []}
    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True)
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' not in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = 'b'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = 'd'
    resp.form['f0$element2$col0'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'b'], ['c', 'd'], ['e', '']]}

    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableRowsField(
            id='0', label='table', type='tablerows', columns=['a', 'b'], required=True, total_row=True
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert '37.00' in resp.text

    # check table is present in received email (via form_details).
    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$element0$col0'] = 'àà'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'ee'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert b'ee' in emails.get('New form (test)')['msg'].get_payload()[1].get_payload(decode=True)


def test_form_new_table_rows_field_draft_recall(pub):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # add new table rows field to formdef
    formdef.fields.append(
        fields.TableRowsField(id='3', label='table', type='tablerows', columns=['a', 'b'], required=False)
    )
    formdef.store()

    # restore form on validation page
    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()

    # validate form
    resp = resp.forms[1].submit()
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'test'
    assert formdef.data_class().select()[0].data['3'] is None


def test_form_table_rows_field_and_prefill(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True),
        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
    ]
    formdef.store()
    formdef.data_class().wipe()

    get_app(pub).get('/test/')


def test_form_table_rows_add_row(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string', require=True),
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 10
    resp = resp.form.submit('f0$add_element')
    assert 'There were errors processing the form' not in resp
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 12
    resp = resp.form.submit('f0$add_element')
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 14
    resp = resp.form.submit('submit')
    assert 'There were errors processing the form' in resp


def test_form_map_field_back_and_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.StringField(
            id='1', label='street', required=True, prefill={'type': 'geolocation', 'value': 'road'}
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.text
    assert 'qommon.geolocation.js' in resp.text
    # with a real user interaction this would get set by javascript
    resp.forms[0]['f0$latlng'].value = '1.234;-1.234'
    assert 'data-geolocation="road"' in resp.text

    # check required field
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' not in resp.text
    assert 'data-geolocation="road"' in resp.text
    resp.forms[0]['f1'].value = 'bla'

    # check summary page
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-init-lng="-1.234"' in resp.text
    assert 'data-init-lat="1.234"' in resp.text

    # get back to the map field
    resp = resp.forms[0].submit('previous')
    # check the field is still marked as holding the road
    assert 'data-geolocation="road"' in resp.text
    assert resp.forms[0]['f0$latlng'].value == '1.234;-1.234'

    # back to summary page
    resp = resp.forms[0].submit('submit')

    # and submitting the form
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'bla', '0': '1.234;-1.234'}


def test_form_map_geolocation_text_field(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.TextField(
            id='1', label='street', required=True, prefill={'type': 'geolocation', 'value': 'road'}
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.text
    assert 'qommon.geolocation.js' in resp.text
    assert 'WCS_DEFAULT_GEOCODING_COUNTRY' not in resp.text

    # check page has default geocoding country in a javascript variable if set
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'default-geocoding-country', 'France')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    resp = get_app(pub).get('/test/')
    assert 'WCS_DEFAULT_GEOCODING_COUNTRY' in resp.text


def test_form_map_field_prefill_address(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='address', required=True, varname='address'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.MapField(id='3', label='map', prefill={'type': 'string', 'value': '{{ form_var_address }}'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '169 rue du chateau, paris'
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (
            None,
            200,
            json.dumps([{'lat': '48.8337085', 'lon': '2.3233693'}]),
            None,
        )
        resp = resp.form.submit('submit')
        assert resp.form['f3$latlng'].value == '48.8337085;2.3233693'
        assert 'chateau' in http_get_page.call_args[0][0]


def test_form_map_field_prefill_coords(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.MapField(id='1', label='map', type='map', varname='map1'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.MapField(id='3', label='map', prefill={'type': 'string', 'value': '{{ form_var_map1 }}'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1$latlng'] = '1.234;-1.234'
    resp = resp.form.submit('submit')
    assert resp.form['f3$latlng'].value == '1.234;-1.234'


def test_form_map_multi_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.MapField(id='1', label='map'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1$latlng'] = '1.234;-1.234'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f1$latlng'].value == '1.234;-1.234'
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': '1.234;-1.234', '3': 'bar'}


def test_form_middle_session_change(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    app.cookiejar.clear()
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/'
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    app.cookiejar.clear()
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text


def test_form_autocomplete_variadic_url(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(
            id='3', label='2nd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
        fields.ItemField(id='1', label='string', type='item', varname='foo', items=['Foo', 'Bar']),
        fields.StringField(
            id='2', label='string2', required=True, data_source={'type': 'jsonp', 'value': '[var_foo]'}
        ),
        fields.PageField(
            id='4', label='3rd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')  # next
    # test javascript will be used to compute the full URL
    assert 'options.wcs_base_url' in resp.text
    assert 'jquery-ui.min.js' in resp.text

    # test going forward (will error out), check it's still a variadic URL (#9786)
    resp.form['f1'] = 'Foo'
    resp = resp.form.submit('submit')
    assert 'options.wcs_base_url' in resp.text


def test_form_page_formula_prefill_user_name(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'form_user_email'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'


def test_form_page_formula_prefill_session_user(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'session_user_email'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    app = login(get_app(pub), username='foo', password='foo')
    for session in pub.session_manager.values():
        session.extra_user_variables = {'foo': 'bar'}
        session.store()
    formdef.fields = [
        fields.StringField(
            id='0', label='string', prefill={'type': 'formula', 'value': 'session_var_user_foo'}
        )
    ]
    formdef.store()
    resp = app.get('/test/')
    assert resp.forms[0]['f0'].value == 'bar'


def test_form_date_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='string', type='date', required=False)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f0'] = '2015-01-01'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert time.strftime('%Y-%m-%d', data.data['0']) == '2015-01-01'

    # without filling the field
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None


def test_form_jsonp_item_field(http_requests, pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='string',
            type='item',
            data_source={'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'},
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert 'data-select2-url="http://remote.example.net/jsonp"' in resp.text
    assert 'select2.min.js' in resp.text


def test_form_string_regex_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(
            id='0',
            label='string',
            type='string',
            validation={'type': 'regex', 'value': r'\d{5}$'},
            required=False,
        )
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # with an invalid input
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    assert 'invalid value' in resp.text


def test_form_text_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TextField(id='0', label='string', type='text', required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # check max length
    formdef.fields = [fields.TextField(id='0', label='string', type='text', maxlength=10)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'x' * 11
    resp = resp.forms[0].submit('submit')
    assert 'too many characters (limit is 10)' in resp.text
    # check it counts characters, not bytes
    resp.forms[0]['f0'] = '☭' * 10
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text


def test_unknown_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(
            id='1', label='string', varname='string', required=False, data_source={'type': 'foobar'}
        ),
        fields.ItemField(
            id='2', label='item', varname='item', required=False, data_source={'type': 'foobar'}
        ),
        fields.ItemsField(
            id='3', label='items', varname='items', required=False, data_source={'type': 'foobar'}
        ),
    ]

    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text


def test_form_items_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemsField(
            id='1', label='items', varname='items', required=False, data_source={'type': 'foobar'}
        )
    ]
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()

    # add the named data source
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert 'f1$elementun' in resp.form.fields
    assert 'f1$elementdeux' in resp.form.fields
    resp.form['f1$elementun'].checked = True
    resp.form['f1$elementdeux'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {
        '1': ['un', 'deux'],
        '1_display': 'un, deux',
        '1_structured': [{'id': 'un', 'text': 'un'}, {'id': 'deux', 'text': 'deux'}],
    }

    data_source.data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]),
    }
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {
        '1': ['1', '2'],
        '1_display': 'un, deux',
        '1_structured': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}],
    }

    data_source.data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un', 'foo': 'bar1'}, {'id': '2', 'text': 'deux', 'foo': 'bar2'}]),
    }
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert data_class.select()[0].data == {
        '1': ['1', '2'],
        '1_structured': [
            {'text': 'un', 'foo': 'bar1', 'id': '1'},
            {'text': 'deux', 'foo': 'bar2', 'id': '2'},
        ],
        '1_display': 'un, deux',
    }
    # along the way, check substitution variables
    substvars = data_class.select()[0].get_substitution_variables()
    assert substvars['form_var_items'] == 'un, deux'
    assert substvars['form_var_items_raw'] == ['1', '2']
    assert substvars['form_var_items_0_foo'] == 'bar1'
    assert substvars['form_var_items_1_foo'] == 'bar2'


def test_form_ranked_items_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.RankedItemsField(
            id='0', label='ranked items', type='ranked-items', required=False, items=['foo', 'bar', 'baz']
        )
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0$element0'] = '1'
    resp.form['f0$element1'] = '2'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == {'bar': 2, 'foo': 1}


def test_form_ranked_items_randomize_order(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.RankedItemsField(
            id='0',
            label='ranked items',
            type='ranked-items',
            required=False,
            randomize_items=True,
            items=['foo', 'bar', 'baz'],
        )
    ]
    formdef.store()
    orders = {}
    for _ in range(10):
        resp = get_app(pub).get('/test/')
        orders['%s-%s-%s' % (resp.text.index('foo'), resp.text.index('bar'), resp.text.index('baz'))] = True
    assert len(orders.keys()) > 1


def test_form_autosave(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'foobar'

    resp.form['f1'] = 'foobar2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar2'

    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    resp.form['f3'] = 'xxx'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx'

    resp.form['f3'] = 'xxx2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx2'

    resp.form['f3'] = 'xxx3'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'foobar3' in resp.text
    assert 'xxx3' in resp.text

    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx3'

    # make sure autosave() doesn't destroy data that would have been submitted
    # in the meantime
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'
    autosave_fields = resp.form.submit_fields()
    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    # post content with 'foobar' as value, it should not be saved
    ajax_resp = app.post('/test/autosave', params=autosave_fields)
    assert json.loads(ajax_resp.text)['result'] == 'error'
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'


def test_form_autosave_with_items_field(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemsField(
            id='3',
            label='items',
            type='items',
            items=[force_str(x) for x in ('pomme', 'poire', 'pêche', 'abricot')],
        ),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') is None

    resp = resp.forms[0].submit('submit')
    resp.form['f3$element1'].checked = True
    resp.form['f3$element3'].checked = True
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == ['poire', 'abricot']


def test_form_autosave_with_parameterized_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(
            id='3',
            label='item',
            type='item',
            data_source={'type': 'formula', 'value': '''[('1', form_var_foo*2)]'''},
        ),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') is None

    resp = resp.forms[0].submit('submit')
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == '1'
    assert formdef.data_class().select()[0].data['3_display'] == 'barbar'


def test_form_string_field_autocomplete(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string', required=False)]
    formdef.fields[0].data_source = {'type': 'jsonp'}
    formdef.store()

    # not filled completed, no call to .autocomplete
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' not in resp.text

    # straight URL
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': 'http://example.org'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.org' in resp.text

    # URL from variable
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': '[site_url]'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.net' in resp.text


def test_form_string_field_autocomplete_named_datasource(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = [
        fields.StringField(
            id='0', label='string', type='string', required=False, data_source={'type': 'foobar'}
        )
    ]
    formdef.store()

    # jsonp datasource
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'jsonp', 'value': 'http://remote.example.net/json'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" in resp.text
    assert "options.url = '/api/autocomplete/" not in resp.text
    assert 'dataType: "jsonp",' in resp.text

    # json datasource
    data_source.data_source['type'] = 'json'
    data_source.query_parameter = 'q'
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" not in resp.text
    assert "options.url = '/api/autocomplete/" in resp.text
    assert 'dataType: "json",' in resp.text

    # card datasource
    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'Foo'
    carddef.fields = []
    carddef.store()

    data_source.data_source['type'] = 'carddef:foo'
    data_source.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" not in resp.text
    assert "options.url = '/api/autocomplete/" in resp.text
    assert 'dataType: "json",' in resp.text


def test_form_workflow_trigger(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    jump2 = JumpWorkflowStatusItem()
    jump2.trigger = 'YYY'
    jump2.status = 'st3'
    jump2.set_marker_on_status = True
    st1.items.append(jump2)
    jump2.parent = st1

    workflow.add_status('Status2', 'st2')
    workflow.add_status('Status3', 'st3')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    login(app, username='foo', password='foo').get('/')
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    jump.by = [role.id]
    workflow.store()
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    user.roles = [role.id]
    user.store()
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st2'

    formdata.status = 'wf-st1'
    formdata.store()
    app.post(formdata.get_url() + 'jump/trigger/YYY', status=403)
    jump2.by = [role.id]
    workflow.store()
    app.post(formdata.get_url() + 'jump/trigger/YYY', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st3'
    assert formdata.workflow_data.get('_markers_stack') == [{'status_id': 'st1'}]

    formdata.status = 'wf-st1'
    formdata.store()
    app.post(
        formdata.get_url() + 'jump/trigger/YYY',
        params=json.dumps({'data': {'foo': 'bar'}}),
        content_type='application/json',
    )
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.workflow_data.get('data') == {'foo': 'bar'}


def test_form_worklow_multiple_identical_status(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    st1.extra_css_class = 'CSS-STATUS1'
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st1'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.allows_backoffice_access = False
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    assert len(formdef.data_class().get(formdata.id).evolution) == 1
    assert formdef.data_class().get(formdata.id).evolution[0].last_jump_datetime is None

    login(app, username='foo', password='foo')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    # status is not changed: no new evolution, only a new last_jump_datetime
    assert len(formdata.evolution) == 1
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].last_jump_datetime is not None
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # add a comment to last evolution, forcing create a new one
    formdata.evolution[-1].comment = 'new-evolution-1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 2
    assert formdata.status == 'wf-st1'
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # again
    formdata.evolution[-1].comment = 'new-evolution-2'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # last evolution is empty, this last trigger does not create a new one
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # finally, 3 evolutions: new-evolution-1, new-evolution-2, empty
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 3
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].comment == 'new-evolution-1'
    assert formdata.evolution[1].comment == 'new-evolution-2'
    assert formdata.evolution[2].comment is None

    resp = app.get(formdata.get_url())
    assert resp.text.count('Status1') == 3  # once in summary and two in journal
    assert resp.text.count('CSS-STATUS1') == 2
    assert resp.text.count('new-evolution-1') == 1
    assert resp.text.count('new-evolution-2') == 1


def test_form_worklow_comments_on_same_status(pub):
    pub.session_manager.session_class.wipe()
    user = create_user(pub)

    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    workflow = Workflow.get_default_workflow()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.perform_workflow()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-new'

    app = get_app(pub)

    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is True
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True
    login(app, username='foo', password='foo')
    resp = app.get(formdata.get_url()).follow()
    resp.form['comment'] = 'TEST COMMENT'
    resp = resp.form.submit('button_commentable')
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    resp = app.get(formdata.get_url()).follow()
    resp = resp.form.submit('button_accept')
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is True
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True


def test_display_message(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st0 = workflow.add_status('Status0', 'st0')
    jump = JumpWorkflowStatusItem()
    jump.status = 'st1'
    st0.items.append(jump)
    jump.parent = st0

    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    display2 = DisplayMessageWorkflowStatusItem()
    display2.message = 'message-to-submitter'
    display2.to = ['_submitter']
    st1.items.append(display2)
    display2.parent = st1

    display3 = DisplayMessageWorkflowStatusItem()
    display3.message = 'message-to-nobody'
    display3.to = ['xxx']
    st1.items.append(display3)
    display3.parent = st1

    display4 = DisplayMessageWorkflowStatusItem()
    display4.message = 'message-to-xxx-and-submitter'
    display4.to = ['_submitter', 'xxx']
    st1.items.append(display4)
    display4.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()

    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-nobody' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') < page.text.index('message-to-xxx-and-submitter')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    # actions alert vs top alert
    display2.position = 'actions'
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # add an action, so display2 will appear again
    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_submitter']
    jump1.status = st1.id
    jump1.parent = st1
    st1.items.append(jump1)
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    jump1.by = ['xxx']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # change to always display at the bottom
    display2.position = 'bottom'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    # set a level
    display2.level = 'warning'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' in page.text

    # check message is not displayed if status is not visible to user
    st1.visibility = ['_receiver']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' not in page.text


def test_workflow_condition_on_message(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'message-to-all' in page.text

    formdata = formdef.data_class().select()[0]
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text

    display1.condition = {'type': 'django', 'value': 'xxx'}
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' not in page.text


def test_session_cookie_flags(pub):
    create_formdef()
    app = get_app(pub)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('sessionid-')
    assert 'HttpOnly' in resp.headers['Set-Cookie']
    assert 'Secure' not in resp.headers['Set-Cookie']

    app = get_app(pub, https=True)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('sessionid-')
    assert 'HttpOnly' in resp.headers['Set-Cookie']
    assert 'Secure' in resp.headers['Set-Cookie']


def test_form_page_profile_verified_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'readonly' not in resp.form['f0'].attrs
    resp.form['f0'].value = 'Hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'Hello'

    user.verified_fields = ['email']
    user.store()

    for prefill_settings in (
        {'type': 'user', 'value': 'email'},  # verified profile
        {'type': 'string', 'value': 'foo@localhost', 'locked': True},  # locked value
    ):
        formdef.confirmation = True
        formdef.fields[0].prefill = prefill_settings
        formdef.store()
        formdef.data_class().wipe()
        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp.form['f0'].value = 'Hello'  # try changing the value
        resp = resp.form.submit('submit')
        assert 'Check values then click submit.' in resp.text
        assert resp.form['f0'].value == 'foo@localhost'  # it is reverted

        resp.form['f0'].value = 'Hello'  # try again changing the value
        resp = resp.form.submit('submit')

        formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
        assert len(formdatas) == 1
        assert formdatas[0].data['0'] == 'foo@localhost'

        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        resp = resp.form.submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp.form['f0'].value = 'Hello'  # try changing
        resp = resp.form.submit('previous')
        assert 'readonly' in resp.form['f0'].attrs
        assert 'Check values then click submit.' not in resp.text
        assert resp.form['f0'].value == 'foo@localhost'

        # try it without validation page
        formdef.confirmation = False
        formdef.store()
        formdef.data_class().wipe()

        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp.form['f0'].value = 'Hello'  # try changing the value
        resp = resp.form.submit('submit')

        formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
        assert len(formdatas) == 1
        assert formdatas[0].data['0'] == 'foo@localhost'


def test_form_worklow_multiple_identical_status_with_wserror(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st1'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    login(app, username='foo', password='foo')

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.allows_backoffice_access = False
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    for _i in range(3):
        resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
        formdata = formdef.data_class().get(formdata.id)
        formdata.evolution[-1].add_part(JournalWsCallErrorPart('bla', 'bla', {}))
        formdata.store()

    resp = app.get(formdata.get_url())
    assert resp.text.count('<li class="msg') == 1

    role.allows_backoffice_access = True
    role.store()
    resp = app.get(formdata.get_url(backoffice=True))
    assert resp.text.count('<li class="msg') == 3


def test_form_page_verified_prefill_error_page(pub):
    user = create_user(pub)
    user.verified_fields = ['email']
    user.store()
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', required=False),
        fields.StringField(id='1', label='string2', required=True),
    ]
    formdef.store()

    for prefill_settings in (
        {'type': 'user', 'value': 'email'},  # verified profile
        {'type': 'string', 'value': 'foo@localhost', 'locked': True},  # locked value
    ):
        formdef.fields[0].prefill = prefill_settings
        formdef.store()
        formdef.data_class().wipe()
        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp = resp.form.submit('submit')
        assert 'There were errors processing the form' in resp.text
        assert 'readonly' in resp.form['f0'].attrs


def test_form_page_profile_verified_date_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.DateField(id='_date', label='date', type='date'))
    user_formdef.store()
    user.form_data = {'_date': time.strptime('2018-09-27', '%Y-%m-%d')}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.DateField(id='0', label='date', prefill={'type': 'user', 'value': '_date'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    assert 'readonly' not in resp.form['f0'].attrs
    resp.form['f0'].value = '2018-09-27'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'

    user.verified_fields = ['_date']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    assert 'readonly' in resp.form['f0'].attrs

    resp.form['f0'].value = '2018-09-24'  # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'  # it is reverted

    resp.form['f0'].value = '2018-09-24'  # try again changing the value
    resp = resp.form.submit('submit')

    formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
    assert len(formdatas) == 1
    assert time.strftime('%Y-%m-%d', formdatas[0].data['0']) == '2018-09-27'


def test_form_page_profile_verified_radio_item_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['bar@localhost', 'foo@localhost', 'baz@localhost'],
            display_mode='radio',
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value is None

    user.verified_fields = ['email']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'disabled' in resp.form['f0'].attrs
    for radio in resp.html.findAll('input'):
        if radio['name'] == 'f0':
            if radio['value'] == 'foo@localhost':
                assert radio.attrs.get('checked')
                assert not radio.attrs.get('disabled')
            else:
                assert not radio.attrs.get('checked')
                assert radio.attrs.get('disabled')

    resp.form['f0'].value = 'baz@localhost'  # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'foo@localhost'  # it is reverted


def test_item_field_from_cards(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].options == [
        ('2', False, 'bar'),
        ('3', False, 'baz'),
        ('1', False, 'foo'),
    ]
    resp.form['f0'] = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'
    assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'


def test_item_field_from_custom_view_on_cards(pub):
    pub.role_class.wipe()
    pub.custom_view_class.wipe()

    user = create_user(pub)
    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.is_admin = True
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.workflow_roles = {'_editor': user.roles[0]}
    carddef.fields = [
        fields.ItemField(id='0', type='item', label='item', varname='item', items=['foo', 'bar', 'baz']),
        fields.StringField(id='1', type='string', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    baz_ids = set()
    for i, value in enumerate(['foo', 'bar', 'baz'] * 10):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '0_display': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()
        if value == 'baz':
            baz_ids.add(str(carddata.id))

    # create custom view
    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/backoffice/data/items/')
    if pub.is_using_postgresql():
        assert resp.text.count('<tr') == 21  # thead + 20 items (max per page)
    else:
        assert resp.text.count('<tr') == 31  # thead + all items
    resp.forms['listing-settings']['filter-0'].checked = True
    resp = resp.forms['listing-settings'].submit()

    resp.forms['listing-settings']['filter-0-value'] = 'baz'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text.count('<tr') == 11  # thead + 10 items

    resp.forms['save-custom-view']['title'] = 'as data source'
    resp.forms['save-custom-view']['visibility'] = 'datasource'
    resp = resp.forms['save-custom-view'].submit()

    custom_view = pub.custom_view_class.select()[0]

    # use custom view as source
    ds = {'type': 'carddef:%s:%s' % (carddef.url_name, custom_view.slug)}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 10
    assert {x[0] for x in resp.form['f0'].options} == baz_ids
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] in baz_ids
    assert formdef.data_class().select()[0].data['0_structured']['item'] == 'baz'


def test_item_field_with_disabled_items(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit')  # -> validation page
        assert 'There were errors processing the form' in resp.text

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=False)
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('option[disabled=disabled][value="1"]')) == 0
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source=ds,
            display_mode='radio',
            display_disabled_items=True,
        )
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit')  # -> validation page
        assert 'There were errors processing the form' in resp.text


def test_items_field_with_disabled_items(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemsField(id='0', label='string', type='items', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['1', '2']
        assert formdef.data_class().select()[0].data['0_display'] == 'hello, world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'disabled' in resp.form['f0$element1'].attrs
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemsField(id='0', label='string', type='items', data_source=ds, display_disabled_items=False)
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'f0$element1' not in resp.form.fields
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'


def test_item_field_autocomplete_json_source(http_requests, pub, error_email, emails):
    if pub.is_using_postgresql():
        pub.loggederror_class.wipe()
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert resp.form['f0'].value == '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][1]

    # check hint is displayed within
    formdef.fields[0].hint = 'help text'
    formdef.store()
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert 'data-hint="help text"' in resp.text
        assert resp.form['f0'].value == ''

    formdef.fields[0].hint = ''
    formdef.store()

    # check with possibility of remote query
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    formdef.data_class().wipe()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?q=hell'
        assert resp2.json == data

        # check unauthorized access
        resp2 = get_app(pub).get(select2_url + '?q=hell', status=403)

    # check error handling in autocomplete endpoint
    formdef.data_class().wipe()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        urlopen.side_effect = ConnectionError('...')
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

        assert emails.count() == 0
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?q=hell'
        assert resp2.json == {'data': [], 'err': '1'}
        assert emails.count() == 0

        data_source.notify_on_errors = True
        data_source.record_on_errors = True
        data_source.store()
        resp2 = app.get(select2_url + '?q=hell')
        assert emails.count() == 1
        assert emails.get_latest('subject') == '[ERROR] [DATASOURCE] Error loading JSON data source (...)'
        if pub.is_using_postgresql():
            assert pub.loggederror_class.count() == 1
            logged_error = pub.loggederror_class.select()[0]
            assert logged_error.workflow_id is None
            assert logged_error.summary == '[DATASOURCE] Error loading JSON data source (...)'

        data_source.notify_on_errors = False
        data_source.store()

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> validation page
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?id=1'
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> submit
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?id=1'
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # same thing with numeric identifiers
    formdef.data_class().wipe()
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json-numeric-id'}
    data_source.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}, {'id': 2, 'text': 'world', 'extra': 'bar'}]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json-numeric-id?q=hell'
        assert resp2.json == data

        # check unauthorized access
        resp2 = get_app(pub).get(select2_url + '?q=hell', status=403)

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> validation page
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json-numeric-id?id=1'
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> submit
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json-numeric-id?id=1'
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # same thing with signed URLs
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.store()
    formdef.data_class().wipe()
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        fd.write(
            '''\
[wscall-secrets]
remote.example.net = 1234
'''
        )

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?q=hell&orig=example.net&')
        assert resp2.json == data

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> validation page
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = resp.form.submit('submit')  # -> submit
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # check with optional field
    formdef.data_class().wipe()
    formdef.fields[0].required = False
    formdef.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    resp = resp.form.submit('submit')  # -> validation page
    assert resp.form['f0'].value == ''
    assert resp.form['f0_label'].value == ''

    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] is None

    # check there's no crash if url is empty
    data_source.data_source = {'type': 'json', 'value': '{% if 0 %}http://remote.example.net/json{% endif %}'}
    data_source.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell', status=403)
        assert urlopen.call_count == 0


def test_item_field_autocomplete_jsonp_source(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        assert select2_url == 'http://remote.example.net/jsonp'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = [Hidden(form=resp.form, tag='input', name='f0_display', pos=10)]
    resp.form.field_order.append(('f0_display', resp.form.fields['f0_display'][0]))
    resp.form['f0'].force_value('1')
    resp.form['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit')  # -> validation page
        assert urlopen.call_count == 0
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit')  # -> submit
        assert urlopen.call_count == 0
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        # no _structured data for pure jsonp sources
        assert '0_structured' not in formdef.data_class().select()[0].data


def test_item_field_autocomplete_cards_source(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source=ds,
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    select2_url = resp.pyquery('select').attr['data-select2-url']
    resp2 = app.get(select2_url + '?q=ba')
    assert [x['text'] for x in resp2.json['data']] == ['bar', 'baz']
    resp.form['f0'].force_value(str(resp2.json['data'][0]['id']))
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'


def test_item_field_autocomplete_ezt_variable_jsonp(http_requests, pub):
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1',
            label='foo',
            type='item',
            varname='foo',
            data_source={'type': 'jsonp', 'value': '[site_url]/foo-jsonp'},
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(
            id='3',
            label='bar',
            type='item',
            data_source={'type': 'jsonp', 'value': '[site_url]/foo-jsonp?a=[form_var_foo_raw]'},
        ),
    ]
    formdef.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        assert select2_url == 'http://example.net/foo-jsonp'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f1_display'] = [Hidden(form=resp.form, tag='input', name='f1_display', pos=10)]
    resp.form.field_order.append(('f1_display', resp.form.fields['f1_display'][0]))
    resp.form['f1'].force_value('1')
    resp.form['f1_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit')  # -> 2nd page
        assert urlopen.call_count == 0
        assert resp.pyquery('select').attr['data-select2-url'] == 'http://example.net/foo-jsonp?a=1'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f3_display'] = [Hidden(form=resp.form, tag='input', name='f3_display', pos=10)]
    resp.form.field_order.append(('f3_display', resp.form.fields['f3_display'][0]))
    resp.form['f3'].force_value('2')
    resp.form['f3_display'].force_value('hello2')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submit
        assert urlopen.call_count == 0
        assert formdef.data_class().select()[0].data['1'] == '1'
        assert formdef.data_class().select()[0].data['1_display'] == 'hello'
        assert formdef.data_class().select()[0].data['3'] == '2'
        assert formdef.data_class().select()[0].data['3_display'] == 'hello2'
        # no _structured data for pure jsonp sources
        assert '1_structured' not in formdef.data_class().select()[0].data
        assert '3_structured' not in formdef.data_class().select()[0].data


def test_form_data_keywords(pub):
    formdef = create_formdef()
    formdef.keywords = 'hello,world'
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1


def test_logged_errors(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    Workflow.wipe()
    workflow = Workflow.get_default_workflow()
    workflow.id = '12'
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = 'rejected'
    jump.condition = {'type': 'python', 'value': '1//0'}  # ZeroDivisionError
    st1 = workflow.possible_status[0]
    st1.items.insert(0, jump)
    jump.parent = st1
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.id = '34'
    formdef.workflow = workflow
    formdef.name = 'test'
    formdef.confirmation = False
    formdef.fields = []
    formdef.store()

    pub.loggederror_class.wipe()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert pub.loggederror_class.count() == 1

    # new expression, but raise the same exception (ZeroDivisionError),
    # just update the created logged error
    jump.condition = {'type': 'python', 'value': '2//0'}
    workflow.store()
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert pub.loggederror_class.count() == 1

    error = list(
        pub.loggederror_class.get_with_indexed_value(
            'tech_id',
            '34-12-just_submitted-_jump-failed-to-evaluate-condition-ZeroDivisionError-integer-division-or-modulo-by-zero',
        )
    )[0]
    assert error.occurences_count == 2
    assert error.expression == '2//0'

    assert len(list(pub.loggederror_class.get_with_indexed_value('formdef_id', '34'))) == 1
    assert len(list(pub.loggederror_class.get_with_indexed_value('formdef_id', 'X'))) == 0

    assert len(list(pub.loggederror_class.get_with_indexed_value('workflow_id', '12'))) == 1
    assert len(list(pub.loggederror_class.get_with_indexed_value('workflow_id', 'X'))) == 0


def test_resubmit(pub):
    user = create_user(pub)

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='string', varname='toto')]
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'form title bis'
    formdef2.enable_tracking_codes = True
    formdef2.fields = [
        fields.StringField(id='1', label='string', varname='titi'),
        fields.StringField(id='2', label='string', varname='toto'),
    ]
    formdef2.store()

    wf = Workflow(name='resubmit')
    st1 = wf.add_status('Status1')
    st2 = wf.add_status('Status2')

    resubmit = ResubmitWorkflowStatusItem()
    resubmit.id = '_resubmit'
    resubmit.by = ['_submitter']
    resubmit.formdef_slug = formdef2.url_name
    st1.items.append(resubmit)
    resubmit.parent = st1

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1

    register_comment = RegisterCommenterWorkflowStatusItem()
    register_comment.id = '_register'
    register_comment.comment = '<p><a href="[resubmit_formdata_draft_url]">new draft</a></p>'
    st2.items.append(register_comment)
    register_comment.parent = st2

    wf.store()

    formdef.workflow_id = wf.id
    formdef.store()

    formdef2.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.data = {'1': 'XXX'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'XXX'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'XXX'

    # anonymous
    formdef2.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/form-title/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'foo'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'foo'


def test_form_custom_select_template(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar', 'Baz']
        )
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' not in resp.text
    formdef.fields[0].extra_css_class = 'template-test'
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' in resp.text
    # make sure request is available in context
    assert '<!-- backoffice: False -->' in resp.text
    assert '<!-- backoffice compat: False -->' in resp.text

    # test for substitution variables being available
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/')
    assert 'substitution variable: http://remote.example.net/' in resp.text


def test_form_status_appearance_keywords(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar', 'Baz']
        )
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    root = PublicFormStatusPage(formdef, formdata, register_workflow_subdirs=False)
    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == root.status_templates

    formdef.appearance_keywords = 'foobar plop'
    formdef.store()

    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == [
        'wcs/front/appearance-foobar/formdata_status.html',
        'wcs/front/appearance-plop/formdata_status.html',
        'wcs/front/formdata_status.html',
        'wcs/appearance-foobar/formdata_status.html',
        'wcs/appearance-plop/formdata_status.html',
        'wcs/formdata_status.html',
    ]

    resp = get_app(pub).get('/test/')
    assert 'class="quixote foobar plop"' in resp.text


def test_user_global_action(pub):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert 'button-action-1' not in resp.text

    trigger.roles = ['_submitter']
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_user_global_action_same_status_store(pub):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'

    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string'),
    ]
    action = workflow.add_global_action('FOOBAR')
    jump = action.append_item('jump')
    jump.status = 'new'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    new_status = workflow.possible_status[1]

    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = new_status
    setbo.fields = [{'field_id': 'bo1', 'value': '123'}]
    new_status.items = [setbo] + new_status.items

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['bo1'] == '123'

    # change global action
    setbo.fields = [{'field_id': 'bo1', 'value': '321'}]
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')  # click global action

    # check status actions are rerun
    resp = app.get(formdata.get_url())
    assert formdef.data_class().get(formdata.id).status == 'wf-new'
    assert formdef.data_class().get(formdata.id).data['bo1'] == '321'


def test_anonymous_user_global_action(pub):
    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.enable_tracking_codes = True
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = get_app(pub)
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    app.cookiejar.clear()

    resp = app.get('/')
    resp.forms[0]['code'] = formdata.tracking_code
    resp = resp.forms[0].submit().follow().follow()

    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_condition_on_action(pub, emails):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.id = '2'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test condition on action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form (test condition on action)')  # no receiver
    assert emails.get('New form2 (test condition on action)')  # submitter

    emails.empty()

    workflow.possible_status[0].items[1].condition = {'type': 'python', 'value': 'False'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with a condition on field data
    formdef.fields = [fields.StringField(id='0', label='string', varname='foobar')]
    formdef.store()
    workflow.possible_status[0].items[1].condition = {'type': 'django', 'value': 'form_var_foobar'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = ''
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with condition evaluating positively
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = 'toto'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert emails.get('New form2 (test condition on action)')


def test_user_global_action_along_form(pub):
    # check it's possible to click on a global action button even if there's a
    # form with required fields.
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    status = workflow.get_status('new')
    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.StringField(id='1', label='blah', type='string', required=True))
    status.items.append(display_form)
    display_form.parent = status

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert resp.form['fxxx_1'].attrs['aria-required'] == 'true'
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_email_actions(pub, emails):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.possible_status[0].items[
        1
    ].body = 'Hello; {% action_button "do-accept" label="Accepté!" %} Adiós.'
    workflow.possible_status[1].items[1].identifier = 'do-accept'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test email action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    assert '/actions/' in action_url
    if docutils:
        assert len(email_data['payloads']) == 2
        assert action_url in force_text(email_data['payloads'][1])

    app = get_app(pub)
    resp = app.get(action_url)
    assert 'Accepté!' in resp.text
    resp = resp.form.submit()
    assert 'The action has been confirmed.' in resp.text
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'wf-accepted'

    # action token has been used, it will now return a custom 404
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # check against independently changed status, it should also return a
    # custom 404.
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    formdata = formdef.data_class().select()[0]
    formdata.jump_status('rejected')
    app = get_app(pub)
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # two buttons on the same line, two urls
    workflow.possible_status[0].items[
        1
    ].body = '{% action_button "ok" label="OK" %}{% action_button "ko" label="KO" %} '
    workflow.store()
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    assert len(re.findall(r'http.*? ', email_data['payload'])) == 2


def test_manager_public_access(pub):
    user, manager = create_user_and_admin(pub)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    manager.is_admin = False
    manager.roles = [role.id]
    manager.store()
    assert manager.can_go_in_backoffice()

    formdef = create_formdef()
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    # user access to own formdata
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to formdata
    app = login(get_app(pub), username='admin', password='admin')
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    # agent access to an unauthorized formdata
    formdef.workflow_roles = {'_receiver': None}
    formdef.store()
    resp = app.get(formdata.get_url(), status=403)

    # agent access via a tracking code (stays in frontoffice)
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.enable_tracking_codes = True
    formdef.store()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp = app.get('/code/%s/load' % code.id)
    resp = resp.follow()  # -> /test/1/
    assert 'backoffice' not in resp.request.path
    assert 'The form has been recorded' in resp.text

    # authorized access but not backoffice access
    app = login(get_app(pub), username='admin', password='admin')  # reset session
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)  # check tracking code is no longer effective
    role.allows_backoffice_access = False
    role.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to own formdata (stays in frontoffice)
    formdata = formdef.data_class()()
    formdata.user_id = manager.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text


def test_form_and_category_same_slug(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.store()

    # check we get to the form, not the category
    resp = get_app(pub).get('/foobar/')
    assert resp.form


def test_field_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(
            type='string',
            id='1',
            label='Bar',
            size='40',
            required=True,
            condition={'type': 'django', 'value': '1'},
        ),
        fields.StringField(
            type='string',
            id='2',
            label='Foo',
            size='40',
            required=True,
            condition={'type': 'django', 'value': '0'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' not in resp.form.fields
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' not in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' not in resp.text


def test_field_unicode_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar == "éléphant"'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_contains_in_list(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar in "éléphant"|split'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_contains_in_string(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': '"éléphant" in form_var_bar'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_in_array(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemsField(
            id='1',
            label='items',
            type='items',
            required=True,
            varname='foo',
            items=['Pomme', 'Poire', 'Pêche', 'Abricot'],
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            type='string',
            id='3',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': '"Pêche" in form_var_foo'},
        ),
        fields.CommentField(type='comment', id='4', label='{{form_var_foo}}'),
        fields.CommentField(
            type='comment', id='5', label='{% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'
        ),
    ]

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'Message {% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1
    workflow.store()

    formdef.workflow = workflow
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp = resp.form.submit('submit')
    assert 'f3' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'f3' in resp.form.fields  # check it's ok in field condition
    resp.form['f3'] = 'hop'
    assert '>Poire, Pêche<' in resp.text  # check it's displayed correctly
    assert 'CHECK OK' in resp.text  # check it's ok in template condition
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert '<p>Message CHECK OK</p>' in resp.text  # check it's ok in workflow template


def test_form_edit_and_backoffice_field_change(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string', varname='plop'),
    ]
    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_foo'}]
    setbo2 = SetBackofficeFieldsWorkflowStatusItem()
    setbo2.parent = st1
    setbo2.fields = [{'field_id': 'bo1', 'value': '="foo" + form_var_plop'}]
    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items = [setbo, setbo2, jump]

    st2 = workflow.add_status('Status2', 'st2')

    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter']
    st2.items.append(editable)
    editable.parent = st2
    editable.status = st1.id
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submitted
    assert 'The form has been recorded' in resp.text

    data_id = formdef.data_class().select()[0].id
    assert formdef.data_class().get(data_id).data['bo1'] == 'foobar'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % data_id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable')
    resp = resp.follow()
    assert resp.form['f1'].value == 'bar'
    resp.form['f1'].value = 'baz'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit').follow()  # -> saved

    assert formdef.data_class().get(data_id).data['bo1'] == 'foobaz'


def test_backoffice_fields_just_after_conditional_form_submit(pub):
    """
    simulate selection of a structured list via condition on form,
    followed by an evaluation on workflow in order to get structured value
    from the selected list.
    ie: test unfeed on ConditionVars
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='first text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='first more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2', 'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'},
    ]
    st1.items.append(setbo)
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more': 'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more': 'moreB1'}, {'id': '2', 'text': 'B2', 'more': 'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'], type='item', label='list to choice'),
        fields.ItemField(
            id='2',
            varname='listA',
            type='item',
            label='list A',
            data_source={'type': 'formula', 'value': str(items_A)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "A"'},
        ),
        fields.ItemField(
            id='3',
            varname='listB',
            type='item',
            label='list B',
            data_source={'type': 'formula', 'value': str(items_B)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "B"'},
        ),
    ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user_and_admin(pub)
    resp = get_app(pub).get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'


def test_backoffice_fields_just_after_conditional_form_edit_action(pub):
    """
    test unfeed on ConditionVars within edit context
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='both text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='both more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2', 'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'},
    ]
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter']
    editable.parent = st1
    editable.status = st1.id
    st1.items = [setbo, editable]
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more': 'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more': 'moreB1'}, {'id': '2', 'text': 'B2', 'more': 'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'], type='item', label='list to choice'),
        fields.ItemField(
            id='2',
            varname='listA',
            type='item',
            label='list A',
            data_source={'type': 'formula', 'value': str(items_A)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "A"'},
        ),
        fields.ItemField(
            id='3',
            varname='listB',
            type='item',
            label='list B',
            data_source={'type': 'formula', 'value': str(items_B)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "B"'},
        ),
    ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    # check unfeed on FormPage::submitted()
    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % formdata.id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable').follow()
    assert resp.form['f1'].value == 'B'
    resp.form['f1'].value = 'A'
    resp = resp.form.submit('submit').follow()  # -> saved
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'A'
    assert formdata.data['2'] == '1'
    assert formdata.data.get('3') is None

    # check unfeed on FormPage::submitted_existing()
    assert formdata.data['bo1'] == 'A1 vs None'
    assert formdata.data['bo2'] == 'moreA1 vs '


def test_backoffice_fields_set_from_live(pub):
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='first text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='first more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var.foo.attr }}'},
        {'field_id': 'bo2', 'value': '{{ form_var.foo.live.var.attr }}'},
    ]
    st1.items.append(setbo)
    workflow.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='string', type='item', varname='foo', data_source=ds, display_disabled_items=True
        )
    ]
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user_and_admin(pub)
    resp = get_app(pub).get('/test/')

    resp.form['f1'].value = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == '2'
    assert formdata.data['bo1'] == 'attr1'
    assert formdata.data['bo2'] == 'attr1'


def test_form_recall_draft(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' not in resp.text

    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s/"' % draft.id in resp.text

    draft2 = formdef.data_class()()
    draft2.user_id = user.id
    draft2.status = 'draft'
    draft2.data = {}
    draft2.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s/"' % draft.id in resp.text
    assert 'href="%s/"' % draft2.id in resp.text


def test_frontoffice_workflow_form_with_conditions(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(id='2', label='Test2', varname='str2', type='string', required=True),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    # check with static condition
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': '0'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' not in resp.form.fields

    # check condition based on formdata
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'form_var_plop'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'form_var_plop != "xxx"'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    # check with live conditions
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'blah_var_str == "xxx"'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': 'blah_1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': 'blah_2'}).attrs.get('style') == 'display: none'
    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['fblah_1'] = ''
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_1']['visible']
    assert not live_resp.json['result']['blah_2']['visible']

    resp.form['fblah_1'] = 'xxx'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_1']['visible']
    assert live_resp.json['result']['blah_2']['visible']

    # check submit doesn't work
    resp = resp.form.submit('submit')
    assert 'There were errors processing your form.' in resp.text

    resp.form['fblah_1'] = 'xxx2'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_1']['visible']
    assert not live_resp.json['result']['blah_2']['visible']

    # check submit does work when second field is hidden
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().get(formdata.id).workflow_data == {
        'blah_var_str': 'xxx2',
        'blah_var_str2': None,
    }


def test_frontoffice_workflow_form_with_dynamic_comment(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.CommentField(id='2', label='value is {{blah_var_str}}', type='comment'),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields

    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['fblah_1'] = 'test'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_2']['visible']
    assert live_resp.json['result']['blah_2']['content'] == '<p>value is test</p>'


def test_frontoffice_workflow_form_with_impossible_condition(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(
            id='1',
            label='Test',
            varname='str',
            type='string',
            condition={'type': 'django', 'value': '0 == 1'},
        ),
        fields.StringField(
            id='2',
            label='Test2',
            type='string',
            condition={'type': 'django', 'value': 'blah_var_str == "toto"'},
        ),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' not in resp.form.fields
    assert resp.html.find('div', {'data-field-id': 'blah_2'}).attrs.get('style') == 'display: none'


def test_choice_button_ignore_form_errors(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.roles = [logged_users_role().id]
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    st2 = wf.add_status('Status2', 'st2')

    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = [logged_users_role().id]
    commentable.required = True
    st1.items.append(commentable)
    commentable.parent = st1

    choice = ChoiceWorkflowStatusItem()
    choice.label = 'Submit'
    choice.by = [logged_users_role().id]
    choice.id = '_x1'
    choice.status = st2.id
    st1.items.append(choice)
    choice.parent = st1

    choice2 = ChoiceWorkflowStatusItem()
    choice2.label = 'Submit no check'
    choice2.by = [logged_users_role().id]
    choice2.id = '_x2'
    choice2.status = st2.id
    choice2.ignore_form_errors = True
    st1.items.append(choice2)
    choice2.parent = st1

    wf.store()

    formdef.workflow = wf
    formdef.store()

    # no comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x1')
    assert 'There were errors processing your form.' in resp.text

    # comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('button_x1').follow()
    assert '<p>plop</p>' in resp.text
    assert '<span class="status">Status2' in resp.text

    # no comment but no check
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x2').follow()
    assert '<span class="status">Status2' in resp.text


def test_form_comment_is_hidden_attribute(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string 1', varname='choice1'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3',
            label='string 2',
            varname='choice2',
            condition={'type': 'django', 'value': 'form_var_choice1 == "1"'},
        ),
        fields.CommentField(
            id='5',
            label='this should not be displayed',
            type='comment',
            condition={'type': 'django', 'value': 'False and form_var_choice2 == "???"'},
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = '1'
    resp = resp.forms[0].submit('submit')
    comment = re.compile('.*comment-field.*"')
    assert 'style="display: none"' in comment.search(resp.forms[0].text).group(0)
    resp = resp.forms[0].submit('previous')
    resp.forms[0]['f1'] = '2'
    resp = resp.forms[0].submit('submit')
    assert 'style="display: none"' in comment.search(resp.forms[0].text).group(0)


@pytest.fixture
def create_formdata(pub):
    FormDef.wipe()

    data = [
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'},
    ]
    ds = {
        'type': 'formula',
        'value': repr(data),
    }
    source_formdef = FormDef()
    source_formdef.name = 'source form'
    source_formdef.fields = [
        fields.StringField(id='0', label='string', varname='toto_string'),
        fields.FileField(id='1', label='file', type='file', varname='toto_file'),
        fields.ItemField(
            id='2', label='item', required=False, data_source=ds, varname='toto_item', hint='hint'
        ),
    ]
    source_formdef.store()

    target_formdef = FormDef()
    target_formdef.name = 'target form'
    target_formdef.enable_tracking_codes = True
    target_formdef.fields = [
        fields.StringField(id='0', label='string', varname='foo_string'),
        fields.FileField(id='1', label='file', type='file', varname='foo_file'),
        fields.ItemField(id='2', label='item', data_source=ds, varname='foo_item'),
    ]
    target_formdef.store()
    wf = Workflow(name='create-formdata')

    st1 = wf.add_status('New')
    st2 = wf.add_status('Resubmit')

    jump = ChoiceWorkflowStatusItem()
    jump.id = '_resubmit'
    jump.label = 'Resubmit'
    jump.by = ['_submitter']
    jump.status = st2.id
    jump.parent = st1
    st1.items.append(jump)

    create_formdata = CreateFormdataWorkflowStatusItem()
    create_formdata.varname = 'resubmitted'
    create_formdata.draft = True
    create_formdata.id = '_create_formdata'
    create_formdata.formdef_slug = target_formdef.url_name
    create_formdata.mappings = [
        Mapping(field_id='0', expression='=form_var_toto_string'),
        Mapping(field_id='1', expression='=form_var_toto_file_raw'),
        Mapping(field_id='2', expression='=form_var_toto_item_raw'),
    ]
    create_formdata.parent = st2
    st2.items.append(create_formdata)

    redirect = RedirectToUrlWorkflowStatusItem()
    redirect.id = '_redirect'
    redirect.url = '{{ form_links_resubmitted.form_url }}'
    redirect.parent = st2
    st2.items.append(redirect)

    display = DisplayMessageWorkflowStatusItem()
    display.id = '_display'
    display.message = '''<div class="linked">{% if form_links_resubmitted %}
<p>Linked status: <span class="status">{{ form_links_resubmitted.form_status }}</span></p>
<p>Target formdata field: <span class="foo_string">{{ form_links_resubmitted.form_var_foo_string }}</span></p>
{% endif %}</div>'''
    display.to = []
    st2.items.append(display)

    wf.store()
    source_formdef.workflow_id = wf.id
    source_formdef.store()
    return locals()


def test_create_formdata_anonymous_draft(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'

    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'draft'
    assert target_formdata.submission_context == {
        'orig_object_type': 'formdef',
        'orig_formdata_id': str(create_formdata['source_formdef'].data_class().select()[0].id),
        'orig_formdef_id': str(create_formdata['source_formdef'].id),
    }

    resp = resp.follow()
    resp = resp.follow()
    assert 'zob' in resp
    resp = resp.forms[1].submit('submit')  # -> validation
    resp = resp.forms[1].submit('submit')  # -> submission
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('2') == '2'
    assert target_formdata.data.get('2_display') == 'deux'
    assert target_formdata.data.get('2_structured') == {'text': 'deux', 'id': '2', 'more': 'bar'}
    assert target_formdata.status == 'wf-new'

    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
    resp = app.get(source_formdata.get_url())
    pq = resp.pyquery.remove_namespaces()
    assert pq('.linked .status').text() == 'New'
    assert pq('.linked .foo_string').text() == 'zob'


def test_create_formdata_anonymous_submitted(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    # submit directly
    create_formdata['wf'].get_status('2').items[0].draft = False
    create_formdata['wf'].store()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'

    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'wf-new'
    assert target_formdata.submission_context == {
        'orig_object_type': 'formdef',
        'orig_formdata_id': str(create_formdata['source_formdef'].data_class().select()[0].id),
        'orig_formdef_id': str(create_formdata['source_formdef'].id),
    }

    resp = resp.follow()
    assert 'New' in resp
    assert 'zob' in resp
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'wf-new'

    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
    resp = app.get(source_formdata.get_url())
    pq = resp.pyquery.remove_namespaces()
    assert pq('.linked .status').text() == 'New'
    assert pq('.linked .foo_string').text() == 'zob'


def test_create_formdata_empty_item_ds_with_id_parameter(pub, create_formdata):
    if pub.is_using_postgresql():
        pub.loggederror_class.wipe()
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/json',
    }
    data_source.id_parameter = 'id'
    data_source.store()
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()
    create_formdata['source_formdef'].fields[2].data_source = {'type': 'foobar'}
    create_formdata['source_formdef'].store()
    create_formdata['target_formdef'].fields[2].data_source = {'type': 'foobar'}
    create_formdata['target_formdef'].store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': create_formdata['data']}
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))

        app = get_app(create_formdata['pub'])
        resp = app.get('/source-form/')
        resp.form['f0'] = 'zob'
        resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submission
        resp = resp.follow()
        assert create_formdata['target_formdef'].data_class().count() == 0
        if pub.is_using_postgresql():
            assert pub.loggederror_class.count() == 0
        resp = resp.form.submit('button_resubmit')
        if pub.is_using_postgresql():
            assert pub.loggederror_class.count() == 0


def test_create_formdata_locked_prefill_parent(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    target_formdef = create_formdata['target_formdef']
    target_formdef.fields[0].prefill = {
        'type': 'string',
        'value': '{{form_parent_form_var_toto_string}}',
        'locked': True,
    }
    target_formdef.store()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'
    assert target_formdata.status == 'draft'

    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'zob'
    assert resp.forms[1]['f0'].attrs['readonly']
    # try altering readonly field
    resp.forms[1]['f0'].value = 'xxx'
    resp = resp.forms[1].submit('submit')
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'zob'
    resp = resp.forms[1].submit('submit')
    resp = resp.forms[1].submit('submit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data['0'] == 'zob'


def test_js_libraries(pub):
    create_formdef()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' not in resp.text
    assert 'jquery.min.js' in resp.text
    assert 'qommon.forms.js' in resp.text

    pub.cfg['debug'] = {'debug_mode': True}
    pub.write_cfg()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' in resp.text
    assert 'jquery.min.js' not in resp.text
    assert 'qommon.forms.js' in resp.text

    pub.cfg['branding'] = {'included_js_libraries': ['jquery.js']}
    pub.write_cfg()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' not in resp.text
    assert 'jquery.min.js' not in resp.text
    assert 'qommon.forms.js' in resp.text


def test_after_submit_location(pub):
    create_user(pub)

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = [logged_users_role().id]
    commentable.required = True
    st1.items.append(commentable)
    commentable.parent = st1

    workflow.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')  # form page
    resp = resp.forms[0].submit('submit')  # confirmation page
    resp = resp.follow()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/test/1/#action-zone'
    resp = resp.follow()

    display = DisplayMessageWorkflowStatusItem()
    display.message = 'message-to-all'
    display.to = []
    st1.items.append(display)
    display.parent = st1
    workflow.store()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/test/1/#'


def test_form_honeypot(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', required=False)]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f00'] = 'honey?'
    resp = resp.forms[0].submit('submit')
    assert 'Honey pot should be left untouched.' in resp


def test_structured_workflow_options(pub):
    create_user_and_admin(pub)

    workflow = Workflow(name='test')
    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
    data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un', 'more': 'foo'}, {'id': '2', 'text': 'deux', 'more': 'bar'}]),
    }
    workflow.variables_formdef.fields = [
        fields.StringField(id='1', label='Test', type='string', varname='foo'),
        fields.ItemField(id='2', label='Test List', type='item', varname='bar', data_source=data_source),
        fields.ItemsField(id='3', label='Test Multi', type='items', varname='baz', data_source=data_source),
        fields.DateField(id='4', label='Date', type='date', varname='date'),
    ]
    st1 = workflow.add_status('Status1', 'st1')
    comment = RegisterCommenterWorkflowStatusItem()
    comment.id = '_comment'
    comment.comment = 'Date option: {{ form_option_date }}'
    st1.items.append(comment)
    comment.parent = st1
    workflow.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='Test List',
            type='item',
            varname='bar',
            data_source={'type': 'formula', 'value': 'form_option_baz_structured'},
        ),
    ]
    formdef.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    # configure workflow options
    resp = login(get_app(pub), username='admin', password='admin').get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click('Options')
    resp.form['f1'].value = 'plop'
    resp.form['f2'].value = '1'
    resp.form['f3$element1'].checked = True
    resp.form['f4'].value = '2020-04-18'
    resp = resp.form.submit('submit')

    formdef = FormDef.get(formdef.id)
    assert formdef.workflow_options == {
        'foo': 'plop',
        'bar': '1',
        'bar_display': 'un',
        'bar_structured': {'id': '1', 'more': 'foo', 'text': 'un'},
        'baz': ['1'],
        'baz_display': 'un',
        'baz_structured': [{'id': '1', 'more': 'foo', 'text': 'un'}],
        'date': time.strptime('2020-04-18', '%Y-%m-%d'),
    }

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert resp.form['f1'].options == [('1', False, 'un')]
    resp = resp.form.submit('submit')  # form page
    resp = resp.form.submit('submit')  # confirmation page
    resp = resp.follow()

    formdata = formdef.data_class().select()[0]
    assert formdata.data == {
        '1': '1',
        '1_display': 'un',
        '1_structured': {'id': '1', 'text': 'un', 'more': 'foo'},
    }
    assert '2020-04-18' in formdata.evolution[0].parts[0].content


def test_exclude_self_condition(pub):
    FormDef.wipe()

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(
            id='1',
            label='1st page',
            type='page',
            post_conditions=[
                {
                    'condition': {
                        'type': 'django',
                        'value': 'form_objects|filter_by:"foo"|filter_value:form_var_foo|exclude_self|count == 0',
                    },
                    'error_message': 'You shall not pass.',
                }
            ],
        ),
        fields.StringField(id='1', label='string', type='string', varname='foo'),
    ]

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = get_app(pub)
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp.text
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()

    # edit is ok
    resp = resp.form.submit('button_editable').follow()
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp

    # 2nd submission
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' in resp.text

    # submission with other value
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'other'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp.text
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()

    # edit is ok
    resp = resp.form.submit('button_editable').follow()
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' in resp


def test_form_item_map_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'geojson',
        'value': 'http://remote.example.net/geojson',
    }
    data_source.id_property = 'id'
    data_source.label_template_property = '{{ text }}'
    data_source.cache_duration = '5'
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', label='map', display_mode='map'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url'] == ''
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-radio-name'] == 'f1$marker_id'

    formdef.fields[0].data_source = {'type': 'foobar'}
    formdef.store()
    resp = app.get('/test/')
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url'] == '/api/geojson/foobar'
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-radio-name'] == 'f1$marker_id'
    app.get('/api/geojson/wrong-foobar', status=404)
    resp_geojson = app.get('/api/geojson/foobar')
    assert len(resp_geojson.json['features']) == 2
    assert http_requests.count() == 1
    assert http_requests.get_last('url') == 'http://remote.example.net/geojson'
    resp_geojson = app.get('/api/geojson/foobar')
    assert http_requests.count() == 1  # cache was used
    assert len(resp_geojson.json['features']) == 2
    # simulate qommon.map.js that will create radio inputs
    resp.form.fields['f1$marker_id'] = [Radio(form=resp.form, tag='input', name='f1$marker_id', pos=5)]
    resp.form.fields['f1$marker_id'][0].options.append(('1', False, None))
    resp.form.fields['f1$marker_id'][0].options.append(('2', False, None))
    resp.form.fields['f1$marker_id'][0].optionPositions.append(5)
    resp.form.fields['f1$marker_id'][0].optionPositions.append(6)
    resp.form.field_order.append(('f1$marker_id', resp.form.fields['f1$marker_id'][0]))
    resp.form['f1$marker_id'].value = '1'  # click on marker
    resp.form['f1$latlng'] = '1;2'  # set via js
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    # selected option is displayed as readonly:
    assert resp.pyquery('input[type=text][value=foo][readonly]')
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp
    assert '<div class="value">foo</div>' in resp
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    formdata = formdef.data_class().get(data_id)
    assert formdata.data['1_structured']['geometry']['coordinates'] == [1, 2]


def test_form_item_dynamic_map_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'geojson',
        'value': 'http://remote.example.net/geojson?{{ form_var_test }}',
    }
    data_source.id_property = 'id'
    data_source.label_template_property = '{{ text }}'
    data_source.cache_duration = '5'
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='test'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(id='3', label='map', display_mode='map', data_source={'type': 'foobar'}),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'plop'
    resp = resp.form.submit('submit')  # -> 2nd page
    markers_url = resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url']
    assert markers_url.startswith('/api/geojson/')
    resp_geojson = app.get(markers_url)
    assert len(resp_geojson.json['features']) == 2
    assert http_requests.count() == 1
    assert http_requests.get_last('url') == 'http://remote.example.net/geojson?plop'
    resp_geojson = app.get(markers_url)
    assert http_requests.count() == 1  # cache was used
    assert len(resp_geojson.json['features']) == 2


def test_form_item_timetable_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/api/datetimes',
    }
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', label='datetime', display_mode='timetable', data_source={'type': 'foobar'}),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            "data": [
                {"id": "1", "datetime": "2021-01-12 10:00:00", "text": "event 1"},
                {"id": "2", "datetime": "2021-01-13 10:20:00", "text": "event 2"},
                {"id": "3", "datetime": "2021-01-14 10:40:00", "text": "event 3"},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))

        resp = app.get('/test/')
        assert 'data-date="2021-01-12"' in resp and 'data-time="10:00"' in resp
        assert 'data-date="2021-01-13"' in resp and 'data-time="10:20"' in resp
        assert 'data-date="2021-01-14"' in resp and 'data-time="10:40"' in resp
        resp.form['f1'] = '2'  # would happen via javascript
        resp = resp.form.submit('submit')
        resp = resp.form.submit('submit')

        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        formdata = formdef.data_class().get(data_id)
        assert formdata.data == {
            '1': '2',
            '1_display': 'event 2',
            '1_structured': {'id': '2', 'datetime': '2021-01-13 10:20:00', 'text': 'event 2'},
        }


def test_form_item_timetable_data_source_with_date_alignment(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/api/datetimes',
    }
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='1', label='page1', type='page'),
        fields.DateField(id='2', label='date', type='date', varname='date'),
        fields.PageField(id='3', label='page2', type='page'),
        fields.ItemField(
            id='4',
            label='datetime',
            display_mode='timetable',
            data_source={'type': 'foobar'},
            initial_date_alignment='{{ form_var_date }}',
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {
            "data": [
                {"id": "1", "datetime": "2021-01-12 10:00:00", "text": "event 1"},
                {"id": "2", "datetime": "2021-01-13 10:20:00", "text": "event 2"},
                {"id": "3", "datetime": "2021-01-14 10:40:00", "text": "event 3"},
            ]
        }
        urlopen.side_effect = lambda *args: io.StringIO(json.dumps(data))

        resp = app.get('/test/')
        resp.form['f2'] = '2021-01-14'
        resp = resp.form.submit('submit')  # -> 2nd page

        assert 'var ALIGN_DATE = "2021-01-14";' in resp
        resp.form['f4'] = '2'  # would happen via javascript
        resp = resp.form.submit('submit')
        resp = resp.form.submit('submit')
        assert formdef.data_class().count() == 1


def test_file_prefill_on_edit(pub, http_requests):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file', varname='foo_file')]
    formdef.store()
    formdef.data_class().wipe()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('New', 'st1')
    st2 = workflow.add_status('CreateFormdata')

    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1

    jump = ChoiceWorkflowStatusItem()
    jump.id = '_resubmit'
    jump.label = 'Resubmit'
    jump.by = ['_submitter']
    jump.status = st2.id
    jump.parent = st1
    st1.items.append(jump)

    create_formdata = CreateFormdataWorkflowStatusItem()
    create_formdata.id = '_create_formdata'
    create_formdata.formdef_slug = formdef.url_name
    create_formdata.mappings = [
        Mapping(field_id='0', expression='{{form_var_foo_file}}'),
    ]
    create_formdata.parent = st2
    st2.items.append(create_formdata)

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    upload = Upload('test.txt', b'foobar', 'text/plain')
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit

    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

    formdata = formdef.data_class().select()[0]
    formdata.user_id = user.id
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % formdata.id)
    assert 'button_editable-button' in resp.text

    # go to edition page
    resp = resp.form.submit('button_editable').follow()
    # file is "prefilled"
    assert 'test.txt' in resp.text

    # go back to form page and trigger formdata creation
    resp = app.get('/test/%s/' % formdata.id)
    resp = resp.form.submit('button_resubmit')
    assert resp.status == '303 See Other'
    resp = resp.follow()

    assert formdef.data_class().count() == 2
    new_formdata = formdef.data_class().select(lambda x: str(x.id) != str(formdata.id))[0]
    assert new_formdata.data['0'].orig_filename == 'test.txt'
    assert new_formdata.data['0'].get_content() == b'foobar'

    resp = app.get('/test/%s/' % new_formdata.id)
    assert 'button_editable-button' in resp.text
    # go to edition page
    resp = resp.form.submit('button_editable').follow()
    # file is "prefilled"
    assert 'test.txt' in resp.text
    # and persist after being saved again
    resp = resp.form.submit('submit').follow()
    assert '<span>test.txt</span>' in resp.text