wcs/tests/form_pages/test_all.py

import hashlib
import io
import itertools
import json
import os
import re
import time
import urllib.parse
import xml.etree.ElementTree as ET
import zipfile
from unittest import mock

import pytest
import responses
from django.utils.encoding import force_bytes, force_text
from webtest import Hidden, Radio, Upload

from wcs import fields
from wcs.blocks import BlockDef
from wcs.carddef import CardDef
from wcs.categories import Category
from wcs.data_sources import NamedDataSource
from wcs.formdef import FormDef
from wcs.forms.root import PublicFormStatusPage
from wcs.qommon import force_str
from wcs.qommon.emails import docutils
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.qommon.misc import ConnectionError
from wcs.qommon.substitution import CompatibilityNamesDict
from wcs.qommon.template import Template
from wcs.roles import logged_users_role
from wcs.tracking_code import TrackingCode
from wcs.wf.create_formdata import Mapping
from wcs.wf.form import WorkflowFormFieldsFormDef
from wcs.wf.wscall import JournalWsCallErrorPart
from wcs.workflows import (
    AttachmentEvolutionPart,
    Workflow,
    WorkflowBackofficeFieldsFormDef,
    WorkflowVariablesFieldsFormDef,
)

from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login


def assert_equal_zip(stream1, stream2):
    with zipfile.ZipFile(stream1) as z1, zipfile.ZipFile(stream2) as z2:
        assert set(z1.namelist()) == set(z2.namelist())
        for name in z1.namelist():
            if name == 'styles.xml':
                continue
            if name in ['content.xml', 'meta.xml']:
                t1, t2 = ET.tostring(ET.XML(z1.read(name))), ET.tostring(ET.XML(z2.read(name)))
                try:
                    # >= python 3.8: tostring preserves attribute order; use canonicalize to sort them
                    t1, t2 = ET.canonicalize(t1), ET.canonicalize(t2)
                except AttributeError:
                    pass
            else:
                t1, t2 = z1.read(name), z2.read(name)
            assert t1 == t2, 'file "%s" differs' % name


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['sql', 'sql-lazy'], indirect=True)


@pytest.fixture
def pub(request):
    pub = create_temporary_pub(lazy_mode=bool('lazy' in request.param))
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    Category.wipe()
    cat = Category(name='foobar')
    cat.store()

    return pub


@pytest.fixture
def error_email(pub):
    pub.cfg['debug'] = {'error_email': 'errors@localhost.invalid'}
    pub.write_cfg()
    pub.set_config()


def teardown_module(module):
    clean_temporary_pub()


def create_formdef():
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()
    return formdef


def create_user(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.name = 'User Name'
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()
    return user


def create_user_and_admin(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()

    admin = pub.user_class()
    admin.email = 'admin@localhost'
    admin.is_admin = True
    admin.store()
    account = PasswordAccount(id='admin')
    account.set_password('admin')
    account.user_id = admin.id
    account.store()
    return user, admin


def test_home(pub):
    create_formdef()
    home = get_app(pub).get('/')
    assert 'category-misc' in home.text
    assert '<a class="" href="test/">test</a>' in home.text


def test_home_with_user_forms(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.category_id = '1'
    wf = Workflow(name='status')
    wf.add_status('Status1', 'st1')
    wf.store()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-st1'
    formdata.data = {}
    formdata.store()
    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="/test/%s/"' % formdata.id in resp
    assert 'Draft' in resp
    assert '<a href="/test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s/' % draft.id, status=302)
    resp = resp.follow(status=200)

    # disable formdef: formdatas are still visible and accessible, drafts are not
    formdef.disabled = True
    formdef.store()
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="/test/%s/"' % formdata.id in resp
    assert 'Draft' not in resp
    assert '<a href="test/%s"' % draft.id not in resp
    resp = app.get('/test/%s/' % formdata.id)
    resp = app.get('/test/%s/' % draft.id, status=302)
    resp = resp.follow(status=403)


def test_home_category(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'category-foobar' in home.text
    assert 'category-misc' not in home.text
    assert '<a class="" href="foobar/test/">test</a>' in home.text


def test_home_two_categories(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'foobar'
    formdef2.fields = []
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text  # 1st formdef
    assert 'category-misc' in resp.text  # 2nd formdef, fake category
    assert '<a class="" href="foobar/test/">test</a>' in resp.text

    cat2 = Category(name='barfoo')
    cat2.store()
    formdef2.category_id = cat2.id
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text  # 1st formdef
    assert 'category-barfoo' in resp.text  # 2nd formdef
    assert 'category-misc' not in resp.text  # no more "misc" category


def test_home_keywords(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.keywords = 'hello, world'
    formdef.store()
    home = get_app(pub).get('/')
    assert home.html.find('div', {'data-keywords': 'hello world'}) or home.html.find(
        'div', {'data-keywords': 'world hello'}
    )
    assert home.html.find('li', {'data-keywords': 'hello world'}) or home.html.find(
        'li', {'data-keywords': 'world hello'}
    )


def test_home_formdef_description(pub):
    formdef = create_formdef()
    formdef.description = 'HELLO WORLD'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'HELLO WORLD' in home.text
    assert '<a class="" href="test/">test</a>' in home.text


def test_home_disabled(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' not in home.text

    # check access is denied
    get_app(pub).get('/test/', status=403)


def test_home_disabled_with_redirect(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.disabled_redirection = 'http://example.org'
    formdef.store()
    resp = get_app(pub).get('/')
    assert '<a class="redirection" href="test/">test</a>' in resp.text
    resp = resp.click('test')
    assert resp.location == 'http://example.org'


def test_home_inaccessible(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.store()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://example.net/login/?next=http%3A%2F%2Fexample.net%2F'


def test_home_always_advertise(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.always_advertise = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' in home.text
    assert '<a href="test/">test</a><span> (authentication required)</span>' in home.text


def test_home_redirect(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/'


def test_home_redirect_var(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/[site_lang]/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/en/'


def test_category_page(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    resp = get_app(pub).get('/foobar/')
    assert '<h2>foobar</h2>' in resp.text
    assert '<a class="" href="test/">test</a>' in resp.text


def test_category_page_redirect(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/'


def test_category_page_redirect_var(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/[site_lang]/[category_slug]/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/en/foobar/'


def test_form_access(pub):
    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    # check a formdef protected by a role cannot be accessed
    formdef.roles = [role.id]
    formdef.store()
    # an unlogged user will ge ta redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # while a logged-in user will get a 403
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=403)

    # unless the user has the right role
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check admin has access, even without specific roles
    user = create_user(pub)
    user.roles = []
    user.is_admin = True
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check special "logged users" role
    formdef.roles = [logged_users_role().id]
    formdef.store()
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)
    resp = get_app(pub).get('/test/', status=302)  # redirect to login

    # check "receiver" can also access the formdef
    formdef = create_formdef()
    formdef.roles = [-2]
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)


def test_form_access_auth_context(pub):
    create_user(pub)

    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'auth-contexts', 'fedict')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    formdef.required_authentication_contexts = ['fedict']
    formdef.roles = [logged_users_role().id]
    formdef.store()

    # an unlogged user will get a redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # a user logged in with a simple username/password tuple will get a page
    # to relogin with a stronger auth
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' in resp.text

    for session in pub.session_manager.values():
        session.saml_authn_context = 'urn:oasis:names:tc:SAML:2.0:ac:classes:SmartcardPKI'
        session.store()
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' not in resp.text
    assert resp.form


def test_form_cancelurl(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()

    # path
    resp = get_app(pub).get('/test/?cancelurl=/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/plop/'

    # full URL
    resp = get_app(pub).get('/test/?cancelurl=http://example.net/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/plop/'

    # remote site
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.add_section('api-secrets')
    pub.site_options.set('api-secrets', 'example.org', 'xyz')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/?cancelurl=http://example.org/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.org/plop/'

    pub.site_options.remove_section('api-secrets')
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.set('options', 'relatable-hosts', 'example.com')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    get_app(pub).get('/test/?cancelurl=http://example.org/plop/', status=400)

    pub.site_options.set('options', 'relatable-hosts', 'example.com, example.org')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/?cancelurl=http://example.org/plop/')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.org/plop/'


def test_form_submit(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert 'None' not in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text


def test_form_submit_no_confirmation(pub):
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1


def test_form_string_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert next_page.pyquery('div.error').text() == 'required field'
    next_page.forms[0]['f0'] = 'foobar'
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': 'foobar'}


def test_form_items_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='List of items',
            type='items',
            required=True,
            varname='foo',
            items=['Foo', 'Bar', 'Three', 'Four', 'Five', 'Six'],
        )
    ]
    formdef.store()
    formdef.data_class().wipe()

    page = get_app(pub).get('/test/')
    assert 'List of items' in page.text
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert next_page.pyquery('div.error').text() == 'required field'
    next_page.forms[0]['f0$element0'].checked = True
    next_page.forms[0]['f0$element1'].checked = True
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar']
    assert data.data['0_display'] == 'Foo, Bar'

    formdef.fields[0].min_choices = 2
    formdef.fields[0].max_choices = 5
    formdef.store()

    page = get_app(pub).get('/test/')
    page.forms[0]['f0$element0'].checked = True
    page = page.forms[0].submit('submit')
    assert page.pyquery('div.error').text() == 'You must select at least 2 answers.'
    page.forms[0]['f0$element1'].checked = True
    page.forms[0]['f0$element2'].checked = True
    page.forms[0]['f0$element3'].checked = True
    page.forms[0]['f0$element4'].checked = True
    page.forms[0]['f0$element5'].checked = True
    page = page.forms[0].submit('submit')
    assert page.pyquery('div.error').text() == 'You must select at most 5 answers.'
    page.forms[0]['f0$element5'].checked = False
    page = next_page.forms[0].submit('submit').follow()
    assert 'The form has been recorded' in page.text


def test_form_items_autocomplete(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='List of items',
            type='items',
            required=True,
            varname='foo',
            display_mode='autocomplete',
            items=['Foo', 'Bar', 'Three', 'Four', 'Five', 'Six'],
        )
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert 'select2.min.js' in resp.text
    resp = resp.forms[0].submit('submit')  # but the field is required
    assert resp.pyquery('div.error').text() == 'required field'
    resp.forms[0]['f0[]'].select_multiple(['Foo', 'Bar'])
    resp = resp.forms[0].submit('submit')
    assert resp.pyquery('[name="f0[]"] option[selected]').text() == 'Foo Bar'
    assert resp.pyquery('#form_f0[readonly]').val() == 'Foo, Bar'
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar']
    assert data.data['0_display'] == 'Foo, Bar'

    formdef.data_class().wipe()

    formdef.fields[0].min_choices = 3
    formdef.fields[0].max_choices = 4
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0[]'].select_multiple(['Foo', 'Bar'])
    resp = resp.forms[0].submit('submit')
    assert resp.pyquery('div.error').text() == 'You must select at least 3 choices.'
    assert resp.forms[0]['f0[]'].value == ['Foo', 'Bar']
    resp.forms[0]['f0[]'].select_multiple(['Foo', 'Bar', 'Three', 'Four', 'Five'])
    resp = resp.forms[0].submit('submit')
    assert resp.pyquery('div.error').text() == 'You must select at most 4 choices.'
    assert resp.forms[0]['f0[]'].value == ['Foo', 'Bar', 'Three', 'Four', 'Five']
    resp.forms[0]['f0[]'].select_multiple(['Foo', 'Bar', 'Three'])
    resp = resp.forms[0].submit('submit')  # -> validation
    resp = resp.forms[0].submit('submit').follow()

    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar', 'Three']

    # check empty options do not crash the widget
    formdef.fields[0].items = []
    formdef.store()
    resp = get_app(pub).get('/test/')


def test_form_items_autocomplete_with_multiple_pages_no_confirmation(pub):
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemsField(
            id='1',
            label='List of items',
            type='items',
            required=True,
            varname='foo',
            display_mode='autocomplete',
            items=['Foo', 'Bar', 'Three', 'Four', 'Five', 'Six'],
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1[]'].select_multiple(['Foo', 'Bar'])
    resp = resp.forms[0].submit('submit')  # -> 2nd page
    resp = resp.forms[0].submit('submit')  # -> submit

    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['1'] == ['Foo', 'Bar']
    assert data.data['1_display'] == 'Foo, Bar'


def test_form_string_with_invalid_xml_chars(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0'] = 'hello\x0b\x0cworld'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    data = formdef.data_class().select()[0]
    assert data.data == {'0': 'helloworld'}


def test_form_submit_handling_role_info(pub):
    role = pub.role_class(name='xxx')
    role.details = 'Managing service'
    role.store()
    formdef = create_formdef()
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp
    assert 'Your case is handled by' in resp
    assert 'Managing service' in resp

    formdata = formdef.data_class().select()[0]
    formdata.jump_status('rejected')
    formdata.store()
    resp = resp.test_app.get(resp.request.url)
    assert 'Your case has been handled by' in resp


def assert_current_page(resp, page_label):
    for li_tag in resp.html.findAll('li'):
        if 'current' in li_tag.attrs['class']:
            assert li_tag.find_all('span')[-1].text == page_label


def test_form_multi_page(pub):
    for initial_condition in (None, 'True'):
        formdef = create_formdef()
        formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2'),
        ]
        formdef.fields[0].condition = {'type': 'python', 'value': initial_condition}
        formdef.store()
        page = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        page.forms[0]['f1'] = 'foo'
        assert page.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
        next_page = page.forms[0].submit('submit')
        assert_current_page(next_page, '2nd page')
        assert next_page.forms[0]['previous']
        next_page.forms[0]['f3'] = 'bar'
        next_page = next_page.forms[0].submit('submit')
        assert_current_page(next_page, 'Validating')
        assert 'Check values then click submit.' in next_page.text
        next_page = next_page.forms[0].submit('submit')
        assert next_page.status_int == 302
        next_page = next_page.follow()
        assert 'The form has been recorded' in next_page.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        data = formdef.data_class().get(data_id)
        assert data.data == {'1': 'foo', '3': 'bar'}


def test_form_multi_page_title_and_subtitle_as_template(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.TitleField(id='4', label='<i>title of second page {{ form_var_foo }}</i>', type='title'),
        fields.SubtitleField(
            id='5', label='<i>subtitle of second page {{ form_var_foo }}</i>', type='subtitle'
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '35 < 42'
    resp = resp.form.submit('submit')
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    expected_label = '&lt;i&gt;title of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="title "><h3>%s</h3></div>' % expected_label in resp.text
    expected_label = '&lt;i&gt;subtitle of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="subtitle "><h4>%s</h4></div>' % expected_label in resp.text


def test_form_multi_page_condition(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']


def test_form_multi_page_condition_select(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar']
        ),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'var_foo == "Foo"'}
        ),
        fields.PageField(
            id='3', label='3rd page', type='page', condition={'type': 'python', 'value': 'var_foo == "Bar"'}
        ),
        fields.StringField(id='4', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert '2nd page' not in resp.text
    assert '3rd page' not in resp.text
    assert resp.forms[0]['f1'].value == 'Foo'  # preset
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert '3rd page' not in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' not in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')


def test_form_multi_page_condition_select_new_varname(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar']
        ),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "Foo"'},
        ),
        fields.PageField(
            id='3',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "Bar"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert '2nd page' not in resp.text
    assert '3rd page' not in resp.text
    resp.forms[0]['f1'] = 'Foo'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert '3rd page' not in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' not in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')


def test_form_multi_page_condition_checkbox(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'].checked = True
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']
    resp.forms[0]['f1'].checked = False
    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_json_check(pub):
    # make sure the json export has no value for fields from hidden pages
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2', varname='st2'),
        fields.PageField(
            id='4',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "True"'},
        ),
        fields.StringField(id='5', label='string 3', varname='st3'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit')  # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit')  # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit()

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None


def test_form_multi_page_condition_no_confirmation_json_check(pub):
    # same as above but without the confirmation page.
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "False"'},
        ),
        fields.StringField(id='3', label='string 2', varname='st2'),
        fields.PageField(
            id='4',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'var_checkbox == "True"'},
        ),
        fields.StringField(id='5', label='string 3', varname='st3'),
        fields.PageField(id='6', label='4th page', type='page'),
        fields.CommentField(id='7', label='Check values then click submit.', type='comment'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit')  # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit')  # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit('submit')

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None


def test_form_multi_page_condition_data_source(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.BoolField(id='1', label='checkbox', varname='checkbox'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    # add the named data source, empty
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_data_source_with_form_variable(pub):
    # this tries to recreate #8272 which is about a json datasource being
    # used in a page condition and taking a value from the given page to
    # filter its content.  It is emulated here with a Python datasource
    # being empty if a field was not set.
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='xxx', required=False),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    # add the named data source, related to a field on the first page
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': 'form_var_xxx and [form_var_xxx] or []'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'HELLO'
    resp = resp.forms[0].submit('submit')  # should go to second page
    assert 'f3' in resp.forms[0].fields


def test_form_multi_page_condition_on_first_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    # should be on second page already
    with pytest.raises(AssertionError):
        resp.form.get('previous')
    resp.form['f3'] = 'foo'
    assert_current_page(resp, '2nd page')
    resp = resp.form.submit('submit')  # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')  # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('previous')  # -> 2nd page
    assert_current_page(resp, '2nd page')
    assert resp.form['f3']
    with pytest.raises(AssertionError):
        resp.form.get('previous')


def test_form_multi_page_condition_on_first_and_next(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
        fields.StringField(id='1', label='string', varname='val1'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'vars().get("form_var_val1") == "foo"'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert resp.form['f3']
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo', '3': 'bar'}

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'xxx'
    resp = resp.form.submit('submit')
    with pytest.raises(AssertionError):
        assert resp.form['f3']
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data.get('1') == 'xxx'
    assert data.data.get('3') is None


def test_form_multi_page_condition_no_visible_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0', label='1st page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'error-page' in resp
    assert 'This form has no visible page.' in resp


def test_form_multi_page_many_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(
            id='0', label='2nd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
    ]

    formdef.store()
    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        get_app(pub).get('/test/')
        call_count = invalidate_cache.call_count

    for i in range(30):
        formdef.fields.append(
            fields.PageField(
                id=str(i + 2),
                label='page %s' % (i + 2),
                type='page',
                condition={'type': 'python', 'value': 'True'},
            )
        )
    formdef.store()

    # check the cache doesn't get invalidated for every page
    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        get_app(pub).get('/test/')
        assert invalidate_cache.call_count <= call_count


def test_form_multi_page_condition_stored_values(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_foo == "toto"'},
        ),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert 'bar' in resp.text
    resp = resp.form.submit('previous')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert 'bar' not in resp.text
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None

    # same without validation page
    formdef.confirmation = False
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None


def test_form_multi_page_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(
            id='2', label='2nd page', type='page', condition={'type': 'python', 'value': 'False'}
        ),
        fields.StringField(id='3', label='string 2'),
        fields.PageField(id='4', label='3rd page', type='page'),
        fields.StringField(id='5', label='string 3', varname='bar'),
    ]

    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foo'
    assert formdef.data_class().select()[0].data['5'] == 'bar'

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'False'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'global-errors' in resp.text
    assert 'You shall not pass.' in resp.text

    formdef.fields[4].post_conditions = [
        {
            'condition': {'type': 'python', 'value': 'form_var_foo == "foo"'},
            'error_message': 'You shall not pass.',
        },
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

    # check a post-condition raising an exception, they should always fail.
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': '1/0'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    # check a post-condition referring to a field on the same page
    formdef.fields[4].post_conditions = [
        {
            'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'},
            'error_message': 'You shall not pass.',
        },
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'foo'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text


def test_form_multi_page_conditions_and_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(
            id='0',
            label='1st page',
            type='page',
            post_conditions=[
                {
                    'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'},
                    'error_message': 'You shall not pass.',
                }
            ],
        ),
        fields.StringField(id='1', label='string', varname='bar'),
        fields.PageField(id='3', label='2nd page', type='page'),
    ]

    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text

    # add a conditional page, this will cause pages to be evaluated first
    # (and would trigger #25197)
    formdef.fields.append(
        fields.PageField(id='4', label='3rd page', type='page', condition={'type': 'python', 'value': 'True'})
    )
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text


def test_form_multi_page_page_name_as_title(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2  # in steps and in main body

    # add a comment that will not be displayed and should therefore not be
    # considered.
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.CommentField(id='5', label='bla bla bla', type='comment'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2  # in steps and in main body


def test_form_submit_with_user(pub, emails):
    create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text
    # check the user received a copy by email
    assert emails.get('New form (test)')
    assert emails.get('New form (test)')['email_rcpt'] == ['foo@localhost']


def test_form_submit_with_just_disabled_user(pub, emails):
    user = create_user(pub)
    formdef = create_formdef()
    app = login(get_app(pub), username='foo', password='foo')
    formdef.data_class().wipe()
    resp = app.get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    user.is_active = False
    user.store()
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp


def test_form_titles(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.SubtitleField(id='5', label='subtitle of 1st page', type='subtitle'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.TitleField(id='6', label='title of second page', type='title'),
        fields.StringField(id='3', label='string 2', required=False),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert '<h3 data-field-id="0">1st page/h3>' not in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp.form['f3'] = 'foo'
    resp = resp.form.submit('submit')  # -> validation page
    assert '<h3>1st page</h3>' in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<div class="title "><h3>1st page</h3></div>' not in resp.text
    assert '<div class="subtitle "><h4>subtitle of 1st page</h4></div>' in resp.text
    assert '<div class="title "><h3>title of second page</h3></div>' in resp.text


def test_form_summary_empty_pages(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='toto'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_var_toto == "foo"'},
        ),
        fields.TitleField(id='6', label='title in second page', type='title'),
        fields.StringField(id='3', label='string'),
        fields.PageField(id='4', label='3rd page', type='page'),
        fields.StringField(id='5', label='string'),
        fields.PageField(id='7', label='4th page', type='page'),
        fields.CommentField(id='8', label='Bla bla bla', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')
    formdata_id = resp.location.split('/')[-2]
    resp = resp.follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('previous')  # -> 1st page
    resp.form['f1'] = 'baz'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' not in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    # change condition to have second page never displayed
    formdef.fields[2].condition['value'] = False
    formdef.store()
    formdata = formdef.data_class().get(formdata_id)
    resp = app.get(formdata.get_url())
    # it was filled by user, it should still appear (conditions should not be
    # replayed)
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text


def test_form_display_locations(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string1', display_locations=[]),
        fields.StringField(id='2', label='string2', display_locations=['validation']),
        fields.StringField(id='3', label='string3', display_locations=['summary']),
        fields.CommentField(
            id='4', label='Bla bla bla', type='comment', display_locations=['validation', 'summary']
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'plop1'
    resp.form['f2'] = 'plop2'
    resp.form['f3'] = 'plop3'
    resp = resp.form.submit('submit')  # -> validation
    pq = resp.pyquery.remove_namespaces()
    assert pq('div[style="display: none;"] [name=f1]')
    assert not pq('div[style="display: none;"] [name=f2]')
    assert pq('div[style="display: none;"] [name=f3]')
    assert 'Bla bla bla' in resp.text

    resp = resp.form.submit('submit').follow()  # -> submit
    assert formdef.data_class().select()[0].data['1'] == 'plop1'
    assert formdef.data_class().select()[0].data['2'] == 'plop2'
    assert formdef.data_class().select()[0].data['3'] == 'plop3'
    assert 'plop1' not in resp.text
    assert 'plop2' not in resp.text
    assert 'plop3' in resp.text
    assert 'Bla bla bla' in resp.text


def test_multipage_form_display_locations(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string1', display_locations=[]),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(id='3', label='Bla bla bla', type='comment', display_locations=['validation']),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'plop1'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit')  # -> validation

    pq = resp.pyquery.remove_namespaces()
    assert '<h3>1st page</h3>' not in resp.text  # page 1 title not displayed
    assert pq('div[style="display: none;"] [name=f1]')  # but page 1 field included, hidden
    assert '<h3>2nd page</h3>' in resp.text  # page 2 title
    assert 'Bla bla bla' in resp.text  # and page 2 comment field


def test_form_visit_existing(pub):
    user = create_user(pub)
    formdef = create_formdef()
    login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.store()

    formdata_user = formdef.data_class()()
    formdata_user.user_id = user.id
    formdata_user.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = get_app(pub).get('/test/%s/' % formdata_user.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % formdata_user.id)
    assert 'The form has been recorded on' in resp


def test_form_no_tracking_code(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert '<h3>Tracking code</h3>' not in resp.text


def test_form_no_tracking_code_variable(pub):
    create_user(pub)
    FormDef.wipe()
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(id='3', label='<p>xxx{{form_tracking_code|default:""}}yyy</p>', type='comment'),
    ]
    formdef.store()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp.form['f1'] = 'foo'
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {
        'result': 'error',
        'reason': 'missing data',
    }
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')
    assert 'xxxyyy' in resp.text
    resp = resp.form.submit('submit')
    assert_current_page(resp, 'Validating')
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo'}
    assert data.tracking_code is None


def get_displayed_tracking_code(resp):
    tracking_code = None
    for a_tag in resp.html.findAll('a'):
        if 'code/' in a_tag['href']:
            tracking_code = a_tag.text
            break
    return tracking_code


def test_form_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()

    # check anonymous user can't get to it from the URL
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('http://example.net/test/%s/' % formdata_id)
    assert resp.location.startswith('http://example.net/login')

    # or logged users that didn't enter the code:
    create_user(pub)
    login(get_app(pub), username='foo', password='foo').get(
        'http://example.net/test/%s/' % formdata_id, status=403
    )

    # check we can also get to it as a logged user
    pub.session_manager.session_class.wipe()
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()

    # go back as anonymous
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1  # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].get_plain_text_comment() == 'hello world'

    # check we can also use it with lowercase letters.
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()


def test_form_tracking_code_verification(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='0', label='string1', required=False),
        fields.StringField(id='1', label='string2', required=False),
        fields.DateField(id='2', label='date', required=False),
    ]
    formdef.enable_tracking_codes = True
    formdef.tracking_code_verify_fields = ['0', '1', '2']
    formdef.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar1'
    resp.forms[0]['f1'] = 'foobar2'
    resp.forms[0]['f2'] = '2022-01-01'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar1'
    assert formdef.data_class().select()[0].data['1'] == 'foobar2'
    assert formdef.data_class().select()[0].data['2'].tm_year == 2022
    formdata = formdef.data_class().select()[0]
    formdata_id = formdata.id

    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert 'Access rights verification' in resp
    resp.forms[0]['f0'] = 'foobar1'
    resp.forms[0]['f1'] = 'foobar2'
    resp.forms[0]['f2'] = '2022-01-01'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()

    # check anonymous user can't get to it from the URL
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('http://example.net/test/%s/' % formdata_id)
    assert resp.location.startswith('http://example.net/login')
    # or logged users that didn't enter the code:
    create_user(pub)
    login(get_app(pub), username='foo', password='foo').get(
        'http://example.net/test/%s/' % formdata_id, status=403
    )

    # verification failure
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    resp.forms[0]['f0'] = 'foobar1'  # ok
    resp.forms[0]['f1'] = 'barfoo2'  # ko
    resp.forms[0]['f2'] = '2022-01-01'  # ok
    resp = resp.forms[0].submit('submit')
    assert 'Access denied: this content does not match the form' in resp
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    resp.forms[0]['f0'] = 'foobar1'  # ok
    resp.forms[0]['f1'] = 'foobar2'  # ok
    resp.forms[0]['f2'] = '2022-01-02'  # ko
    resp = resp.forms[0].submit('submit')
    assert 'Access denied: this content does not match the form' in resp

    # draft with an empty field: do not verify it
    formdata.data['0'] = None
    formdata.store()
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    assert 'f0' not in resp.forms[0].fields
    assert 'f1' in resp.forms[0].fields
    resp.forms[0]['f1'] = 'foobar2'
    resp.forms[0]['f2'] = '2022-01-01'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()

    # empty draft: no verification
    formdata.data['1'] = None
    formdata.data['2'] = None
    formdata.store()
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')

    # not a draft: all validation fields are required
    formdata.status = 'wf-new'
    formdata.data['0'] = 'foobar1'
    formdata.store()
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    assert 'f0' in resp.forms[0].fields
    assert 'f1' in resp.forms[0].fields
    assert 'f2' in resp.forms[0].fields
    resp.forms[0]['f0'] = 'foobar1'
    resp.forms[0]['f1'] = ''
    resp.forms[0]['f2'] = ''
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'foobar1' in resp.text
    assert 'form_comment' in resp.text  # user is treated as submitter

    # verification failure
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    resp.forms[0]['f0'] = 'foobar1'  # ok
    resp.forms[0]['f1'] = 'not empty'  # ko
    resp.forms[0]['f2'] = ''  # ok
    resp = resp.forms[0].submit('submit')
    assert 'Access denied: this content does not match the form.' in resp
    resp = get_app(pub).get('http://example.net/code/%s/load' % tracking_code)
    assert 'Access rights verification' in resp
    resp.forms[0]['f0'] = 'foobar1'  # ok
    resp.forms[0]['f1'] = ''  # ok
    resp.forms[0]['f2'] = '2022-02-02'  # ko (not empty)
    resp = resp.forms[0].submit('submit')
    assert 'Access denied: this content does not match the form.' in resp


def test_form_tracking_code_rate_limit(pub, freezer):
    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'rate-limit', '2/2s')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    # twice
    freezer.move_to('2018-12-01T00:00:00')
    get_app(pub).get('/code/ABC/load', status=404)
    get_app(pub).get('/code/ABC/load', status=404)
    # and out
    get_app(pub).get('/code/ABC/load', status=403)
    get_app(pub).get('/code/ABC/load', status=403)
    # wait two second
    freezer.move_to('2018-12-01T00:00:02')
    # and ok again
    get_app(pub).get('/code/ABC/load', status=404)


def test_form_tracking_code_as_user(pub, nocache):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1  # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert str(formdef.data_class().select()[0].user_id) == str(user.id)
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].get_plain_text_comment() == 'hello world'

    # and check we can also get back to it as anonymous
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert 'form_comment' in resp.text  # makes sure user is treated as submitter

    # and check a bot is not allowed to get it
    app = get_app(pub)
    resp = app.get('/code/%s/load' % tracking_code, headers={'User-agent': 'Googlebot'}, status=403)


def test_form_tracking_code_prefill(pub, nocache):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.enable_tracking_codes = True
    formdef.store()

    # first time
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'You already started to fill this form.' not in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')

    # second time, invitation to load an existing draft
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'You already started to fill this form.' in resp.text


def test_form_empty_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # check we get a 404 if we use the tracking code before it gets any data
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow(status=404)


def test_form_tracking_code_email(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'), fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp = resp.forms[0].submit()
    assert emails.get('Tracking Code reminder')
    assert tracking_code in emails.get('Tracking Code reminder')['payload']
    assert resp.location == 'http://example.net/test/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'barfoo'


def test_form_tracking_code_email_and_verification(pub, emails, nocache):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='0', label='string1', required=False),
        fields.StringField(id='1', label='string2', required=False),
        fields.DateField(id='2', label='date', required=False),
    ]
    formdef.enable_tracking_codes = True
    formdef.tracking_code_verify_fields = ['0', '1', '2']
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = app.get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp = resp.forms[0].submit()
    assert emails.get('Tracking Code reminder')
    assert tracking_code in emails.get('Tracking Code reminder')['payload']
    assert resp.location == 'http://example.net/test/code/%s/load' % tracking_code

    # returns to the form, without verification: formdata is mine
    resp = resp.follow()
    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'barfoo'


def test_form_tracking_code_email_antibot(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'), fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp.forms[0]['validation'].checked = True  # stupit bot will do that
    resp = resp.forms[0].submit()
    assert not emails.count()


def test_form_tracking_code_email_too_early(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'), fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    # do nothing more: formdata does not even exists
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert 'Form is empty: tracking code is not yet available.' in resp.text


def test_form_tracking_code_remove_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '1'
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # visit page, check there's no remove draft button
    resp = app.get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' not in resp.text

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    resp = resp.forms[0].submit()  # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0


def test_form_tracking_code_remove_empty_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('previous')
    resp_autosave = app.post('/test/autosave', params=resp.form.submit_fields())
    assert resp_autosave.json == {'result': 'success'}
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '0'

    # make draft empty
    formdata = formdef.data_class().select()[0]
    formdata.data = {}
    formdata.store()
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    assert resp.forms[1]['f0'].value == ''

    resp = resp.forms[0].submit()  # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0


def test_form_discard_draft(pub, nocache):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    formdef.data_class().wipe()

    # anonymous user, no tracking code (-> no draft)
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == []
    assert 'Cancel' in resp.text
    assert 'Discard' not in resp.text
    resp = resp.form.submit('cancel')

    # anonymous user, tracking code (-> draft)
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, no tracking code
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, tracking code
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # anonymous user, tracking code, recalled
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    tracking_code = get_displayed_tracking_code(resp)

    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()
    assert resp.forms[1]['f0'].value == 'foobar'
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']

    # logged-in user, no tracking code, recalled
    formdef.data_class().wipe()
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.click('Continue with draft').follow()
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']


def test_form_invalid_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    # create a secondary formdef, to always have the tracking code form
    # displayed on homepage
    formdef2 = FormDef()
    formdef2.name = 'test2'
    formdef2.fields = []
    formdef2.enable_tracking_codes = True
    formdef2.store()

    resp = get_app(pub).get('/')

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.store()

    # check we can go back to it
    formdef.data_class().wipe()

    code = pub.tracking_code_class()
    code.formdata = formdata  # this will save it again
    code.store()

    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s/' % formdata.id
    resp = resp.follow()

    # check we get a not found error message on non-existent code
    fake_code = TrackingCode().get_new_id()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = fake_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % fake_code
    resp = resp.follow(status=404)

    # check we also get an error if tracking code access is disabled after the
    # fact
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow(status=404)


def test_form_tracking_code_as_variable(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(type='comment', id='3', label='!{{ form_tracking_code }}!'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foobar'
    resp = resp.form.submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    assert '!%s!' % tracking_code in resp.text


def test_form_draft_with_file(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.pyquery('.filename').text() == 'test.txt'
    # check file is downloadable
    r2 = resp.click('test.txt')
    assert r2.content_type == 'text/plain'
    assert r2.text == 'foobar'

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit')  # -> confirmation page
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert resp.click('test.txt').follow().text == 'foobar'


def test_form_draft_with_file_direct_validation(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert 'test2.txt' in resp.text

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert resp.click('test2.txt').follow().text == 'foobar2'


def test_form_draft_with_date(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='date', type='date')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0'] = '2012-02-12'
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert '2012-02-12' in resp.text

    # check submitted form keeps the date
    resp = resp.forms[1].submit('submit')  # -> done
    resp = resp.follow()

    assert '2012-02-12' in resp.text


@pytest.mark.parametrize('field_type', ['string', 'item'])
@pytest.mark.parametrize('logged_in', ['anonymous', 'logged-in'])
def test_form_draft_from_prefill(pub, field_type, logged_in):
    create_user(pub)

    formdef = create_formdef()
    if field_type == 'string':
        formdef.fields = [fields.StringField(id='0', label='string')]
    else:
        formdef.fields = [fields.ItemField(id='0', label='item', type='item', items=['foo', 'bar', 'hello'])]
    formdef.enable_tracking_codes = True
    formdef.store()
    formdef.data_class().wipe()

    app = get_app(pub)
    if logged_in == 'logged-in':
        login(app, username='foo', password='foo')

    # no draft
    app.get('/test/')
    assert formdef.data_class().count() == 0
    formdef.data_class().wipe()

    # draft created if there's been some prefilled fields
    formdef.fields[0].prefill = {'type': 'string', 'value': '{{request.GET.test|default:""}}'}
    formdef.store()
    app.get('/test/?test=hello')
    assert formdef.data_class().count() == 1
    formdef.data_class().wipe()

    # unless the call was made from an application
    app.get('/test/?test=hello', headers={'User-agent': 'python-requests/0'})
    assert formdef.data_class().count() == 0

    # or a bot
    app.get('/test/?test=hello', headers={'User-agent': 'Googlebot'})
    assert formdef.data_class().count() == 0

    # check there's no leftover draft after submission
    for with_tracking_code in (False, True):
        formdef.enable_tracking_codes = with_tracking_code
        formdef.store()
        formdef.data_class().wipe()
        resp = app.get('/test/?test=hello')
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().count() == 1


@pytest.mark.parametrize('tracking_code', [True, False])
def test_form_direct_draft_access(pub, tracking_code):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = tracking_code
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.status = 'draft'
    formdata.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    formdata.user_id = user.id
    formdata.store()
    resp = get_app(pub).get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/test/?mt=')

    formdata.user_id = 1000
    formdata.store()
    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s/' % formdata.id, status=403)


def form_password_field_submit(app, password):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [fields.PasswordField(id='0', label='password', formats=['sha1', 'md5', 'cleartext'])]
    formdef.store()
    page = app.get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')  # but the field is required
    assert [x.text for x in next_page.pyquery('div.error p')] == ['required field'] * 2
    next_page.forms[0]['f0$pwd1'] = password
    next_page.forms[0]['f0$pwd2'] = password
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {
        '0': {
            'sha1': force_str(hashlib.sha1(force_bytes(password)).hexdigest()),
            'md5': force_str(hashlib.md5(force_bytes(password)).hexdigest()),
            'cleartext': force_str(password),
        }
    }


def test_form_password_field_submit(pub):
    create_user(pub)
    form_password_field_submit(get_app(pub), 'foobar')
    form_password_field_submit(get_app(pub), force_str('•	83003706'))
    form_password_field_submit(login(get_app(pub), username='foo', password='foo'), 'foobar\u00eb')


def test_form_multi_page_formdef_count_condition(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(
            id='2',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'form_objects.count > 0'},
        ),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text

    # add a formdata this will make the second page appear.
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')  # should NOT go straight to validation
    assert 'Check values then click submit.' not in resp.text


def test_form_multi_page_post_edit(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter', '_receiver']
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    next_page.forms[0]['f3'] = 'barXYZ'
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text

    data_id = formdef.data_class().select()[0].id

    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    # check there's no new "phantom" history entry
    assert len(formdef.data_class().get(data_id).evolution) == 1
    assert resp.forms[0]['f1'].value == 'foo'
    resp.forms[0]['f1'] = 'foo2'

    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1'].value == 'foo2'
    resp = resp.forms[0].submit('submit')
    assert 'Save Changes' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo2' in resp.text  # modified value is there
    assert 'barXYZ' in resp.text  # unchanged value is still there
    assert len(formdef.data_class().get(data_id).evolution) == 2  # new history entry
    assert formdef.data_class().get(data_id).evolution[-1].who == '_submitter'
    assert formdef.data_class().get(data_id).evolution[-1].status is None

    # modify workflow to jump to another status after the edition
    st2 = workflow.add_status('Status2', 'st2')
    editable.status = st2.id
    workflow.store()

    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo2'
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    assert (
        formdef.data_class().get(data_id).data['1'] == 'foo2'
    )  # check foo3 has not been overwritten in database
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo3' in resp.text  # modified value is there
    assert 'barXYZ' in resp.text  # unchanged value is still there
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st2.id
    assert len(formdef.data_class().get(data_id).evolution) == 3  # single new history entry
    assert formdef.data_class().get(data_id).evolution[-1].who == '_submitter'
    assert formdef.data_class().get(data_id).evolution[-1].status == 'wf-%s' % st2.id

    # jump to a nonexistent status == do not jump, but add a LoggedError
    pub.loggederror_class.wipe()
    assert pub.loggederror_class.count() == 0
    editable.status = 'deleted_status_id'
    workflow.store()
    # go back to st1
    formdata = formdef.data_class().get(data_id)
    formdata.status = 'wf-%s' % st1.id
    formdata.store()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    assert pub.loggederror_class.count() == 1
    logged_error = pub.loggederror_class.select()[0]
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.formdef_id == formdef.id
    assert logged_error.workflow_id == workflow.id
    assert logged_error.status_id == st1.id
    assert logged_error.status_item_id == editable.id
    assert logged_error.occurences_count == 1

    # do it again: increment logged_error.occurences_count
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    assert pub.loggederror_class.count() == 1
    logged_error = pub.loggederror_class.select()[0]
    assert logged_error.occurences_count == 2


def test_form_edit_autocomplete_list(pub):
    create_user(pub)

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter', '_receiver']
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = get_app(pub)
    login(app, username='foo', password='foo')

    with responses.RequestsMock() as rsps:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        rsps.get('http://remote.example.net/json', json=data)

        resp = app.get('/test/')
        assert 'data-select2-url=' in resp.text
        # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
        resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
        resp.form['f0'].force_value('1')
        resp.form.fields['f0_display'].force_value('hello')
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

        resp = resp.follow()
        url = resp.request.url
        resp = resp.form.submit('button_editable')
        assert 'wfedit' in resp.location
        resp = resp.follow()
        assert 'data-value="1"' in resp
        assert 'data-initial-display-value="hello"' in resp

        # relogin
        app = get_app(pub)
        login(app, username='foo', password='foo')
        resp = app.get(url)
        resp = resp.form.submit('button_editable')
        resp = resp.follow()
        assert 'data-value="1"' in resp
        assert 'data-initial-display-value="hello"' in resp


def test_form_edit_with_internal_id_condition(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.StringField(id='1', label='1st field', type='string'),
        fields.StringField(
            id='2',
            label='2nd field',
            type='string',
            condition={'type': 'django', 'value': 'form_internal_id'},
        ),
    ]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter', '_receiver']
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = get_app(pub)
    login(app, username='foo', password='foo')

    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'test'
    assert 'f2' not in resp.form.fields
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['1'] == 'test'

    resp = resp.follow()
    resp = resp.form.submit('button_editable')
    assert 'wfedit' in resp.location
    resp = resp.follow()
    assert 'f2' in resp.form.fields


def test_form_count_dispatching(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = st1.add_action('jump')
    jump.condition = {'type': 'python', 'value': 'form_objects.count_status_st2 < 1'}
    jump.status = 'st2'
    workflow.add_status('Status2', 'st2')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text  # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 0
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text  # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 1


def test_preview_form(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = []
    formdef.disabled = True
    formdef.store()

    # check the preview page is not accessible to regular users
    get_app(pub).get('/preview/test/', status=403)

    # check it's accessible to admins
    user.is_admin = True
    user.store()
    page = login(get_app(pub), username='foo', password='foo').get('/preview/test/')

    # check the form is marked as a preview (this disables autosave calls)
    assert page.pyquery('form[data-autosave=false]').length

    # check no formdata gets stored
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 200
    assert 'This was only a preview: form was not actually submitted.' in next_page.text
    assert len([x for x in formdef.data_class().select() if not x.is_draft()]) == 0

    # check no drafts are proposed for recall
    formdef.data_class().wipe()
    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/preview/test/')
    assert 'You already started to fill this form.' not in resp.text


def test_form_item_data_source_field_submit(pub):
    def submit_item_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0'] = '1'
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux')]),
    }
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un', 'more': 'foo'}, {'id': '2', 'text': 'deux', 'more': 'bar'}])
    assert submit_item_data_source_field(ds) == {
        '0': '1',
        '0_display': 'un',
        '0_structured': {'id': '1', 'text': 'un', 'more': 'foo'},
    }

    # numeric identifiers
    ds['value'] = repr([{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    # json source
    ds = {
        'type': 'json',
        'value': 'http://www.example.net/plop',
    }

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://www.example.net/plop',
            json={'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]},
        )
        assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    # numeric identifiers
    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://www.example.net/plop', json={'data': [{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}]}
        )
        assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}


@pytest.mark.parametrize('fail_after_count_page', range(2, 8))
@pytest.mark.parametrize('fail_after_count_validation', range(0, 2))
@responses.activate
def test_form_item_data_source_error(pub, monkeypatch, fail_after_count_page, fail_after_count_validation):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://www.example.net/plop'}
    data_source.id_parameter = 'id'
    data_source.store()

    normal_get_structured_value = NamedDataSource.get_structured_value

    class failing_get_structured_value:
        def __init__(self, fail_after_count):
            self.fail_after_count = fail_after_count
            self.count = 0

        def __call__(self, *args):
            import inspect

            for frame in inspect.stack():
                if frame.function in ['store_display_value', 'store_structured_value']:
                    count = self.count
                    self.count += 1
                    if count >= self.fail_after_count:
                        return None
            return normal_get_structured_value(*args)

        @property
        def method(self):
            def f(*args):
                return self(*args)

            return f

    responses.get(
        'http://www.example.net/plop', json={'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]}
    )

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(id='1', label='string', data_source={'type': 'foobar'}),
    ]

    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = '1'

    # fail in get_structured_value
    monkeypatch.setattr(
        NamedDataSource, 'get_structured_value', failing_get_structured_value(fail_after_count_page).method
    )
    resp = resp.forms[0].submit('submit')
    assert 'Technical error, please try again' in resp.text

    # fix transient failure
    monkeypatch.setattr(NamedDataSource, 'get_structured_value', normal_get_structured_value)
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

    # fail in get_structured_value
    monkeypatch.setattr(
        NamedDataSource,
        'get_structured_value',
        failing_get_structured_value(fail_after_count_validation).method,
    )
    resp = resp.forms[0].submit('submit')
    assert 'Technical error, please try again' in resp.text

    # fix transient failure
    monkeypatch.setattr(NamedDataSource, 'get_structured_value', normal_get_structured_value)
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    return formdef.data_class().get(data_id).data


@pytest.mark.parametrize('fail_after_count_page', range(2, 8))
@responses.activate
def test_form_item_data_source_error_no_confirmation(pub, monkeypatch, fail_after_count_page):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://www.example.net/plop'}
    data_source.id_parameter = 'id'
    data_source.store()

    normal_get_structured_value = NamedDataSource.get_structured_value

    class failing_get_structured_value:
        def __init__(self, fail_after_count):
            self.fail_after_count = fail_after_count
            self.count = 0

        def __call__(self, *args):
            import inspect

            for frame in inspect.stack():
                if frame.function in ['store_display_value', 'store_structured_value']:
                    count = self.count
                    self.count += 1
                    if count >= self.fail_after_count:
                        return None
            return normal_get_structured_value(*args)

        @property
        def method(self):
            def f(*args):
                return self(*args)

            return f

    responses.get(
        'http://www.example.net/plop', json={'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]}
    )

    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(id='1', label='string', data_source={'type': 'foobar'}),
    ]

    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = '1'

    # fail in get_structured_value
    monkeypatch.setattr(
        NamedDataSource, 'get_structured_value', failing_get_structured_value(fail_after_count_page).method
    )
    resp = resp.forms[0].submit('submit')
    assert 'Technical error, please try again' in resp.text

    # fix transient failure
    monkeypatch.setattr(NamedDataSource, 'get_structured_value', normal_get_structured_value)
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    return formdef.data_class().get(data_id).data


def test_form_items_data_source_field_submit(pub):
    def submit_items_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemsField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0$element1'].checked = True
        resp.forms[0]['f0$element3'].checked = True
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux'), ('3', 'trois')]),
    }
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [{'id': '1', 'text': 'un'}, {'id': '3', 'text': 'trois'}],
    }

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}, {'id': '3', 'text': 'trois'}])
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [{'id': '1', 'text': 'un'}, {'id': '3', 'text': 'trois'}],
    }

    ds['value'] = repr(
        [
            {'id': '1', 'text': 'un', 'more': 'foo'},
            {'id': '2', 'text': 'deux', 'more': 'bar'},
            {'id': '3', 'text': 'trois', 'more': 'baz'},
        ]
    )
    assert submit_items_data_source_field(ds) == {
        '0': ['1', '3'],
        '0_display': 'un, trois',
        '0_structured': [
            {'id': '1', 'more': 'foo', 'text': 'un'},
            {'id': '3', 'more': 'baz', 'text': 'trois'},
        ],
    }


def test_form_page_string_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text


def test_form_page_profile_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'


def test_form_page_profile_first_name_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields = [
        fields.StringField(
            id='_first_name', label='first name', type='string', extra_css_class='autocomplete-given-name'
        ),
        fields.StringField(
            id='_city', label='city', type='string', extra_css_class='autocomplete-address-level2'
        ),
        fields.StringField(id='_plop', label='plop', type='string', extra_css_class='xxx'),
    ]
    user_formdef.store()
    user.form_data = {'_first_name': 'plop', '_city': 'mytown'}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': '_first_name'}),
        fields.StringField(id='1', label='string', prefill={'type': 'user', 'value': '_city'}),
        fields.StringField(id='2', label='string', prefill={'type': 'user', 'value': '_plop'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''
    assert resp.forms[0]['f0'].attrs['autocomplete'] == 'given-name'  # html5
    assert resp.forms[0]['f1'].value == ''
    assert resp.forms[0]['f1'].attrs['autocomplete'] == 'address-level2'  # html5

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'plop'
    assert resp.forms[0]['f1'].value == 'mytown'


def test_form_page_formula_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': repr('HELLO WORLD')})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text


def test_form_page_template_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(
            id='0', label='string', prefill={'type': 'string', 'value': '{{session_user_display_name}}'}
        )
    ]
    formdef.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert resp.form['f0'].value == 'User Name'
    assert 'widget-prefilled' in resp.text

    # erroneous prefill
    formdef.fields = [
        fields.StringField(
            id='0',
            label='string',
            prefill={'type': 'string', 'value': '{{session_user_display_name|unknown}}'},
        )
    ]
    formdef.store()

    resp = app.get('/test/')
    assert resp.form['f0'].value == ''
    # still marked with a css class, in case of live changes.
    assert 'widget-prefilled' in resp.text


def test_form_page_session_var_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'session_var_foo'})
    ]
    formdef.store()

    # check it's empty if it doesn't exist
    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    # check it's not set if it's not whitelisted
    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check it works
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        fd.write(
            '''[options]
query_string_allowed_vars = foo,bar
'''
        )

    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello'

    # check it survives a login
    resp = get_app(pub).get('/?session_var_foo=hello2')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('Login')
    resp = resp.follow()
    resp.forms[0]['username'] = 'foo'
    resp.forms[0]['password'] = 'foo'
    resp = resp.forms[0].submit()
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello2'

    # check repeated options are ignored
    resp = get_app(pub).get('/?session_var_foo=hello&session_var_foo=hello2')
    assert urllib.parse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check extra query string parameters are not lost
    resp = get_app(pub).get('/?session_var_foo=hello&foo=bar')
    assert urllib.parse.urlparse(resp.location).path == '/'
    assert urllib.parse.urlparse(resp.location).query == 'foo=bar'

    os.unlink(os.path.join(pub.app_dir, 'site-options.cfg'))


def test_form_page_template_list_prefill(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='1',
            type='item',
            label='item',
            varname='item',
            required=True,
            items=['Foo', 'Bar'],
            prefill={'type': 'string', 'value': 'Foo'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == 'Foo'

    formdef.fields[0].prefill['value'] = 'Bar'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == 'Bar'

    formdef.fields[0].prefill['value'] = 'Baz'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'invalid value selected' in resp.text

    formdef.fields[0].prefill['value'] = '{{plop|default:""}}'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'invalid value selected' not in resp.text


def test_form_page_query_string_list_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='item',
            varname='item',
            required=False,
            data_source={'type': 'foobar'},
            prefill={'type': 'string', 'value': '{{request.GET.preselect}}'},
        )
    ]
    formdef.store()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un'},
                {'id': '2', 'text': 'deux'},
                {'id': '3', 'text': 'trois'},
                {'id': '4', 'text': 'quatre'},
            ]
        ),
    }
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == '1'

    resp = get_app(pub).get('/test/?preselect=2')
    assert resp.form['f1'].value == '2'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert 'deux' in resp.text


def test_form_page_profile_prefill_list(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['', 'bar@localhost', 'foo@localhost'],
            required=False,
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    # invalid value
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['', 'bar@localhost'],
            required=False,
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'invalid value selected' in resp.text
    assert resp.forms[0]['f0'].value == ''


def test_form_page_formula_prefill_items_field(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='items',
            items=['foo', 'bar', 'baz'],
            prefill={'type': 'formula', 'value': '["foo", "baz"]'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0$element0'].checked
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked
    assert 'widget-prefilled' in resp.text

    resp.form['f0$element0'].checked = False
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert '>foo<' not in resp.text
    assert '>bar<' not in resp.text
    assert '>baz<' in resp.text

    # check with remote json
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='items',
            data_source=ds,
            display_disabled_items=True,
            prefill={'type': 'formula', 'value': '["2"]'},
        )
    ]

    resp = get_app(pub).get('/test/')
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked


def test_form_page_checkbox_prefill(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.BoolField(id='0', label='check', type='bool', prefill={'type': 'formula', 'value': 'True'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].checked
    resp = resp.forms[0].submit('submit')  # -> validation
    assert resp.forms[0]['f0'].value == 'True'
    assert resp.forms[0]['f0disabled'].checked
    assert resp.forms[0]['f0disabled'].attrs['disabled']

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'].checked = False
    resp = resp.forms[0].submit('submit')  # -> validation
    assert resp.forms[0]['f0'].value == 'False'
    assert not resp.forms[0]['f0disabled'].checked
    assert resp.forms[0]['f0disabled'].attrs['disabled']

    # check with locked value
    formdef.fields[0].prefill['locked'] = True
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].attrs['onclick']
    assert resp.forms[0]['f0'].checked
    resp.forms[0]['f0'].checked = False  # alter value while it's not allowed
    resp = resp.forms[0].submit('submit')  # -> validation
    assert resp.forms[0]['f0'].value == 'True'
    assert resp.forms[0]['f0disabled'].checked
    assert resp.forms[0]['f0disabled'].attrs['disabled']


def test_form_page_template_prefill_items_field(pub):
    BlockDef.wipe()
    create_user(pub)

    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.StringField(id='123', required=True, label='Test', type='string', varname='test'),
    ]
    block.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemsField(
            id='0',
            label='items',
            items=['foo', 'bar', 'baz'],
            prefill={'type': 'string', 'value': 'foo,baz'},
        ),
        fields.FileField(id='1', label='file', type='file', varname='file'),
        fields.BlockField(id='2', label='test', type='block:foobar', varname='foobar'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0$element0'].checked
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked
    # this selection will be reused in the complex data test
    resp.form['f0$element1'].checked = True
    resp.form['f0$element2'].checked = False
    assert 'widget-prefilled' in resp.text
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2$element0$f123'] = 'plop'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1

    # check with remote json
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields[0] = fields.ItemsField(
        id='0',
        label='items',
        data_source=ds,
        display_disabled_items=True,
        prefill={'type': 'string', 'value': 'foo,baz'},
    )
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={
                'data': [
                    {'id': 'foo', 'text': 'hello'},
                    {'id': 'bar', 'text': 'world'},
                    {'id': 'baz', 'text': '!'},
                ]
            },
        )

        resp = get_app(pub).get('/test/')
        assert resp.form['f0$elementfoo'].checked
        assert not resp.form['f0$elementbar'].checked
        assert resp.form['f0$elementbaz'].checked

        # check with template returning a complex data
        formdef.fields[0] = fields.ItemsField(
            id='0',
            varname='items',
            label='items',
            data_source=ds,
            display_disabled_items=True,
            prefill={'type': 'string', 'value': '{{form_objects|first|get:"form_var_items_raw"}}'},
        )
        formdef.store()

        # it will use foo,bar as selected in the first part of this test
        resp = get_app(pub).get('/test/')
        assert resp.form['f0$elementfoo'].checked
        assert resp.form['f0$elementbar'].checked
        assert not resp.form['f0$elementbaz'].checked

        # check with complex data of wrong type
        for invalid_prefill_value in [
            {'type': 'string', 'value': '{{form_objects|first|get:"form_var_file_raw"}}'},
            {'type': 'string', 'value': '{{form_objects|first|get:"form_var_foobar"}}'},
            {'type': 'formula', 'value': '[{"a": "foo", "b": "baz"}]'},
        ]:
            formdef.fields[0] = fields.ItemsField(
                id='0',
                varname='items',
                label='items',
                data_source=ds,
                display_disabled_items=True,
                prefill=invalid_prefill_value,
            )
            formdef.store()

            resp = get_app(pub).get('/test/')
            assert not resp.form['f0$elementfoo'].checked
            assert not resp.form['f0$elementbar'].checked
            assert not resp.form['f0$elementbaz'].checked

            assert pub.loggederror_class.count() == 1
            logged_error = pub.loggederror_class.select()[0]
            assert logged_error.summary == 'Invalid value for items prefill on field "items"'
            pub.loggederror_class.wipe()

        # check with a "none" explicit prefill, or a None value
        for none_prefill_value in [
            {},
            {'type': 'none'},
            {'type': 'string', 'value': '{{ None }}'},
            {'type': 'formula', 'value': 'None'},
        ]:
            formdef.fields[0] = fields.ItemsField(
                id='0',
                varname='items',
                label='items',
                data_source=ds,
                display_disabled_items=True,
                prefill=none_prefill_value,
            )
            formdef.store()

            # all checkboxes will be left unchecked
            resp = get_app(pub).get('/test/')
            assert not resp.form['f0$elementfoo'].checked
            assert not resp.form['f0$elementbar'].checked
            assert not resp.form['f0$elementbaz'].checked
            assert pub.loggederror_class.count() == 0


def test_form_page_changing_prefill(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3', label='string 2', prefill={'type': 'string', 'value': '{{ form_var_foo }} World'}
        ),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('previous')  # back to 1st page
    assert resp.form['f1'].value == 'Hello'
    resp.form['f1'] = 'Goodbye Cruel'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Goodbye Cruel World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    resp.form['f3'].value = 'Changed value'
    resp = resp.form.submit('previous')  # back to 1st page
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Changed value'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Changed value'


def test_form_page_changing_prefill_draft(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3', label='string 2', prefill={'type': 'string', 'value': '{{ form_var_foo }} World'}
        ),
        fields.PageField(id='4', label='3rd page', type='page'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp = resp.form.submit('previous')  # back to 2nd page
    assert resp.form['f3'].value == 'Hello World'
    resp = resp.form.submit('submit')  # -> 3rd page
    tracking_code = get_displayed_tracking_code(resp)

    # start with a new session and restore draft using the tracking code
    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()
    assert_current_page(resp, '3rd page')
    resp = resp.forms[1].submit('previous')  # back to 2nd page
    assert resp.forms[1]['f3'].value == 'Hello World'
    resp = resp.forms[1].submit('previous')  # back to 1st page
    assert resp.forms[1]['f1'].value == 'Hello'
    resp.forms[1]['f1'] = 'Goodbye Cruel'
    resp = resp.forms[1].submit('submit')  # -> 2nd page
    assert resp.forms[1]['f3'].value == 'Goodbye Cruel World'


def test_prefill_query_parameter(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(
            id='1',
            label='str',
            varname='foo',
            required=False,
            prefill={'type': 'string', 'value': '{{request.GET.prefill}}'},
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/?prefill=Hello')
    assert resp.form['f1'].value == 'Hello'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp = resp.form.submit('previous')  # back to 1st page
    # check it has not be reset to the empty string (as there's no request.GET
    # anymore)
    assert resp.form['f1'].value == 'Hello'


def test_form_captcha(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.store()

    # test authenticated users are not presented with a captcha
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' not in resp.text

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text

    session_id = list(app.cookies.values())[0].strip('"')
    session = pub.session_class.get(session_id)
    resp.form['captcha$q'] = session.get_captcha_token(resp.forms[0]['captcha$token'].value)['answer']
    resp = resp.form.submit('submit')
    assert resp.status_code == 302  # redirect when formdata is created

    # and check it gets it only once
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' not in resp.text


def test_form_captcha_and_no_validation_page(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.confirmation = False
    formdef.store()

    # test authenticated users are not stopped on a confirmation page
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert resp.status_code == 302  # redirect when formdata is created

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text


def test_form_table_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableField(
            id='0',
            label='table',
            type='table',
            rows=[force_str('à'), 'b'],
            columns=['c', 'd', force_str('e')],
            required=False,
        )
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': [['', '', ''], ['', '', '']]}
    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableField(
            id='0', label='table', type='table', rows=['a', 'b'], columns=['c', 'd', 'e'], required=True
        )
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' not in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$c-0-0'] = 'a'
    resp.form['f0$c-1-0'] = 'b'
    resp.form['f0$c-0-1'] = 'c'
    resp.form['f0$c-1-1'] = 'd'
    resp.form['f0$c-0-2'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'c', 'e'], ['b', 'd', '']]}

    # check table is present in received email (via form_details).
    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$c-0-0'] = 'àà'  # would trigger column length bug (#23072)
    resp.form['f0$c-1-0'] = 'bb'
    resp.form['f0$c-0-1'] = 'cc'
    resp.form['f0$c-1-1'] = 'dd'
    resp.form['f0$c-0-2'] = 'ee'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    # check rst2html didn't fail
    assert b'ee' in emails.get('New form (test)')['msg'].get_payload()[1].get_payload(decode=True)


def test_form_table_rows_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=False)
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': []}
    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True)
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' not in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = 'b'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = 'd'
    resp.form['f0$element2$col0'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'b'], ['c', 'd'], ['e', '']]}

    formdef.data_class().wipe()

    formdef.fields = [
        fields.TableRowsField(
            id='0', label='table', type='tablerows', columns=['a', 'b'], required=True, total_row=True
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert '37.00' in resp.text

    # check table is present in received email (via form_details).
    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$element0$col0'] = 'àà'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'ee'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert b'ee' in emails.get('New form (test)')['msg'].get_payload()[1].get_payload(decode=True)


def test_form_new_table_rows_field_draft_recall(pub):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # add new table rows field to formdef
    formdef.fields.append(
        fields.TableRowsField(id='3', label='table', type='tablerows', columns=['a', 'b'], required=False)
    )
    formdef.store()

    # restore form on validation page
    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()

    # validate form
    resp = resp.forms[1].submit()
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'test'
    assert formdef.data_class().select()[0].data['3'] is None


def test_form_table_rows_field_and_prefill(pub, emails):
    formdef = create_formdef()
    formdef.fields = [
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True),
        fields.StringField(id='1', label='string', prefill={'type': 'string', 'value': 'HELLO WORLD'}),
    ]
    formdef.store()
    formdef.data_class().wipe()

    get_app(pub).get('/test/')


def test_form_table_rows_add_row(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string', require=True),
        fields.TableRowsField(id='0', label='table', type='tablerows', columns=['a', 'b'], required=True),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 10
    resp = resp.form.submit('f0$add_element')
    assert 'There were errors processing the form' not in resp
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 12
    resp = resp.form.submit('f0$add_element')
    assert len(resp.pyquery.find('input[name^="f0$element"]')) == 14
    resp = resp.form.submit('submit')
    assert 'There were errors processing the form' in resp


def test_form_map_field_back_and_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.StringField(
            id='1', label='street', required=True, prefill={'type': 'geolocation', 'value': 'road'}
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.text
    assert 'qommon.geolocation.js' in resp.text
    # with a real user interaction this would get set by javascript
    resp.forms[0]['f0$latlng'].value = '1.234;-1.234'
    assert 'data-geolocation="road"' in resp.text

    # check required field
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' not in resp.text
    assert 'data-geolocation="road"' in resp.text
    resp.forms[0]['f1'].value = 'bla'

    # check summary page
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-init-lng="-1.234"' in resp.text
    assert 'data-init-lat="1.234"' in resp.text

    # get back to the map field
    resp = resp.forms[0].submit('previous')
    # check the field is still marked as holding the road
    assert 'data-geolocation="road"' in resp.text
    assert resp.forms[0]['f0$latlng'].value == '1.234;-1.234'

    # back to summary page
    resp = resp.forms[0].submit('submit')

    # and submitting the form
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'bla', '0': '1.234;-1.234'}


def test_form_map_initial_zoom_level(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'data-initial_zoom="13"' in resp.text
    pub.cfg['misc']['default-zoom-level'] = '16'
    pub.write_cfg()
    resp = get_app(pub).get('/test/')
    assert 'data-initial_zoom="16"' in resp.text

    formdef.fields[0].initial_zoom = '11'
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert 'data-initial_zoom="11"' in resp.text


def test_form_map_geolocation_text_field(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.TextField(
            id='1', label='street', required=True, prefill={'type': 'geolocation', 'value': 'road'}
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.text
    assert 'qommon.geolocation.js' in resp.text
    assert 'WCS_DEFAULT_GEOCODING_COUNTRY' not in resp.text

    # check page has default geocoding country in a javascript variable if set
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'default-geocoding-country', 'France')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    resp = get_app(pub).get('/test/')
    assert 'WCS_DEFAULT_GEOCODING_COUNTRY' in resp.text


def test_form_map_field_prefill_address(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='address', required=True, varname='address'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.MapField(id='3', label='map', prefill={'type': 'string', 'value': '{{ form_var_address }}'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '169 rue du chateau, paris'
    with responses.RequestsMock() as rsps:
        rsps.get('https://nominatim.entrouvert.org/search', json=[{'lat': '48.8337085', 'lon': '2.3233693'}])
        resp = resp.form.submit('submit')
        assert resp.form['f3$latlng'].value == '48.8337085;2.3233693'
        assert 'chateau' in rsps.calls[0].request.url


def test_form_map_field_prefill_coords(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.MapField(id='1', label='map', type='map', varname='map1'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.MapField(id='3', label='map', prefill={'type': 'string', 'value': '{{ form_var_map1 }}'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1$latlng'] = '1.234;-1.234'
    resp = resp.form.submit('submit')
    assert resp.form['f3$latlng'].value == '1.234;-1.234'


def test_form_map_multi_page(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.MapField(id='1', label='map'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1$latlng'] = '1.234;-1.234'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f1$latlng'].value == '1.234;-1.234'
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': '1.234;-1.234', '3': 'bar'}


def test_form_middle_session_change(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    app.cookiejar.clear()
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/'
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    app.cookiejar.clear()
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text


def test_form_autocomplete_variadic_url(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(
            id='3', label='2nd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
        fields.ItemField(id='1', label='string', type='item', varname='foo', items=['Foo', 'Bar']),
        fields.StringField(
            id='2', label='string2', required=True, data_source={'type': 'jsonp', 'value': '[var_foo]'}
        ),
        fields.PageField(
            id='4', label='3rd page', type='page', condition={'type': 'python', 'value': 'True'}
        ),
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')  # next
    # test javascript will be used to compute the full URL
    assert 'options.wcs_base_url' in resp.text
    assert 'jquery-ui.min.js' in resp.text

    # test going forward (will error out), check it's still a variadic URL (#9786)
    resp.form['f1'] = 'Foo'
    resp = resp.form.submit('submit')
    assert 'options.wcs_base_url' in resp.text


def test_form_page_formula_prefill_user_name(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'form_user_email'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'


def test_form_page_formula_prefill_session_user(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', prefill={'type': 'formula', 'value': 'session_user_email'})
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    app = login(get_app(pub), username='foo', password='foo')
    for session in pub.session_manager.values():
        session.extra_user_variables = {'foo': 'bar'}
        session.store()
    formdef.fields = [
        fields.StringField(
            id='0', label='string', prefill={'type': 'formula', 'value': 'session_var_user_foo'}
        )
    ]
    formdef.store()
    resp = app.get('/test/')
    assert resp.forms[0]['f0'].value == 'bar'


def test_form_date_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='string', type='date', required=False)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f0'] = '2015-01-01'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert time.strftime('%Y-%m-%d', data.data['0']) == '2015-01-01'

    # without filling the field
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None


def test_form_jsonp_item_field(http_requests, pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='string',
            type='item',
            data_source={'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'},
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert 'data-select2-url="http://remote.example.net/jsonp"' in resp.text
    assert 'select2.min.js' in resp.text


def test_form_string_regex_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(
            id='0',
            label='string',
            type='string',
            validation={'type': 'regex', 'value': r'\d{5}$'},
            required=False,
        )
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # with an invalid input
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    assert 'invalid value' in resp.text


def test_form_text_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TextField(id='0', label='string', type='text', required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # check max length
    formdef.fields = [fields.TextField(id='0', label='string', type='text', maxlength=10)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'x' * 11
    resp = resp.forms[0].submit('submit')
    assert 'too many characters (limit is 10)' in resp.text
    # check it counts characters, not bytes
    resp.forms[0]['f0'] = '☭' * 10
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text


def test_unknown_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(
            id='1', label='string', varname='string', required=False, data_source={'type': 'foobar'}
        ),
        fields.ItemField(
            id='2', label='item', varname='item', required=False, data_source={'type': 'foobar'}
        ),
        fields.ItemsField(
            id='3', label='items', varname='items', required=False, data_source={'type': 'foobar'}
        ),
    ]

    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text


def test_form_items_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemsField(
            id='1', label='items', varname='items', required=False, data_source={'type': 'foobar'}
        )
    ]
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()

    # add the named data source
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')  # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert 'f1$elementun' in resp.form.fields
    assert 'f1$elementdeux' in resp.form.fields
    resp.form['f1$elementun'].checked = True
    resp.form['f1$elementdeux'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {
        '1': ['un', 'deux'],
        '1_display': 'un, deux',
        '1_structured': [{'id': 'un', 'text': 'un'}, {'id': 'deux', 'text': 'deux'}],
    }

    data_source.data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]),
    }
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {
        '1': ['1', '2'],
        '1_display': 'un, deux',
        '1_structured': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}],
    }

    data_source.data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un', 'foo': 'bar1'}, {'id': '2', 'text': 'deux', 'foo': 'bar2'}]),
    }
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert data_class.select()[0].data == {
        '1': ['1', '2'],
        '1_structured': [
            {'text': 'un', 'foo': 'bar1', 'id': '1'},
            {'text': 'deux', 'foo': 'bar2', 'id': '2'},
        ],
        '1_display': 'un, deux',
    }
    # along the way, check substitution variables
    substvars = data_class.select()[0].get_substitution_variables()
    assert substvars['form_var_items'] == 'un, deux'
    assert substvars['form_var_items_raw'] == ['1', '2']
    assert substvars['form_var_items_0_foo'] == 'bar1'
    assert substvars['form_var_items_1_foo'] == 'bar2'


def test_form_ranked_items_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.RankedItemsField(
            id='0', label='ranked items', type='ranked-items', required=False, items=['foo', 'bar', 'baz']
        )
    ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0$element0'] = '1'
    resp.form['f0$element1'] = '2'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == {'bar': 2, 'foo': 1}


def test_form_ranked_items_randomize_order(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.RankedItemsField(
            id='0',
            label='ranked items',
            type='ranked-items',
            required=False,
            randomize_items=True,
            items=['foo', 'bar', 'baz'],
        )
    ]
    formdef.store()
    orders = {}
    for _ in range(10):
        resp = get_app(pub).get('/test/')
        orders['%s-%s-%s' % (resp.text.index('foo'), resp.text.index('bar'), resp.text.index('baz'))] = True
    assert len(orders.keys()) > 1


def test_form_autosave(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'

    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'foobar'

    resp.form['f1'] = 'foobar2'
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().select()[0].data['1'] == 'foobar2'

    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    resp.form['f3'] = 'xxx'
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx'

    resp.form['f3'] = 'xxx2'
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx2'

    resp.form['f3'] = 'xxx3'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'foobar3' in resp.text
    assert 'xxx3' in resp.text

    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx3'

    # make sure autosave() doesn't destroy data that would have been submitted
    # in the meantime
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'
    autosave_fields = resp.form.submit_fields()
    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    # post content with 'foobar' as value, it should not be saved
    ajax_resp = app.post('/test/autosave', params=autosave_fields)
    assert json.loads(ajax_resp.text)['result'] == 'error'
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'


def test_form_autosave_timeout(pub, monkeypatch):
    from wcs.forms.root import FormPage

    monkeypatch.setattr(FormPage, 'AUTOSAVE_TIMEOUT', 0.0001)

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string 2'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'

    resp = app.post('/test/autosave', params=resp.form.submit_fields())
    assert resp.json == {'reason': 'too long', 'result': 'error'}


def test_form_autosave_with_items_field(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemsField(
            id='3',
            label='items',
            type='items',
            items=[force_str(x) for x in ('pomme', 'poire', 'pêche', 'abricot')],
        ),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') is None

    resp = resp.forms[0].submit('submit')
    resp.form['f3$element1'].checked = True
    resp.form['f3$element3'].checked = True
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == ['poire', 'abricot']


def test_form_autosave_with_invalid_data(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.EmailField(id='1', label='email', type='email'),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'  # not a valid email

    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'foobar'

    # restore draft
    tracking_code = get_displayed_tracking_code(resp)
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert resp.forms[1]['f1'].value == 'foobar'  # not a valid email


@responses.activate
def test_form_autosave_item_field_data_source_error(pub):
    ds = {'type': 'json', 'value': 'http://www.example.net/plop'}
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', label='string', data_source=ds),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    responses.get(
        'http://www.example.net/plop', json={'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]}
    )

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = '1'  # not a valid email

    # make the ds fails
    with mock.patch.object(NamedDataSource, 'get_structured_value', lambda *args: None):

        autosave_resp = app.post('/test/autosave', params=resp.form.submit_fields())
    assert autosave_resp.json == {
        'reason': 'form deserialization failed: a datasource is unavailable',
        'result': 'error',
    }

    autosave_resp = app.post('/test/autosave', params=resp.form.submit_fields())
    assert autosave_resp.json['result'] == 'success'


def test_form_autosave_with_parameterized_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(
            id='3',
            label='item',
            type='item',
            data_source={'type': 'formula', 'value': '''[('1', form_var_foo*2)]'''},
        ),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') is None

    resp = resp.forms[0].submit('submit')
    assert app.post('/test/autosave', params=resp.form.submit_fields()).json == {'result': 'success'}
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == '1'
    assert formdef.data_class().select()[0].data['3_display'] == 'barbar'


def test_form_autosave_never_overwrite(mocker, pub, settings):
    create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string1'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='3', label='string2'),
    ]
    formdef.store()

    app = get_app(pub)
    login(app, username='foo', password='foo')

    resp = app.get('/test/')
    resp.form['f1'] = '1'
    # go to the second page
    resp = resp.form.submit('submit')
    resp.form['f3'] = 'tmp'
    # autosave this temporary data
    autosave_data = dict(resp.form.submit_fields())
    resp_autosave = app.post('/test/autosave', params=autosave_data)
    assert resp_autosave.json == {'result': 'success'}
    # check the draft has been modified
    formdata = formdef.data_class().select()[0]
    formdata.refresh_from_storage()
    assert formdata.data['3'] == 'tmp'
    # now finish submitting with new value
    resp.form['f3'] = '1'
    resp = resp.form.submit('submit')  # -> validation page
    formdata.refresh_from_storage()
    assert formdata.data['3'] == '1'
    # autosave wrong data
    # _ajax_form_token is just a form_token, so take the current one to
    # simulate a rogue autosave from the previous page
    autosave_data['_ajax_form_token'] = resp.form['_form_id'].value
    resp_autosave = app.post('/test/autosave', params=autosave_data)
    formdata.refresh_from_storage()
    assert resp_autosave.json != {'result': 'success'}
    assert formdata.data['3'] == '1'
    # validate
    resp = resp.form.submit('submit')  # -> submit

    # everything is still fine in the end, even for pickle storage
    # (as the overwritten # data are recreated from validation page)
    assert formdef.data_class().select()[0].data == {'1': '1', '3': '1'}


def test_form_string_field_autocomplete(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string', required=False)]
    formdef.fields[0].data_source = {'type': 'jsonp'}
    formdef.store()

    # not filled completed, no call to .autocomplete
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' not in resp.text

    # straight URL
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': 'http://example.org'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.org' in resp.text

    # URL from variable
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': '[site_url]'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.net' in resp.text


def test_form_string_field_autocomplete_named_datasource(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = [
        fields.StringField(
            id='0', label='string', type='string', required=False, data_source={'type': 'foobar'}
        )
    ]
    formdef.store()

    # jsonp datasource
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'jsonp', 'value': 'http://remote.example.net/json'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" in resp.text
    assert "options.url = '/api/autocomplete/" not in resp.text
    assert 'dataType: "jsonp",' in resp.text

    # json datasource
    data_source.data_source['type'] = 'json'
    data_source.query_parameter = 'q'
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" not in resp.text
    assert "options.url = '/api/autocomplete/" in resp.text
    assert 'dataType: "json",' in resp.text

    # card datasource
    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'Foo'
    carddef.fields = []
    carddef.store()

    data_source.data_source['type'] = 'carddef:foo'
    data_source.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert "options.url = 'http://remote.example.net/json'" not in resp.text
    assert "options.url = '/api/autocomplete/" in resp.text
    assert 'dataType: "json",' in resp.text


def test_form_autocomplete_named_datasource_expired_token(pub):
    CardDef.wipe()
    FormDef.wipe()
    pub.token_class.wipe()

    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = [
        fields.StringField(
            id='0', label='string', type='string', required=False, data_source={'type': 'foobar'}
        )
    ]
    formdef.store()

    carddef = CardDef()
    carddef.name = 'Foo'
    carddef.fields = []
    carddef.store()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'carddef:foo'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert pub.token_class.count() == 1
    token = pub.token_class.select()[0]
    assert '/api/autocomplete/%s' % token.id in resp.text
    token.set_expiration_delay(-1)  # expired
    token.store()

    # check a new token is generated
    resp = get_app(pub).get('/test/')
    assert '/api/autocomplete/%s' % token.id not in resp.text


def test_form_workflow_trigger(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = st1.add_action('jump')
    jump.trigger = 'XXX'
    jump.status = 'st2'

    jump2 = st1.add_action('jump')
    jump2.trigger = 'YYY'
    jump2.status = 'st3'
    jump2.set_marker_on_status = True

    workflow.add_status('Status2', 'st2')
    workflow.add_status('Status3', 'st3')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    login(app, username='foo', password='foo').get('/')
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    jump.by = [role.id]
    workflow.store()
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    user.roles = [role.id]
    user.store()
    app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st2'

    formdata.status = 'wf-st1'
    formdata.store()
    app.post(formdata.get_url() + 'jump/trigger/YYY', status=403)
    jump2.by = [role.id]
    workflow.store()
    app.post(formdata.get_url() + 'jump/trigger/YYY', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st3'
    assert formdata.workflow_data.get('_markers_stack') == [{'status_id': 'st1'}]

    formdata.status = 'wf-st1'
    formdata.store()
    app.post(
        formdata.get_url() + 'jump/trigger/YYY',
        params=json.dumps({'data': {'foo': 'bar'}}),
        content_type='application/json',
    )
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.workflow_data.get('data') == {'foo': 'bar'}


def test_form_worklow_multiple_identical_status(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    st1.extra_css_class = 'CSS-STATUS1'
    jump = st1.add_action('jump')
    jump.trigger = 'XXX'
    jump.status = 'st1'
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.allows_backoffice_access = False
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    assert len(formdef.data_class().get(formdata.id).evolution) == 1
    assert formdef.data_class().get(formdata.id).evolution[0].last_jump_datetime is None

    login(app, username='foo', password='foo')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    # status is not changed: no new evolution, only a new last_jump_datetime
    assert len(formdata.evolution) == 1
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].last_jump_datetime is not None
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # add a comment to last evolution, forcing create a new one
    formdata.evolution[-1].comment = 'new-evolution-1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 2
    assert formdata.status == 'wf-st1'
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # again
    formdata.evolution[-1].comment = 'new-evolution-2'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # last evolution is empty, this last trigger does not create a new one
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # finally, 3 evolutions: new-evolution-1, new-evolution-2, empty
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 3
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].comment == 'new-evolution-1'
    assert formdata.evolution[1].comment == 'new-evolution-2'
    assert formdata.evolution[2].comment is None

    # mark user as owner so it can check the UI
    formdata.user_id = user.id
    formdata.store()
    resp = app.get(formdata.get_url())
    assert resp.text.count('Status1') == 3  # once in summary and two in journal
    assert resp.text.count('CSS-STATUS1') == 2
    assert resp.text.count('new-evolution-1') == 1
    assert resp.text.count('new-evolution-2') == 1


def test_form_worklow_comments_on_same_status(pub):
    pub.session_manager.session_class.wipe()
    user = create_user(pub)

    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    workflow = Workflow.get_default_workflow()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.perform_workflow()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-new'

    app = get_app(pub)

    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is True
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True
    login(app, username='foo', password='foo')
    resp = app.get(formdata.get_url()).follow()
    resp.form['comment'] = 'TEST COMMENT'
    resp = resp.form.submit('button_commentable')
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is False
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    resp = app.get(formdata.get_url()).follow()
    resp = resp.form.submit('button_accept')
    assert (
        formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed']
        is True
    )
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True


def test_form_worklow_double_comments(pub):
    Workflow.wipe()

    create_user(pub)

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    commentable = st1.add_action('commentable', id='1')
    commentable.by = [logged_users_role().id]
    commentable = st1.add_action('commentable', id='2')
    commentable.by = [logged_users_role().id]
    wf.store()

    formdef = create_formdef()
    formdef.workflow = wf
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.perform_workflow()
    formdata.store()

    app = get_app(pub)
    login(app, username='foo', password='foo')
    resp = app.get(formdata.get_url())
    resp.form['comment'] = 'TEST COMMENT'
    resp = resp.form.submit('button_commentable').follow()
    assert resp.text.count('TEST COMMENT') == 1


def test_display_message(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st0 = workflow.add_status('Status0', 'st0')
    jump = st0.add_action('jump')
    jump.status = 'st1'

    st1 = workflow.add_status('Status1', 'st1')

    display1 = st1.add_action('displaymsg')
    display1.message = 'message-to-all'
    display1.to = []

    display2 = st1.add_action('displaymsg')
    display2.message = 'message-to-submitter'
    display2.to = ['_submitter']

    display3 = st1.add_action('displaymsg')
    display3.message = 'message-to-nobody'
    display3.to = ['xxx']

    display4 = st1.add_action('displaymsg')
    display4.message = 'message-to-xxx-and-submitter'
    display4.to = ['_submitter', 'xxx']

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()

    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-nobody' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') < page.text.index('message-to-xxx-and-submitter')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    # actions alert vs top alert
    display2.position = 'actions'
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # add an action, so display2 will appear again
    jump1 = st1.add_action('choice', id='_jump1')
    jump1.label = 'Jump 1'
    jump1.by = ['_submitter']
    jump1.status = st1.id
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    jump1.by = ['xxx']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # change to always display at the bottom
    display2.position = 'bottom'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    # set a level
    display2.level = 'warning'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' in page.text

    # check message is not displayed if status is not visible to user
    st1.visibility = ['_receiver']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' not in page.text


def test_workflow_condition_on_message(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = st1.add_action('displaymsg')
    display1.message = 'message-to-all'
    display1.to = []

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit')  # form page
    page = page.forms[0].submit('submit')  # confirmation page
    page = page.follow()
    assert 'message-to-all' in page.text

    formdata = formdef.data_class().select()[0]
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text

    display1.condition = {'type': 'django', 'value': 'xxx'}
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' not in page.text


def test_workflow_message_with_template_error(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = st1.add_action('displaymsg')
    display1.message = '<p>{% for x in 0 %}crash{% endfor %}'
    display1.to = []

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')  # form page
    resp = resp.forms[0].submit('submit')  # confirmation page
    resp = resp.follow()
    assert 'Error rendering message.' in resp.text

    assert pub.loggederror_class.count() == 1
    logged_error = pub.loggederror_class.select()[0]
    assert logged_error.summary == "Error in template of workflow message ('int' object is not iterable)"


def test_session_cookie_flags(pub):
    create_formdef()
    app = get_app(pub)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('sessionid-')
    assert 'HttpOnly' in resp.headers['Set-Cookie']
    assert 'Secure' not in resp.headers['Set-Cookie']

    app = get_app(pub, https=True)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('sessionid-')
    assert 'HttpOnly' in resp.headers['Set-Cookie']
    assert 'Secure' in resp.headers['Set-Cookie']


def test_form_page_profile_verified_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'readonly' not in resp.form['f0'].attrs
    resp.form['f0'].value = 'Hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'Hello'

    user.verified_fields = ['email']
    user.store()

    for prefill_settings in (
        {'type': 'user', 'value': 'email'},  # verified profile
        {'type': 'string', 'value': 'foo@localhost', 'locked': True},  # locked value
    ):
        formdef.confirmation = True
        formdef.fields[0].prefill = prefill_settings
        formdef.store()
        formdef.data_class().wipe()
        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp.form['f0'].value = 'Hello'  # try changing the value
        resp = resp.form.submit('submit')
        assert 'Check values then click submit.' in resp.text
        assert resp.form['f0'].value == 'foo@localhost'  # it is reverted

        resp.form['f0'].value = 'Hello'  # try again changing the value
        resp = resp.form.submit('submit')

        formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
        assert len(formdatas) == 1
        assert formdatas[0].data['0'] == 'foo@localhost'

        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        resp = resp.form.submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp.form['f0'].value = 'Hello'  # try changing
        resp = resp.form.submit('previous')
        assert 'readonly' in resp.form['f0'].attrs
        assert 'Check values then click submit.' not in resp.text
        assert resp.form['f0'].value == 'foo@localhost'

        # try it without validation page
        formdef.confirmation = False
        formdef.store()
        formdef.data_class().wipe()

        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp.form['f0'].value = 'Hello'  # try changing the value
        resp = resp.form.submit('submit')

        formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
        assert len(formdatas) == 1
        assert formdatas[0].data['0'] == 'foo@localhost'


def test_form_worklow_multiple_identical_status_with_wserror(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = st1.add_action('jump')
    jump.trigger = 'XXX'
    jump.status = 'st1'
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    login(app, username='foo', password='foo')

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.allows_backoffice_access = False
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    for _i in range(3):
        resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
        formdata = formdef.data_class().get(formdata.id)
        formdata.evolution[-1].add_part(JournalWsCallErrorPart('bla', 'bla', {}))
        formdata.store()

    # mark user as owner so it can check the UI
    formdata.user_id = user.id
    formdata.store()
    resp = app.get(formdata.get_url())
    assert resp.text.count('<li class="msg') == 1

    role.allows_backoffice_access = True
    role.store()
    resp = app.get(formdata.get_url(backoffice=True))
    assert resp.text.count('<li class="msg') == 3


def test_form_page_verified_prefill_error_page(pub):
    user = create_user(pub)
    user.verified_fields = ['email']
    user.store()
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string', required=False),
        fields.StringField(id='1', label='string2', required=True),
    ]
    formdef.store()

    for prefill_settings in (
        {'type': 'user', 'value': 'email'},  # verified profile
        {'type': 'string', 'value': 'foo@localhost', 'locked': True},  # locked value
    ):
        formdef.fields[0].prefill = prefill_settings
        formdef.store()
        formdef.data_class().wipe()
        resp = login(get_app(pub), username='foo', password='foo').get('/test/')
        assert resp.form['f0'].value == 'foo@localhost'
        assert 'readonly' in resp.form['f0'].attrs

        resp = resp.form.submit('submit')
        assert 'There were errors processing the form' in resp.text
        assert 'readonly' in resp.form['f0'].attrs


def test_form_page_profile_verified_date_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.DateField(id='_date', label='date', type='date'))
    user_formdef.store()
    user.form_data = {'_date': time.strptime('2018-09-27', '%Y-%m-%d')}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.DateField(id='0', label='date', prefill={'type': 'user', 'value': '_date'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    assert 'readonly' not in resp.form['f0'].attrs
    assert resp.form['f0'].attrs['type'] == 'date'
    resp.form['f0'].value = '2018-09-27'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'

    user.verified_fields = ['_date']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    # for readonly values there is a <input type=hidden> with the real value
    # then an unnamed <input type=text> with the formatted value.
    assert resp.form['f0'].attrs['type'] == 'hidden'
    assert resp.pyquery('input#form_f0[type=text][readonly=readonly]')

    resp.form['f0'].value = '2018-09-24'  # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'  # it is reverted

    resp.form['f0'].value = '2018-09-24'  # try again changing the value
    resp = resp.form.submit('submit')

    formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
    assert len(formdatas) == 1
    assert time.strftime('%Y-%m-%d', formdatas[0].data['0']) == '2018-09-27'


def test_form_page_profile_date_as_locked_string_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.DateField(id='_date', label='date', type='date'))
    user_formdef.store()
    user.form_data = {'_date': time.strptime('2018-09-27', '%Y-%m-%d')}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.PageField(id='1', label='1st page', type='page'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(id='0', label='date', prefill={'type': 'user', 'value': '_date', 'locked': True}),
    ]
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> second page
    assert resp.form['f0'].value == '2018-09-27'
    assert 'readonly' in resp.form['f0'].attrs
    resp.form['f0'].value = '2015-09-27'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'


def test_form_page_profile_verified_radio_item_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='item',
            type='item',
            items=['bar@localhost', 'foo@localhost', 'baz@localhost'],
            display_mode='radio',
            prefill={'type': 'user', 'value': 'email'},
        )
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value is None

    user.verified_fields = ['email']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'disabled' in resp.form['f0'].attrs
    for radio in resp.html.findAll('input'):
        if radio['name'] == 'f0':
            if radio['value'] == 'foo@localhost':
                assert radio.attrs.get('checked')
                assert not radio.attrs.get('disabled')
            else:
                assert not radio.attrs.get('checked')
                assert radio.attrs.get('disabled')

    resp.form['f0'].value = 'baz@localhost'  # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'foo@localhost'  # it is reverted


def test_item_field_from_cards(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].options == [
        ('2', False, 'bar'),
        ('3', False, 'baz'),
        ('1', False, 'foo'),
    ]
    resp.form['f0'] = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'
    assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'


def test_item_field_from_cards_id_identifier(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='id'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].options == [
        ('2', False, 'bar'),
        ('3', False, 'baz'),
        ('1', False, 'foo'),
    ]
    resp.form['f0'] = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'
    assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'


def test_item_field_from_cards_then_comment_related_card(pub):
    # https://dev.entrouvert.org/issues/58292
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}

    carddef2 = CardDef()
    carddef2.name = 'others'
    carddef2.fields = [fields.ItemField(id='0', varname='card1', type='item', data_source=ds)]
    carddef2.store()
    carddata2 = carddef2.data_class()()
    carddata2.data = {'0': str(carddata.id), '0_display': 'baz'}
    carddata2.just_created()
    carddata2.store()

    formdef.fields = [
        fields.PageField(id='1', label='1st page', type='page'),
        fields.ItemField(
            id='0', label='string', type='item', data_source=ds, display_disabled_items=True, varname='card'
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.CommentField(
            id='3',
            label='<p>card value: {{ cards|objects:"others"|filter_by:"card1"|filter_value:form_var_card|first|get:"form_number" }}</p>',
            type='comment',
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f0'] = '3'
    resp = resp.form.submit('submit')  # -> second page
    assert 'card value: %s' % carddata2.get_display_id() in resp.text


def test_item_field_from_custom_view_on_cards(pub):
    pub.role_class.wipe()
    pub.custom_view_class.wipe()

    user = create_user(pub)
    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.is_admin = True
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_attr}}'}
    carddef.workflow_roles = {'_editor': user.roles[0]}
    carddef.fields = [
        fields.ItemField(id='0', type='item', label='item', varname='item', items=['foo', 'bar', 'baz']),
        fields.StringField(id='1', type='string', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    baz_ids = set()
    for i, value in enumerate(['foo', 'bar', 'baz'] * 10):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '0_display': value,
            '1': 'attr%s' % (i + 1),
        }
        carddata.just_created()
        carddata.store()
        if value == 'baz':
            baz_ids.add(str(carddata.id))

    # create custom view
    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/backoffice/data/items/')
    assert resp.text.count('<tr') == 21  # thead + 20 items (max per page)
    resp.forms['listing-settings']['filter-0'].checked = True
    resp = resp.forms['listing-settings'].submit()

    resp.forms['listing-settings']['filter-0-value'] = 'baz'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text.count('<tr') == 11  # thead + 10 items

    resp.forms['save-custom-view']['title'] = 'as data source'
    resp.forms['save-custom-view']['visibility'] = 'datasource'
    resp = resp.forms['save-custom-view'].submit()

    custom_view = pub.custom_view_class.select()[0]

    # use custom view as source
    ds = {'type': 'carddef:%s:%s' % (carddef.url_name, custom_view.slug)}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 10
    baz_id = list(baz_ids)[0]
    assert {x[0] for x in resp.form['f0'].options} == baz_ids
    resp.form['f0'].value = baz_id
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == baz_id
    assert formdef.data_class().select()[0].data['0_display'] == 'attr%s' % baz_id
    assert formdef.data_class().select()[0].data['0_structured']['item'] == 'baz'

    # give custom view it a custom digest
    formdef.data_class().wipe()
    carddef.digest_templates['custom-view:%s' % custom_view.slug] = 'X{{form_var_attr}}Y'
    carddef.store()
    # compute digests
    for carddata in carddef.data_class().select():
        carddata.store()

    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 10
    assert {x[0] for x in resp.form['f0'].options} == baz_ids
    resp.form['f0'].value = baz_id
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == baz_id
    assert formdef.data_class().select()[0].data['0_display'] == 'Xattr%sY' % baz_id


def test_item_field_from_custom_view_on_cards_filter_status(pub):
    pub.role_class.wipe()
    pub.custom_view_class.wipe()

    user = create_user(pub)
    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.is_admin = True
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    card_workflow = CardDef.get_default_workflow()
    st1 = card_workflow.add_status('Status1', 'st1')
    card_workflow.id = None
    card_workflow.store()

    CardDef.wipe()
    carddef = CardDef()
    carddef.workflow_id = card_workflow.id
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_attr}}'}
    carddef.workflow_roles = {'_editor': user.roles[0]}
    carddef.fields = [
        fields.ItemField(id='0', type='item', label='item', varname='item', items=['foo', 'bar', 'baz']),
        fields.StringField(id='1', type='string', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '0_display': value,
            '1': 'attr%s' % (i + 1),
        }
        carddata.just_created()
        carddata.store()

    carddata.jump_status(st1.id)
    carddata.store()

    # create custom view
    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/backoffice/data/items/')
    resp.forms['listing-settings']['filter-status'].checked = True
    resp = resp.forms['listing-settings'].submit()
    resp.forms['listing-settings']['filter-operator'].value = 'ne'
    resp.forms['listing-settings']['filter'].value = 'st1'
    resp = resp.forms['listing-settings'].submit()
    assert resp.pyquery('tbody tr').length == 2

    resp.forms['save-custom-view']['title'] = 'as data source'
    resp.forms['save-custom-view']['visibility'] = 'datasource'
    resp = resp.forms['save-custom-view'].submit()

    custom_view = pub.custom_view_class.select()[0]
    assert custom_view.filters == {'filter-operator': 'ne', 'filter': 'st1', 'filter-status': 'on'}

    # use custom view as source
    ds = {'type': 'carddef:%s:%s' % (carddef.url_name, custom_view.slug)}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 2
    assert {x[2] for x in resp.form['f0'].options} == {'attr1', 'attr2'}

    custom_view.filters['filter-operator'] = 'eq'
    custom_view.store()
    resp = get_app(pub).get('/test/')
    assert [x[2] for x in resp.form['f0'].options] == ['attr3']

    custom_view.filters['filter'] = 'all'
    custom_view.store()
    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 3

    custom_view.filters['filter'] = 'all'
    custom_view.filters['filter-operator'] = 'ne'
    custom_view.store()
    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 1
    assert [x[2] for x in resp.form['f0'].options] == ['---']


@pytest.mark.parametrize('filter_value', ['{{ "foo" }}', 'foo'])
def test_items_field_from_custom_view_on_cards(pub, filter_value):
    pub.role_class.wipe()
    pub.custom_view_class.wipe()

    user = create_user(pub)
    role = pub.role_class(name='xxx')
    role.store()
    user.roles = [role.id]
    user.is_admin = True
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()

    items = ['foo', 'bar', 'baz', 'buz']
    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_attr}} - {{form_var_item}}'}
    carddef.workflow_roles = {'_editor': user.roles[0]}
    carddef.fields = [
        fields.ItemsField(id='0', type='items', label='item', varname='item', items=items),
        fields.StringField(id='1', type='string', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    foo_bar_ids = set()
    for i, (v1, v2) in enumerate([(v1, v2) for (v1, v2) in itertools.product(items, items) if v1 != v2]):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': [v1, v2],
            '0_display': '%s,%s' % (v1, v2),
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()
        if 'foo' in {v1, v2}:
            foo_bar_ids.add(str(carddata.id))

    # create custom view
    app = login(get_app(pub), username='foo', password='foo')

    # we must force the ordering to have a determinist test
    resp = app.get('/backoffice/data/items/?order_by=id')
    assert resp.text.count('<tr') == 13  # thead + 12 items (max per page)
    resp.forms['listing-settings']['filter-0'].checked = True
    resp = resp.forms['listing-settings'].submit()

    resp.forms['listing-settings']['filter-0-value'].force_value(filter_value)
    resp = resp.forms['listing-settings'].submit()

    resp.forms['save-custom-view']['title'] = 'as data source'
    resp.forms['save-custom-view']['visibility'] = 'datasource'
    resp = resp.forms['save-custom-view'].submit()

    custom_view = pub.custom_view_class.select()[0]

    # use custom view as source
    ds = {'type': 'carddef:%s:%s' % (carddef.url_name, custom_view.slug)}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert len(resp.form['f0'].options) == 6  # 12 - ['baz,buz', 'buz,baz']
    assert {x[0] for x in resp.form['f0'].options} == foo_bar_ids
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    formdata = formdef.data_class().select()[0]
    assert formdata.data['0'] in foo_bar_ids
    assert formdata.data['0_structured']['text'] == 'attr0 - foo,bar'


def test_item_field_with_disabled_items(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit')  # -> validation page
        assert 'There were errors processing the form' in resp.text

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=False)
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('option[disabled=disabled][value="1"]')) == 0
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source=ds,
            display_mode='radio',
            display_disabled_items=True,
        )
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit')  # -> validation page
        assert 'There were errors processing the form' in resp.text


def test_items_field_with_disabled_items(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [
        fields.ItemsField(id='0', label='string', type='items', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['1', '2']
        assert formdef.data_class().select()[0].data['0_display'] == 'hello, world'

    formdef.data_class().wipe()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        assert 'disabled' in resp.form['f0$element1'].attrs
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [
        fields.ItemsField(id='0', label='string', type='items', data_source=ds, display_disabled_items=False)
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://remote.example.net/json',
            json={'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]},
        )
        resp = get_app(pub).get('/test/')
        assert 'f0$element1' not in resp.form.fields
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'


def test_item_field_autocomplete_json_source(http_requests, pub, error_email, emails):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    with responses.RequestsMock() as rsps:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        rsps.get('http://remote.example.net/json', json=data)
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert resp.form['f0'].value == '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit')  # -> validation page
        resp = resp.form.submit('submit')  # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][1]

    # check hint is displayed within
    formdef.fields[0].hint = 'help text'
    formdef.store()
    with responses.RequestsMock() as rsps:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        rsps.get('http://remote.example.net/json', json=data)
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert 'data-hint="help text"' in resp.text
        assert resp.form['f0'].value == ''

    formdef.fields[0].hint = ''
    formdef.store()

    # check with possibility of remote query
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    formdef.data_class().wipe()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp2 = app.get(select2_url + '?q=hell')
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json?q=hell'
        assert resp2.json == data

        # check unauthorized access
        resp2 = get_app(pub).get(select2_url + '?q=hell', status=403)

    # check error handling in autocomplete endpoint
    formdef.data_class().wipe()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        rsps.get('http://remote.example.net/json', body=ConnectionError('...'))
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

        assert emails.count() == 0
        resp2 = app.get(select2_url + '?q=hell')
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json?q=hell'
        assert resp2.json == {'data': [], 'err': '1'}
        assert emails.count() == 0

        data_source.notify_on_errors = True
        data_source.record_on_errors = True
        data_source.store()
        resp2 = app.get(select2_url + '?q=hell')
        assert emails.count() == 1
        assert emails.get_latest('subject') == '[ERROR] [DATASOURCE] Error loading JSON data source (...)'
        assert pub.loggederror_class.count() == 1
        logged_error = pub.loggederror_class.select()[0]
        assert logged_error.workflow_id is None
        assert logged_error.summary == '[DATASOURCE] Error loading JSON data source (...)'

        data_source.notify_on_errors = False
        data_source.store()

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp = resp.form.submit('submit')  # -> validation page
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json?id=1'
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp = resp.form.submit('submit')  # -> submit
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json?id=1'
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # same thing with numeric identifiers
    formdef.data_class().wipe()
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json-numeric-id'}
    data_source.store()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json-numeric-id', json=data)
        resp2 = app.get(select2_url + '?q=hell')
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json-numeric-id?q=hell'
        assert resp2.json == data

        # check unauthorized access
        resp2 = get_app(pub).get(select2_url + '?q=hell', status=403)

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json-numeric-id', json=data)
        resp = resp.form.submit('submit')  # -> validation page
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json-numeric-id?id=1'
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': 1, 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json-numeric-id', json=data)
        resp = resp.form.submit('submit')  # -> submit
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url == 'http://remote.example.net/json-numeric-id?id=1'
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # same thing with signed URLs
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.store()
    formdef.data_class().wipe()
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        fd.write(
            '''\
[wscall-secrets]
remote.example.net = 1234
'''
        )

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp2 = app.get(select2_url + '?q=hell')
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url.startswith(
            'http://remote.example.net/json?q=hell&orig=example.net&'
        )
        assert resp2.json == data

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp = resp.form.submit('submit')  # -> validation page
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url.startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with responses.RequestsMock() as rsps:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        rsps.get('http://remote.example.net/json', json=data)
        resp = resp.form.submit('submit')  # -> submit
        assert len(rsps.calls) == 1
        assert rsps.calls[-1].request.url.startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # check with optional field
    formdef.data_class().wipe()
    formdef.fields[0].required = False
    formdef.store()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        data = {
            'data': [
                {'id': '1', 'text': 'hello', 'extra': 'foo'},
                {'id': '2', 'text': 'world', 'extra': 'bar'},
            ]
        }
        rsps.get('http://remote.example.net/json', json=data)
        resp = app.get('/test/')
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        rsps.reset()

    resp = resp.form.submit('submit')  # -> validation page
    assert resp.form['f0'].value == ''
    assert resp.form['f0_label'].value == ''

    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] is None

    # check there's no crash if url is empty
    data_source.data_source = {'type': 'json', 'value': '{% if 0 %}http://remote.example.net/json{% endif %}'}
    data_source.store()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with responses.RequestsMock() as rsps:
        resp2 = app.get(select2_url + '?q=hell', status=403)
        assert len(rsps.calls) == 0


def test_item_field_autocomplete_jsonp_source(http_requests, pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        resp = app.get('/test/')
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        assert select2_url == 'http://remote.example.net/jsonp'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = [Hidden(form=resp.form, tag='input', name='f0_display', pos=10)]
    resp.form.field_order.append(('f0_display', resp.form.fields['f0_display'][0]))
    resp.form['f0'].force_value('1')
    resp.form['f0_display'].force_value('hello')

    with responses.RequestsMock() as rsps:
        resp = resp.form.submit('submit')  # -> validation page
        assert len(rsps.calls) == 0
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with responses.RequestsMock() as rsps:
        resp = resp.form.submit('submit')  # -> submit
        assert len(rsps.calls) == 0
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        # no _structured data for pure jsonp sources
        assert '0_structured' not in formdef.data_class().select()[0].data


def test_item_field_autocomplete_cards_source(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source=ds,
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    select2_url = resp.pyquery('select').attr['data-select2-url']
    resp2 = app.get(select2_url + '?q=ba')
    assert [x['text'] for x in resp2.json['data']] == ['bar', 'baz']
    resp.form['f0'].force_value(str(resp2.json['data'][0]['id']))
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'


def test_item_field_autocomplete_ezt_variable_jsonp(http_requests, pub):
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(
            id='1',
            label='foo',
            type='item',
            varname='foo',
            data_source={'type': 'jsonp', 'value': '[site_url]/foo-jsonp'},
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(
            id='3',
            label='bar',
            type='item',
            data_source={'type': 'jsonp', 'value': '[site_url]/foo-jsonp?a=[form_var_foo_raw]'},
        ),
    ]
    formdef.store()

    app = get_app(pub)
    with responses.RequestsMock() as rsps:
        resp = app.get('/test/')
        assert len(rsps.calls) == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        assert select2_url == 'http://example.net/foo-jsonp'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f1_display'] = [Hidden(form=resp.form, tag='input', name='f1_display', pos=10)]
    resp.form.field_order.append(('f1_display', resp.form.fields['f1_display'][0]))
    resp.form['f1'].force_value('1')
    resp.form['f1_display'].force_value('hello')

    with responses.RequestsMock() as rsps:
        resp = resp.form.submit('submit')  # -> 2nd page
        assert len(rsps.calls) == 0
        assert resp.pyquery('select').attr['data-select2-url'] == 'http://example.net/foo-jsonp?a=1'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f3_display'] = [Hidden(form=resp.form, tag='input', name='f3_display', pos=10)]
    resp.form.field_order.append(('f3_display', resp.form.fields['f3_display'][0]))
    resp.form['f3'].force_value('2')
    resp.form['f3_display'].force_value('hello2')

    with responses.RequestsMock() as rsps:
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submit
        assert len(rsps.calls) == 0
        assert formdef.data_class().select()[0].data['1'] == '1'
        assert formdef.data_class().select()[0].data['1_display'] == 'hello'
        assert formdef.data_class().select()[0].data['3'] == '2'
        assert formdef.data_class().select()[0].data['3_display'] == 'hello2'
        # no _structured data for pure jsonp sources
        assert '1_structured' not in formdef.data_class().select()[0].data
        assert '3_structured' not in formdef.data_class().select()[0].data


def test_form_data_keywords(pub):
    formdef = create_formdef()
    formdef.keywords = 'hello,world'
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1


def test_logged_errors(pub):
    Workflow.wipe()
    workflow = Workflow.get_default_workflow()
    workflow.id = '12'
    st1 = workflow.possible_status[0]
    jump = st1.add_action('jump', id='_jump', prepend=True)
    jump.id = '_jump'
    jump.status = 'rejected'
    jump.condition = {'type': 'python', 'value': '1//0'}  # ZeroDivisionError
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.id = '34'
    formdef.workflow = workflow
    formdef.name = 'test'
    formdef.confirmation = False
    formdef.fields = []
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert pub.loggederror_class.count() == 1

    # new expression, but raise the same exception (ZeroDivisionError),
    # just update the created logged error
    jump.condition = {'type': 'python', 'value': '2//0'}
    workflow.store()
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert pub.loggederror_class.count() == 1

    error = list(
        pub.loggederror_class.get_with_indexed_value(
            'tech_id',
            '34-12-just_submitted-_jump-failed-to-evaluate-condition-ZeroDivisionError-integer-division-or-modulo-by-zero',
        )
    )[0]
    assert error.occurences_count == 2
    assert error.expression == '2//0'

    assert len(list(pub.loggederror_class.get_with_indexed_value('formdef_id', '34'))) == 1
    assert len(list(pub.loggederror_class.get_with_indexed_value('formdef_id', 'X'))) == 0

    assert len(list(pub.loggederror_class.get_with_indexed_value('workflow_id', '12'))) == 1
    assert len(list(pub.loggederror_class.get_with_indexed_value('workflow_id', 'X'))) == 0


def test_resubmit(pub):
    user = create_user(pub)

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='string', varname='toto')]
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'form title bis'
    formdef2.enable_tracking_codes = True
    formdef2.fields = [
        fields.StringField(id='1', label='string', varname='titi'),
        fields.StringField(id='2', label='string', varname='toto'),
    ]
    formdef2.store()

    wf = Workflow(name='resubmit')
    st1 = wf.add_status('Status1')
    st2 = wf.add_status('Status2')

    resubmit = st1.add_action('resubmit', id='_resubmit')
    resubmit.by = ['_submitter']
    resubmit.formdef_slug = formdef2.url_name

    jump = st1.add_action('jumponsubmit', id='_jump')
    jump.status = st2.id

    register_comment = st2.add_action('register-comment', id='_register')
    register_comment.comment = '<p><a href="[resubmit_formdata_draft_url]">new draft</a></p>'

    wf.store()

    formdef.workflow_id = wf.id
    formdef.store()

    formdef2.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.data = {'1': 'XXX'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'XXX'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'XXX'

    # anonymous
    formdef2.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/form-title/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'foo'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'foo'


def test_form_custom_select_template(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar', 'Baz']
        )
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' not in resp.text
    formdef.fields[0].extra_css_class = 'template-test'
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' in resp.text
    # make sure request is available in context
    assert '<!-- backoffice: False -->' in resp.text
    assert '<!-- backoffice compat: False -->' in resp.text

    # test for substitution variables being available
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/')
    assert 'substitution variable: http://remote.example.net/' in resp.text


def test_form_status_appearance_keywords(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='select', type='item', required=True, varname='foo', items=['Foo', 'Bar', 'Baz']
        )
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    root = PublicFormStatusPage(formdef, formdata, register_workflow_subdirs=False)
    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == root.status_templates

    formdef.appearance_keywords = 'foobar plop'
    formdef.store()

    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == [
        'wcs/front/appearance-foobar/formdata_status.html',
        'wcs/front/appearance-plop/formdata_status.html',
        'wcs/front/formdata_status.html',
        'wcs/appearance-foobar/formdata_status.html',
        'wcs/appearance-plop/formdata_status.html',
        'wcs/formdata_status.html',
    ]

    resp = get_app(pub).get('/test/')
    assert 'class="quixote foobar plop"' in resp.text


def test_user_global_action(pub):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.add_action('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.add_action('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert 'button-action-1' not in resp.text

    trigger.roles = ['_submitter']
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_user_global_action_same_status_store(pub):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'

    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string'),
    ]
    action = workflow.add_global_action('FOOBAR')
    jump = action.add_action('jump')
    jump.status = 'new'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    new_status = workflow.possible_status[1]

    setbo = new_status.add_action('set-backoffice-fields', prepend=True)
    setbo.fields = [{'field_id': 'bo1', 'value': '123'}]

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['bo1'] == '123'

    # change global action
    setbo.fields = [{'field_id': 'bo1', 'value': '321'}]
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')  # click global action

    # check status actions are rerun
    resp = app.get(formdata.get_url())
    assert formdef.data_class().get(formdata.id).status == 'wf-new'
    assert formdef.data_class().get(formdata.id).data['bo1'] == '321'


def test_anonymous_user_global_action(pub):
    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.add_action('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.add_action('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.enable_tracking_codes = True
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = get_app(pub)
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    app.cookiejar.clear()

    resp = app.get('/')
    resp.forms[0]['code'] = formdata.tracking_code
    resp = resp.forms[0].submit().follow().follow()

    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_condition_on_action(pub, emails):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.id = '2'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test condition on action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form (test condition on action)')  # no receiver
    assert emails.get('New form2 (test condition on action)')  # submitter

    emails.empty()

    workflow.possible_status[0].items[1].condition = {'type': 'python', 'value': 'False'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with a condition on field data
    formdef.fields = [fields.StringField(id='0', label='string', varname='foobar')]
    formdef.store()
    workflow.possible_status[0].items[1].condition = {'type': 'django', 'value': 'form_var_foobar'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = ''
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with condition evaluating positively
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = 'toto'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert emails.get('New form2 (test condition on action)')


def test_user_global_action_along_form(pub):
    # check it's possible to click on a global action button even if there's a
    # form with required fields.
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.add_action('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.add_action('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    status = workflow.get_status('new')
    display_form = status.add_action('form', id='_x')
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.StringField(id='1', label='blah', type='string', required=True))

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert resp.form['fxxx_1'].attrs['aria-required'] == 'true'
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'


def test_email_actions(pub, emails):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.possible_status[0].items[
        1
    ].body = 'Hello; {% action_button "do-accept" label="Accepté!" %} Adiós.'
    workflow.possible_status[1].items[1].identifier = 'do-accept'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test email action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    assert '/actions/' in action_url
    if docutils:
        assert len(email_data['payloads']) == 2
        assert action_url in force_text(email_data['payloads'][1])

    app = get_app(pub)
    resp = app.get(action_url)
    assert 'Accepté!' in resp.text
    resp = resp.form.submit()
    assert 'The action has been confirmed.' in resp.text
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'wf-accepted'
    assert formdata.evolution[-1].parts[-1].event == 'email-button'

    # action token has been used, it will now return a custom 404
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # check against independently changed status, it should also return a
    # custom 404.
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    formdata = formdef.data_class().select()[0]
    formdata.jump_status('rejected')
    app = get_app(pub)
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # check action link referencing a deleted formdata
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    formdata = formdef.data_class().select()[0]
    formdata.remove_self()
    app = get_app(pub)
    resp = app.get(action_url, status=404)
    assert 'This action link is no longer valid' in resp.text

    # two buttons on the same line, two urls
    workflow.possible_status[0].items[
        1
    ].body = '{% action_button "ok" label="OK" %}{% action_button "ko" label="KO" %} '
    workflow.store()
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    assert len(re.findall(r'http.*? ', email_data['payload'])) == 2

    # custom messages
    workflow.possible_status[0].items[
        1
    ].body = 'Hello {% action_button "do-accept" label="ok" message="FOOmessageBAR" done_message="FOOdoneBAR" %} bye.'
    workflow.store()
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    app = get_app(pub)
    resp = app.get(action_url)
    assert 'FOOmessageBAR' in resp.text
    resp = resp.form.submit()
    assert 'FOOdoneBAR' in resp.text


def test_card_email_actions(pub, emails):
    create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'

    workflow.possible_status[0].items[0].subject = None  # disable first mail
    workflow.possible_status[0].items[1].subject = 'New card'
    workflow.possible_status[0].items[1].body = 'XXX {% action_button "do-accept" label="Accepté!" %}'
    workflow.possible_status[0].items[1].to = ['test@example.net']  # force recipient
    workflow.possible_status[1].items[1].identifier = 'do-accept'
    workflow.store()

    carddef = CardDef()
    carddef.name = 'test email action'
    carddef.fields = []
    carddef.workflow_id = workflow.id
    carddef.workflow_roles = {'_receiver': 1}
    carddef.store()
    carddef.data_class().wipe()

    carddata = carddef.data_class()()
    carddata.just_created()
    carddata.store()
    carddata.perform_workflow()
    carddata.store()

    email_data = emails.get('New card')
    action_url = re.findall(r'\shttp.*\s', email_data['payload'])[0].strip()
    assert '/actions/' in action_url
    if docutils:
        assert len(email_data['payloads']) == 2
        assert action_url in force_text(email_data['payloads'][1])

    app = get_app(pub)
    resp = app.get(action_url)
    assert 'Accepté!' in resp.text
    resp = resp.form.submit()
    assert 'The action has been confirmed.' in resp.text
    assert carddef.data_class().count() == 1
    carddata = carddef.data_class().select()[0]
    assert carddata.status == 'wf-accepted'


def test_manager_public_access(pub):
    user, manager = create_user_and_admin(pub)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    manager.is_admin = False
    manager.roles = [role.id]
    manager.store()
    assert manager.can_go_in_backoffice()

    formdef = create_formdef()
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    # user access to own formdata
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to formdata
    app = login(get_app(pub), username='admin', password='admin')
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    # agent access to an unauthorized formdata
    formdef.workflow_roles = {'_receiver': None}
    formdef.store()
    resp = app.get(formdata.get_url(), status=403)

    # agent access via a tracking code (stays in frontoffice)
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.enable_tracking_codes = True
    formdef.store()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp = app.get('/code/%s/load' % code.id)
    resp = resp.follow()  # -> /test/1/
    assert 'backoffice' not in resp.request.path
    assert 'The form has been recorded' in resp.text

    # authorized access but not backoffice access
    app = login(get_app(pub), username='admin', password='admin')  # reset session
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)  # check tracking code is no longer effective
    role.allows_backoffice_access = False
    role.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to own formdata (stays in frontoffice)
    formdata = formdef.data_class()()
    formdata.user_id = manager.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text


def test_form_and_category_same_slug(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.store()

    # check we get to the form, not the category
    resp = get_app(pub).get('/foobar/')
    assert resp.form


def test_field_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(
            type='string',
            id='1',
            label='Bar',
            size='40',
            required=True,
            condition={'type': 'django', 'value': '1'},
        ),
        fields.StringField(
            type='string',
            id='2',
            label='Foo',
            size='40',
            required=True,
            condition={'type': 'django', 'value': '0'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' not in resp.form.fields
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' not in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' not in resp.text


def test_field_unicode_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar == "éléphant"'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_contains_in_list(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar in "éléphant"|split'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_contains_in_string(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40', required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(
            type='string',
            id='4',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': '"éléphant" in form_var_bar'},
        ),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'f4' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields


def test_field_unicode_condition_in_array(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemsField(
            id='1',
            label='items',
            type='items',
            required=True,
            varname='foo',
            items=['Pomme', 'Poire', 'Pêche', 'Abricot'],
        ),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            type='string',
            id='3',
            label='Baz',
            size='40',
            required=True,
            varname='baz',
            condition={'type': 'django', 'value': '"Pêche" in form_var_foo'},
        ),
        fields.CommentField(type='comment', id='4', label='{{form_var_foo}}'),
        fields.CommentField(
            type='comment', id='5', label='{% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'
        ),
    ]

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    display1 = st1.add_action('displaymsg')
    display1.message = 'Message {% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'
    display1.to = []
    workflow.store()

    formdef.workflow = workflow
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp = resp.form.submit('submit')
    assert 'f3' not in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'f3' in resp.form.fields  # check it's ok in field condition
    resp.form['f3'] = 'hop'
    assert '>Poire, Pêche<' in resp.text  # check it's displayed correctly
    assert 'CHECK OK' in resp.text  # check it's ok in template condition
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert '<p>Message CHECK OK</p>' in resp.text  # check it's ok in workflow template


def test_form_edit_and_backoffice_field_change(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='foo'),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string', varname='plop'),
    ]
    st1 = workflow.add_status('Status1', 'st1')
    setbo = st1.add_action('set-backoffice-fields')
    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_foo'}]
    setbo2 = st1.add_action('set-backoffice-fields')
    setbo2.fields = [{'field_id': 'bo1', 'value': '="foo" + form_var_plop'}]
    jump = st1.add_action('jump')
    jump.status = 'st2'

    st2 = workflow.add_status('Status2', 'st2')

    editable = st2.add_action('editable', id='_editable')
    editable.by = ['_submitter']
    editable.status = st1.id
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submitted
    assert 'The form has been recorded' in resp.text

    data_id = formdef.data_class().select()[0].id
    assert formdef.data_class().get(data_id).data['bo1'] == 'foobar'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % data_id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable')
    resp = resp.follow()
    assert resp.form['f1'].value == 'bar'
    resp.form['f1'].value = 'baz'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit').follow()  # -> saved

    assert formdef.data_class().get(data_id).data['bo1'] == 'foobaz'


def test_backoffice_fields_just_after_conditional_form_submit(pub):
    """
    simulate selection of a structured list via condition on form,
    followed by an evaluation on workflow in order to get structured value
    from the selected list.
    ie: test unfeed on ConditionVars
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='first text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='first more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = st1.add_action('set-backoffice-fields')
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2', 'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'},
    ]
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more': 'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more': 'moreB1'}, {'id': '2', 'text': 'B2', 'more': 'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'], type='item', label='list to choice'),
        fields.ItemField(
            id='2',
            varname='listA',
            type='item',
            label='list A',
            data_source={'type': 'formula', 'value': str(items_A)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "A"'},
        ),
        fields.ItemField(
            id='3',
            varname='listB',
            type='item',
            label='list B',
            data_source={'type': 'formula', 'value': str(items_B)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "B"'},
        ),
    ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user_and_admin(pub)
    resp = get_app(pub).get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'


def test_backoffice_fields_just_after_conditional_form_edit_action(pub):
    """
    test unfeed on ConditionVars within edit context
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='both text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='both more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = st1.add_action('set-backoffice-fields')
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2', 'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'},
    ]
    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter']
    editable.status = st1.id
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more': 'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more': 'moreB1'}, {'id': '2', 'text': 'B2', 'more': 'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'], type='item', label='list to choice'),
        fields.ItemField(
            id='2',
            varname='listA',
            type='item',
            label='list A',
            data_source={'type': 'formula', 'value': str(items_A)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "A"'},
        ),
        fields.ItemField(
            id='3',
            varname='listB',
            type='item',
            label='list B',
            data_source={'type': 'formula', 'value': str(items_B)},
            condition={'type': 'python', 'value': 'form_var_choice_raw == "B"'},
        ),
    ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    # check unfeed on FormPage::submitted()
    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % formdata.id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable').follow()
    assert resp.form['f1'].value == 'B'
    resp.form['f1'].value = 'A'
    resp = resp.form.submit('submit').follow()  # -> saved
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'A'
    assert formdata.data['2'] == '1'
    assert formdata.data.get('3') is None

    # check unfeed on FormPage::submitted_existing()
    assert formdata.data['bo1'] == 'A1 vs None'
    assert formdata.data['bo2'] == 'moreA1 vs '


def test_backoffice_fields_set_from_live(pub):
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='first text', type='string', varname='both_text'),
        fields.StringField(id='bo2', label='first more', type='string', varname='both_more'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = st1.add_action('set-backoffice-fields')
    setbo.fields = [
        {'field_id': 'bo1', 'value': '{{ form_var.foo.attr }}'},
        {'field_id': 'bo2', 'value': '{{ form_var.foo.live.var.attr }}'},
    ]
    workflow.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1', label='string', type='item', varname='foo', data_source=ds, display_disabled_items=True
        )
    ]
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user_and_admin(pub)
    resp = get_app(pub).get('/test/')

    resp.form['f1'].value = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == '2'
    assert formdata.data['bo1'] == 'attr1'
    assert formdata.data['bo2'] == 'attr1'


def test_form_recall_draft(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' not in resp.text

    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s/"' % draft.id in resp.text

    draft2 = formdef.data_class()()
    draft2.user_id = user.id
    draft2.status = 'draft'
    draft2.data = {}
    draft2.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s/"' % draft.id in resp.text
    assert 'href="%s/"' % draft2.id in resp.text


def test_frontoffice_workflow_form_with_conditions(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(id='2', label='Test2', varname='str2', type='string', required=True),
    ]

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    # check with static condition
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': '0'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' not in resp.form.fields

    # check condition based on formdata
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'form_var_plop'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'form_var_plop != "xxx"'},
        ),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields
    assert 'fblah_2' in resp.form.fields

    for variable_name in (
        'blah_var_str',
        'form_workflow_data_blah_var_str',
        'form_workflow_form_blah_var_str',
    ):
        # check with live conditions
        display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(
                id='2',
                label='Test2',
                varname='str2',
                type='string',
                required=True,
                condition={'type': 'django', 'value': '%s == "xxx"' % variable_name},
            ),
        ]
        wf.store()

        resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
        assert 'fblah_1' in resp.form.fields
        assert 'fblah_2' in resp.form.fields
        assert resp.html.find('div', {'data-field-id': 'blah_1'}).attrs['data-live-source'] == 'true'
        assert resp.html.find('div', {'data-field-id': 'blah_2'}).attrs.get('style') == 'display: none'
        live_url = resp.html.find('form').attrs['data-live-url']
        resp.form['fblah_1'] = ''
        live_resp = app.post(live_url, params=resp.form.submit_fields())
        assert live_resp.json['result']['blah_1']['visible']
        assert not live_resp.json['result']['blah_2']['visible']

        resp.form['fblah_1'] = 'xxx'
        live_resp = app.post(live_url, params=resp.form.submit_fields())
        assert live_resp.json['result']['blah_1']['visible']
        assert live_resp.json['result']['blah_2']['visible']

    # check submit doesn't work
    resp = resp.form.submit('submit')
    assert 'There were errors processing your form.' in resp.text

    resp.form['fblah_1'] = 'xxx2'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_1']['visible']
    assert not live_resp.json['result']['blah_2']['visible']

    # check submit does work when second field is hidden
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().get(formdata.id).workflow_data == {
        'blah_var_str': 'xxx2',
        'blah_var_str2': None,
    }


def test_frontoffice_workflow_form_with_dynamic_comment(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.CommentField(id='2', label='value is {{blah_var_str}}', type='comment'),
    ]

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' in resp.form.fields

    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['fblah_1'] = 'test'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['blah_2']['visible']
    assert live_resp.json['result']['blah_2']['content'] == '<p>value is test</p>'


def test_frontoffice_workflow_form_with_dynamic_list(pub):
    user = create_user(pub)
    wf = Workflow('dynamic list in workflow')
    status = wf.add_status('st1')
    status2 = wf.add_status('st2')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.ItemField(id='1', label='Test', varname='foo', type='item', items=['10', '20']),
        fields.ItemField(
            id='2',
            label='Test2',
            type='item',
            varname='item2',
            data_source={'type': 'json', 'value': 'http://example.org/{{form_workflow_form_blah_var_foo}}'},
        ),
    ]
    jump1 = status.add_action('choice', id='_jump')
    jump1.label = 'Jump'
    jump1.by = ['_submitter']
    jump1.status = status2.id

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))

    with responses.RequestsMock() as rsps:
        rsps.get(
            'http://example.org/10',
            json={
                'data': [
                    {'id': '1', 'text': 'hello', 'extra': 'foo'},
                    {'id': '2', 'text': 'world', 'extra': 'bar'},
                ]
            },
        )
        rsps.get(
            'http://example.org/20',
            json={
                'data': [
                    {'id': '11', 'text': 'hello2', 'extra': 'foo'},
                    {'id': '21', 'text': 'world2', 'extra': 'bar'},
                ]
            },
        )

        live_url = resp.html.find('form').attrs['data-live-url']
        live_resp = app.post(live_url + '?modified_field_id=init', params=resp.form.submit_fields())
        assert [x['id'] for x in live_resp.json['result']['blah_2']['items']] == ['1', '2']
        resp.form['fblah_1'] = '20'
        live_resp = app.post(live_url + '?modified_field_id=blah_1', params=resp.form.submit_fields())
        assert [x['id'] for x in live_resp.json['result']['blah_2']['items']] == ['11', '21']

        resp.form['fblah_2'].force_value('11')
        resp = resp.form.submit('submit').follow()
        assert 'Technical error, please try again' not in resp.text
        formdata.refresh_from_storage()
        pub.substitutions.feed(formdata)
        context = pub.substitutions.get_context_variables(mode='lazy')
        assert context['form_workflow_form_blah_var_item2'] == 'hello2'
        assert context['form_workflow_data_blah_var_item2'] == 'hello2'


@pytest.mark.parametrize('button_position', ['before', 'after'])
def test_frontoffice_workflow_form_and_other_button(pub, button_position):
    user = create_user(pub)
    wf = Workflow('form and other button')
    status = wf.add_status('st1')
    status2 = wf.add_status('st2')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='foo', type='string', required=True),
        fields.StringField(id='2', label='Test2', varname='foo2', type='string', required=True),
    ]
    jump1 = status.add_action('choice', id='_jump', prepend=bool(button_position == 'before'))
    jump1.label = 'Jump'
    jump1.by = ['_submitter']
    jump1.status = status2.id

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'] = 'a'
    resp.form['fblah_2'] = 'b'
    resp = resp.form.submit('submit')
    formdata.refresh_from_storage()
    pub.substitutions.reset()
    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    assert 'form_workflow_form_blah_var_foo' in context
    assert context['form_workflow_form_blah_var_foo'] == 'a'
    assert context['form_workflow_data_blah_var_foo'] == 'a'

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'] = 'a'
    resp.form['fblah_2'] = 'b'
    resp = resp.form.submit('button_jump')
    formdata.refresh_from_storage()
    pub.substitutions.reset()
    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    # check workflow form data is saved if button comes after form action
    if button_position == 'before':
        assert 'form_workflow_form_blah_var_foo' not in context
    else:
        assert 'form_workflow_form_blah_var_foo' in context
    # but legacy behaviout it leaks into workflow_data :/
    assert context['form_workflow_data_blah_var_foo'] == 'a'

    # check it also happens with invalid/partial form
    jump1.ignore_form_errors = True
    wf.store()
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'] = 'a'
    resp.form['fblah_2'] = ''
    resp = resp.form.submit('button_jump')
    formdata.refresh_from_storage()
    pub.substitutions.reset()
    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    # check workflow form data is not saved (good)
    assert 'form_workflow_form_blah_var_foo' not in context
    # but legacy behaviout it leaks into workflow_data :/
    assert context['form_workflow_data_blah_var_foo'] == 'a'


def test_frontoffice_workflow_form_with_impossible_condition(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(
            id='1',
            label='Test',
            varname='str',
            type='string',
            condition={'type': 'django', 'value': '0 == 1'},
        ),
        fields.StringField(
            id='2',
            label='Test2',
            type='string',
            condition={'type': 'django', 'value': 'blah_var_str == "toto"'},
        ),
    ]

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'fblah_1' not in resp.form.fields
    assert 'fblah_2' not in resp.form.fields


def test_frontoffice_workflow_form_with_attachment_and_python_datasource(pub):
    # case from https://dev.entrouvert.org/issues/60254
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]),
    }
    data_source.store()

    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []

    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.ItemField(
            id='2', label='Test List', type='item', varname='bar', data_source={'type': data_source.slug}
        ),
    ]

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.status = 'wf-new'
    formdata.data = {}
    formdata.evolution[-1].parts = [
        AttachmentEvolutionPart('hello.txt', fp=io.BytesIO(b'hello world'), varname='testfile')
    ]
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'].value = 'blah'
    resp.form['fblah_2'].value = '1'
    resp = resp.form.submit('submit')


@responses.activate
def test_frontoffice_workflow_form_with_disappearing_option(pub, monkeypatch):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://www.example.net/plop'}
    data_source.id_parameter = 'id'
    data_source.store()

    responses.get(
        'http://www.example.net/plop', json={'data': [{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]}
    )

    user = create_user(pub)
    wf = Workflow(name='select')
    st1 = wf.add_status('st1')
    st2 = wf.add_status('st2')
    display_form = st1.add_action('form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.ItemField(id='1', label='Test', varname='foo', type='item', data_source={'type': 'foobar'}),
    ]
    jump = st1.add_action('jumponsubmit')
    jump.status = st2.id

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    # normal case, status changes and data is recorded
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'] = '1'
    resp = resp.form.submit('submit').follow()
    formdata.refresh_from_storage()
    assert formdata.status == 'wf-%s' % st2.id
    assert formdata.workflow_data['blah_var_foo_raw'] == '1'
    assert formdata.workflow_data['blah_var_foo'] == 'un'

    # simulate an option disappearing during submit
    monkeypatch.setattr(
        NamedDataSource,
        'get_value_by_id',
        lambda *args: None,
    )
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1'] = '1'

    resp = resp.form.submit('submit')
    assert 'Technical error, please try again' in resp.text
    formdata.refresh_from_storage()
    assert formdata.status == 'wf-%s' % st1.id
    assert not formdata.workflow_data


def test_choice_button_ignore_form_errors(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.roles = [logged_users_role().id]
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    st2 = wf.add_status('Status2', 'st2')

    commentable = st1.add_action('commentable', id='_commentable')
    commentable.by = [logged_users_role().id]
    commentable.required = True

    choice = st1.add_action('choice', id='_x1')
    choice.label = 'Submit'
    choice.by = [logged_users_role().id]
    choice.status = st2.id

    choice2 = st1.add_action('choice', id='_x2')
    choice2.label = 'Submit no check'
    choice2.by = [logged_users_role().id]
    choice2.status = st2.id
    choice2.ignore_form_errors = True

    wf.store()

    formdef.workflow = wf
    formdef.store()

    # no comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x1')
    assert 'There were errors processing your form.' in resp.text

    # comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('button_x1').follow()
    assert resp.pyquery('.comment').text() == 'plop'
    assert '<span class="status">Status2' in resp.text

    # no comment but no check
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x2').follow()
    assert '<span class="status">Status2' in resp.text


def test_form_comment_is_hidden_attribute(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string 1', varname='choice1'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(
            id='3',
            label='string 2',
            varname='choice2',
            condition={'type': 'django', 'value': 'form_var_choice1 == "1"'},
        ),
        fields.CommentField(
            id='5',
            label='this should not be displayed',
            type='comment',
            condition={'type': 'django', 'value': 'False and form_var_choice2 == "???"'},
        ),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = '1'
    resp = resp.forms[0].submit('submit')
    comment = re.compile('.*comment-field.*"')
    assert resp.html.find('div', {'data-field-id': '5'})
    assert 'style="display: none"' in comment.search(resp.forms[0].text).group(0)
    resp = resp.forms[0].submit('previous')
    resp.forms[0]['f1'] = '2'
    resp = resp.forms[0].submit('submit')
    assert not resp.html.find('div', {'data-field-id': '5'})


@pytest.fixture
def create_formdata(pub):
    FormDef.wipe()

    data = [
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'},
    ]
    ds = {
        'type': 'formula',
        'value': repr(data),
    }
    source_formdef = FormDef()
    source_formdef.name = 'source form'
    source_formdef.fields = [
        fields.StringField(id='0', label='string', varname='toto_string'),
        fields.FileField(id='1', label='file', type='file', varname='toto_file'),
        fields.ItemField(
            id='2', label='item', required=False, data_source=ds, varname='toto_item', hint='hint'
        ),
    ]
    source_formdef.store()

    target_formdef = FormDef()
    target_formdef.name = 'target form'
    target_formdef.enable_tracking_codes = True
    target_formdef.fields = [
        fields.StringField(id='0', label='string', varname='foo_string'),
        fields.FileField(id='1', label='file', type='file', varname='foo_file'),
        fields.ItemField(id='2', label='item', data_source=ds, varname='foo_item'),
    ]
    target_formdef.store()
    wf = Workflow(name='create-formdata')

    st1 = wf.add_status('New')
    st2 = wf.add_status('Resubmit')

    jump = st1.add_action('choice', id='_resubmit')
    jump.label = 'Resubmit'
    jump.by = ['_submitter']
    jump.status = st2.id

    create_formdata = st2.add_action('create_formdata', id='_create_formdata')
    create_formdata.varname = 'resubmitted'
    create_formdata.draft = True
    create_formdata.formdef_slug = target_formdef.url_name
    create_formdata.mappings = [
        Mapping(field_id='0', expression='=form_var_toto_string'),
        Mapping(field_id='1', expression='=form_var_toto_file_raw'),
        Mapping(field_id='2', expression='=form_var_toto_item_raw'),
    ]

    redirect = st2.add_action('redirect_to_url', id='_redirect')
    redirect.url = '{{ form_links_resubmitted.form_url }}'

    display = st2.add_action('displaymsg', id='_display')
    display.message = '''<div class="linked">{% if form_links_resubmitted %}
<p>Linked status: <span class="status">{{ form_links_resubmitted.form_status }}</span></p>
<p>Target formdata field: <span class="foo_string">{{ form_links_resubmitted.form_var_foo_string }}</span></p>
{% endif %}</div>'''
    display.to = []

    wf.store()
    source_formdef.workflow_id = wf.id
    source_formdef.store()
    return locals()


def test_create_formdata_anonymous_draft(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'

    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'draft'
    assert target_formdata.submission_context == {
        'orig_object_type': 'formdef',
        'orig_formdata_id': str(create_formdata['source_formdef'].data_class().select()[0].id),
        'orig_formdef_id': str(create_formdata['source_formdef'].id),
    }

    resp = resp.follow()
    resp = resp.follow()
    assert 'zob' in resp
    assert resp.click('test.txt').text == 'foobar'
    resp = resp.forms[1].submit('submit')  # -> validation
    resp = resp.forms[1].submit('submit')  # -> submission
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('2') == '2'
    assert target_formdata.data.get('2_display') == 'deux'
    assert target_formdata.data.get('2_structured') == {'text': 'deux', 'id': '2', 'more': 'bar'}
    assert target_formdata.status == 'wf-new'

    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
    resp = app.get(source_formdata.get_url())
    pq = resp.pyquery.remove_namespaces()
    assert pq('.linked .status').text() == 'New'
    assert pq('.linked .foo_string').text() == 'zob'


def test_create_formdata_anonymous_submitted(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    # submit directly
    create_formdata['wf'].get_status('2').items[0].draft = False
    create_formdata['wf'].store()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'

    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'wf-new'
    assert target_formdata.submission_context == {
        'orig_object_type': 'formdef',
        'orig_formdata_id': str(create_formdata['source_formdef'].data_class().select()[0].id),
        'orig_formdef_id': str(create_formdata['source_formdef'].id),
    }

    resp = resp.follow()
    assert 'New' in resp
    assert 'zob' in resp
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.data.get('1').get_content() == b'foobar'
    assert target_formdata.status == 'wf-new'

    source_formdata = create_formdata['source_formdef'].data_class().select()[0]
    resp = app.get(source_formdata.get_url())
    pq = resp.pyquery.remove_namespaces()
    assert pq('.linked .status').text() == 'New'
    assert pq('.linked .foo_string').text() == 'zob'


def test_create_formdata_empty_item_ds_with_id_parameter(pub, create_formdata):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/json',
    }
    data_source.id_parameter = 'id'
    data_source.store()
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()
    create_formdata['source_formdef'].fields[2].data_source = {'type': 'foobar'}
    create_formdata['source_formdef'].store()
    create_formdata['target_formdef'].fields[2].data_source = {'type': 'foobar'}
    create_formdata['target_formdef'].store()

    with responses.RequestsMock() as rsps:
        data = {'data': create_formdata['data']}
        rsps.get('http://remote.example.net/json', json=data)

        app = get_app(create_formdata['pub'])
        resp = app.get('/source-form/')
        resp.form['f0'] = 'zob'
        resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
        resp = resp.form.submit('submit')  # -> validation
        resp = resp.form.submit('submit')  # -> submission
        resp = resp.follow()
        assert create_formdata['target_formdef'].data_class().count() == 0
        assert pub.loggederror_class.count() == 0
        resp = resp.form.submit('button_resubmit')
        assert pub.loggederror_class.count() == 0


def test_create_formdata_locked_prefill_parent(create_formdata):
    create_formdata['source_formdef'].data_class().wipe()
    create_formdata['target_formdef'].data_class().wipe()

    target_formdef = create_formdata['target_formdef']
    target_formdef.fields[0].prefill = {
        'type': 'string',
        'value': '{{form_parent_form_var_toto_string}}',
        'locked': True,
    }
    target_formdef.store()

    app = get_app(create_formdata['pub'])
    resp = app.get('/source-form/')
    resp.form['f0'] = 'zob'
    resp.form['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp.form['f2'] = '2'
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()
    assert create_formdata['target_formdef'].data_class().count() == 0
    resp = resp.form.submit('button_resubmit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data.get('0') == 'zob'
    assert target_formdata.status == 'draft'

    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'zob'
    assert resp.forms[1]['f0'].attrs['readonly']
    # try altering readonly field
    resp.forms[1]['f0'].value = 'xxx'
    resp = resp.forms[1].submit('submit')
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'zob'
    resp = resp.forms[1].submit('submit')
    resp = resp.forms[1].submit('submit')
    assert create_formdata['target_formdef'].data_class().count() == 1
    target_formdata = create_formdata['target_formdef'].data_class().select()[0]
    assert target_formdata.data['0'] == 'zob'


def test_js_libraries(pub):
    create_formdef()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' not in resp.text
    assert 'jquery.min.js' in resp.text
    assert 'qommon.forms.js' in resp.text

    pub.cfg['debug'] = {'debug_mode': True}
    pub.write_cfg()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' in resp.text
    assert 'jquery.min.js' not in resp.text
    assert 'qommon.forms.js' in resp.text

    pub.cfg['branding'] = {'included_js_libraries': ['jquery.js']}
    pub.write_cfg()
    resp = get_app(pub).get('/test/', status=200)
    assert 'jquery.js' not in resp.text
    assert 'jquery.min.js' not in resp.text
    assert 'qommon.forms.js' in resp.text


def test_after_submit_location(pub):
    create_user(pub)

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    commentable = st1.add_action('commentable', id='_commentable')
    commentable.by = [logged_users_role().id]
    commentable.required = True

    workflow.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')  # form page
    resp = resp.forms[0].submit('submit')  # confirmation page
    resp = resp.follow()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/test/1/#action-zone'
    resp = resp.follow()

    display = st1.add_action('displaymsg')
    display.message = 'message-to-all'
    display.to = []
    workflow.store()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('submit')
    assert resp.location == 'http://example.net/test/1/#'


def test_form_honeypot(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', required=False)]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f00'] = 'honey?'
    resp = resp.forms[0].submit('submit')
    assert 'Honey pot should be left untouched.' in resp


def test_structured_workflow_options(pub):
    create_user_and_admin(pub)

    workflow = Workflow(name='test')
    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
    data_source = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un', 'more': 'foo'}, {'id': '2', 'text': 'deux', 'more': 'bar'}]),
    }
    workflow.variables_formdef.fields = [
        fields.StringField(id='1', label='Test', type='string', varname='foo'),
        fields.ItemField(id='2', label='Test List', type='item', varname='bar', data_source=data_source),
        fields.ItemsField(id='3', label='Test Multi', type='items', varname='baz', data_source=data_source),
        fields.DateField(id='4', label='Date', type='date', varname='date'),
    ]
    st1 = workflow.add_status('Status1', 'st1')
    comment = st1.add_action('register-comment', id='_comment')
    comment.comment = 'Date option: {{ form_option_date }}'
    workflow.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(
            id='1',
            label='Test List',
            type='item',
            varname='bar',
            data_source={'type': 'formula', 'value': 'form_option_baz_structured'},
        ),
    ]
    formdef.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    # configure workflow options
    resp = login(get_app(pub), username='admin', password='admin').get('/backoffice/forms/%s/' % formdef.id)
    resp = resp.click('Options')
    resp.form['f1'].value = 'plop'
    resp.form['f2'].value = '1'
    resp.form['f3$element1'].checked = True
    resp.form['f4'].value = '2020-04-18'
    resp = resp.form.submit('submit')

    formdef = FormDef.get(formdef.id)
    assert formdef.workflow_options == {
        'foo': 'plop',
        'bar': '1',
        'bar_display': 'un',
        'bar_structured': {'id': '1', 'more': 'foo', 'text': 'un'},
        'baz': ['1'],
        'baz_display': 'un',
        'baz_structured': [{'id': '1', 'more': 'foo', 'text': 'un'}],
        'date': time.strptime('2020-04-18', '%Y-%m-%d'),
    }

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert resp.form['f1'].options == [('1', False, 'un')]
    resp = resp.form.submit('submit')  # form page
    resp = resp.form.submit('submit')  # confirmation page
    resp = resp.follow()

    formdata = formdef.data_class().select()[0]
    assert formdata.data == {
        '1': '1',
        '1_display': 'un',
        '1_structured': {'id': '1', 'text': 'un', 'more': 'foo'},
    }
    assert '2020-04-18' in formdata.evolution[0].parts[0].content


def test_exclude_self_condition(pub):
    FormDef.wipe()

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [
        fields.PageField(
            id='1',
            label='1st page',
            type='page',
            post_conditions=[
                {
                    'condition': {
                        'type': 'django',
                        'value': 'form_objects|filter_by:"foo"|filter_value:form_var_foo|exclude_self|count == 0',
                    },
                    'error_message': 'You shall not pass.',
                }
            ],
        ),
        fields.StringField(id='1', label='string', type='string', varname='foo'),
    ]

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter', '_receiver']
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = get_app(pub)
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp.text
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()

    # edit is ok
    resp = resp.form.submit('button_editable').follow()
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp

    # 2nd submission
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' in resp.text

    # submission with other value
    resp = app.get(formdef.get_url())
    resp.form['f1'] = 'other'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' not in resp.text
    resp = resp.form.submit('submit')  # -> submit
    resp = resp.follow()

    # edit is ok
    resp = resp.form.submit('button_editable').follow()
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')  # -> validation page
    assert 'You shall not pass.' in resp


def test_form_item_map_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'geojson',
        'value': 'http://remote.example.net/geojson',
    }
    data_source.id_property = 'id'
    data_source.label_template_property = '{{ text }}'
    data_source.cache_duration = '5'
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', label='map', display_mode='map'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url'] == ''
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-radio-name'] == 'f1$marker_id'

    formdef.fields[0].data_source = {'type': 'foobar'}
    formdef.store()
    resp = app.get('/test/')
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url'] == '/api/geojson/foobar'
    assert resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-radio-name'] == 'f1$marker_id'
    app.get('/api/geojson/wrong-foobar', status=404)
    resp_geojson = app.get('/api/geojson/foobar')
    assert len(resp_geojson.json['features']) == 2
    assert http_requests.count() == 1
    assert http_requests.get_last('url') == 'http://remote.example.net/geojson'
    resp_geojson = app.get('/api/geojson/foobar')
    assert http_requests.count() == 1  # cache was used
    assert len(resp_geojson.json['features']) == 2
    # simulate qommon.map.js that will create radio inputs
    resp.form.fields['f1$marker_id'] = [Radio(form=resp.form, tag='input', name='f1$marker_id', pos=5)]
    resp.form.fields['f1$marker_id'][0].options.append(('1', False, None))
    resp.form.fields['f1$marker_id'][0].options.append(('2', False, None))
    resp.form.fields['f1$marker_id'][0].optionPositions.append(5)
    resp.form.fields['f1$marker_id'][0].optionPositions.append(6)
    resp.form.field_order.append(('f1$marker_id', resp.form.fields['f1$marker_id'][0]))
    resp.form['f1$marker_id'].value = '1'  # click on marker
    resp.form['f1$latlng'] = '1;2'  # set via js
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    # selected option is displayed as readonly:
    assert resp.pyquery('input[type=text][value=foo][readonly]')
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp
    assert '<div class="value">foo</div>' in resp
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    formdata = formdef.data_class().get(data_id)
    assert formdata.data['1_structured']['geometry']['coordinates'] == [1, 2]


def test_form_item_dynamic_map_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'geojson',
        'value': 'http://remote.example.net/geojson?{{ form_var_test }}',
    }
    data_source.id_property = 'id'
    data_source.label_template_property = '{{ text }}'
    data_source.cache_duration = '5'
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='test'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.ItemField(id='3', label='map', display_mode='map', data_source={'type': 'foobar'}),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'plop'
    resp = resp.form.submit('submit')  # -> 2nd page
    markers_url = resp.pyquery('div[data-markers-radio-name]')[0].attrib['data-markers-url']
    assert markers_url.startswith('/api/geojson/')
    resp_geojson = app.get(markers_url)
    assert len(resp_geojson.json['features']) == 2
    assert http_requests.count() == 1
    assert http_requests.get_last('url') == 'http://remote.example.net/geojson?plop'
    resp_geojson = app.get(markers_url)
    assert http_requests.count() == 1  # cache was used
    assert len(resp_geojson.json['features']) == 2


def test_form_item_timetable_data_source(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/api/datetimes',
    }
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', label='datetime', display_mode='timetable', data_source={'type': 'foobar'}),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)

    with responses.RequestsMock() as rsps:
        data = {
            "data": [
                {"id": "1", "datetime": "2021-01-12 10:00:00", "text": "event 1"},
                {"id": "2", "datetime": "2021-01-13 10:20:00", "text": "event 2"},
                {"id": "3", "datetime": "2021-01-14 10:40:00", "text": "event 3"},
            ]
        }
        rsps.get('http://remote.example.net/api/datetimes', json=data)

        resp = app.get('/test/')
        assert 'data-date="2021-01-12"' in resp and 'data-time="10:00"' in resp
        assert 'data-date="2021-01-13"' in resp and 'data-time="10:20"' in resp
        assert 'data-date="2021-01-14"' in resp and 'data-time="10:40"' in resp
        resp.form['f1'] = '2'  # would happen via javascript
        resp = resp.form.submit('submit')
        resp = resp.form.submit('submit')

        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        formdata = formdef.data_class().get(data_id)
        assert formdata.data == {
            '1': '2',
            '1_display': 'event 2',
            '1_structured': {'id': '2', 'datetime': '2021-01-13 10:20:00', 'text': 'event 2'},
        }


def test_form_item_timetable_data_source_with_date_alignment(pub, http_requests):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'json',
        'value': 'http://remote.example.net/api/datetimes',
    }
    data_source.store()

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='1', label='page1', type='page'),
        fields.DateField(id='2', label='date', type='date', varname='date'),
        fields.PageField(id='3', label='page2', type='page'),
        fields.ItemField(
            id='4',
            label='datetime',
            display_mode='timetable',
            data_source={'type': 'foobar'},
            initial_date_alignment='{{ form_var_date }}',
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()
    app = get_app(pub)

    with responses.RequestsMock() as rsps:
        data = {
            "data": [
                {"id": "1", "datetime": "2021-01-12 10:00:00", "text": "event 1"},
                {"id": "2", "datetime": "2021-01-13 10:20:00", "text": "event 2"},
                {"id": "3", "datetime": "2021-01-14 10:40:00", "text": "event 3"},
            ]
        }
        rsps.get('http://remote.example.net/api/datetimes', json=data)

        resp = app.get('/test/')
        resp.form['f2'] = '2021-01-14'
        resp = resp.form.submit('submit')  # -> 2nd page

        assert 'var ALIGN_DATE = "2021-01-14";' in resp
        resp.form['f4'] = '2'  # would happen via javascript
        resp = resp.form.submit('submit')
        resp = resp.form.submit('submit')
        assert formdef.data_class().count() == 1


def test_file_prefill_on_edit(pub, http_requests):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file', varname='foo_file')]
    formdef.store()
    formdef.data_class().wipe()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('New', 'st1')
    st2 = workflow.add_status('CreateFormdata')

    editable = st1.add_action('editable', id='_editable')
    editable.by = ['_submitter', '_receiver']

    jump = st1.add_action('choice', id='_resubmit')
    jump.label = 'Resubmit'
    jump.by = ['_submitter']
    jump.status = st2.id

    create_formdata = st2.add_action('create_formdata', id='_create_formdata')
    create_formdata.formdef_slug = formdef.url_name
    create_formdata.mappings = [
        Mapping(field_id='0', expression='{{form_var_foo_file}}'),
    ]

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    upload = Upload('test.txt', b'foobar', 'text/plain')
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit

    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

    formdata = formdef.data_class().select()[0]
    formdata.user_id = user.id
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % formdata.id)
    assert 'button_editable-button' in resp.text

    # go to edition page
    resp = resp.form.submit('button_editable').follow()
    # file is "prefilled"
    assert 'test.txt' in resp.text

    # go back to form page and trigger formdata creation
    resp = app.get('/test/%s/' % formdata.id)
    resp = resp.form.submit('button_resubmit')
    assert resp.status == '303 See Other'
    resp = resp.follow()

    assert formdef.data_class().count() == 2
    new_formdata = formdef.data_class().select(lambda x: str(x.id) != str(formdata.id))[0]
    assert new_formdata.data['0'].orig_filename == 'test.txt'
    assert new_formdata.data['0'].get_content() == b'foobar'

    resp = app.get('/test/%s/' % new_formdata.id)
    assert 'button_editable-button' in resp.text
    # go to edition page
    resp = resp.form.submit('button_editable').follow()
    # file is "prefilled"
    assert 'test.txt' in resp.text
    # and persist after being saved again
    resp = resp.form.submit('submit').follow()
    assert '<span>test.txt</span>' in resp.text


def test_workflow_form_structured_data(pub):
    FormDef.wipe()
    Workflow.wipe()
    BlockDef.wipe()

    user = create_user(pub)

    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.StringField(id='123', required=True, label='Test', type='string', varname='test'),
    ]
    block.store()

    wf = Workflow(name='test')
    status = wf.add_status('New', 'st1')
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', varname='fooblock'),
    ]
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1$element0$f123'] = 'ABC'
    resp = resp.form.submit('submit').follow()

    resp.form['fblah_1$element0$f123'] = 'XYZ'
    resp = resp.form.submit('submit').follow()

    formdata.refresh_from_storage()
    assert formdata.workflow_data == {
        'blah_var_fooblock_raw': {'data': [{'123': 'XYZ'}], 'schema': {'123': 'string'}},
        'blah_var_fooblock': 'foobar',
    }

    substvars = CompatibilityNamesDict()
    substvars.update(formdata.get_substitution_variables())
    keys = substvars.get_flat_keys()
    for key in keys:
        # noqa pylint: disable=unused-variable
        var = substvars[key]  # check it doesn't raise, ignore the value

    assert substvars['form_workflow_form_blah_var_fooblock_var_test'] == 'XYZ'
    assert substvars['form_workflow_form_blah_0_var_fooblock_var_test'] == 'ABC'
    assert substvars['form_workflow_form_blah_1_var_fooblock_var_test'] == 'XYZ'

    # disable dumping in workflow_data
    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'disable-workflow-form-to-workflow-data', 'true')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1$element0$f123'] = 'ABC'
    resp = resp.form.submit('submit').follow()

    resp.form['fblah_1$element0$f123'] = 'XYZ'
    resp = resp.form.submit('submit').follow()

    formdata.refresh_from_storage()
    assert not formdata.workflow_data


def test_workflow_form_file_access(pub):
    FormDef.wipe()
    Workflow.wipe()
    BlockDef.wipe()

    user = create_user(pub)

    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.FileField(id='123', required=True, label='Test', type='file', varname='test'),
    ]
    block.store()

    wf = Workflow(name='test')
    status = wf.add_status('New', 'st1')
    next_status = wf.add_status('Next', 'st2')

    status.items = []
    display_form = status.add_action('form', id='_display_form')
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', varname='fooblock', max_items=3),
        fields.FileField(id='2', label='test2', type='file', varname='file'),
    ]

    jump = status.add_action('jumponsubmit', id='_jump')
    jump.status = next_status.id

    register_comment = next_status.add_action('register-comment', id='_register')
    register_comment.comment = '''<p>
        <a href="{{ form_workflow_form_blah_var_fooblock_0_test_url }}" id="t1">1st file in block</a>
        <a href="{{ form_workflow_form_blah_var_fooblock_1_test_url }}" id="t2">2nd file in block</a>
        <a href="{{ form_workflow_form_blah_0_var_fooblock_0_test_url }}" id="t3">again 1st file in block</a>
        <a href="{{ form_workflow_form_blah_0_var_file_url }}" id="t4">file field</a>
    </p>'''

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    resp.form['fblah_1$element0$f123$file'] = Upload('test1.txt', b'foobar1', 'text/plain')
    resp = resp.form.submit('fblah_1$add_element')
    resp.form['fblah_1$element1$f123$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
    resp.form['fblah_2$file'] = Upload('test3.txt', b'foobar3', 'text/plain')
    resp = resp.form.submit('submit').follow()
    assert app.get(resp.pyquery('#t1').attr.href).body == b'foobar1'
    assert app.get(resp.pyquery('#t2').attr.href).body == b'foobar2'
    assert app.get(resp.pyquery('#t3').attr.href).body == b'foobar1'
    assert app.get(resp.pyquery('#t4').attr.href).body == b'foobar3'
    app.get(resp.pyquery('#t4').attr.href + 'X', status=404)  # wrong URL, unknown file

    # unlogged user
    assert '/login' in get_app(pub).get(resp.pyquery('#t1').attr.href).location

    # other user
    user = pub.user_class()
    user.name = 'Second user'
    user.store()
    account = PasswordAccount(id='foo2')
    account.set_password('foo2')
    account.user_id = user.id
    account.store()
    login(get_app(pub), username='foo2', password='foo2').get(resp.pyquery('#t1').attr.href, status=403)


def test_rich_commentable_action(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.roles = [logged_users_role().id]
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    commentable = st1.add_action('commentable', id='_commentable')
    commentable.by = [logged_users_role().id]
    commentable.required = True

    choice = st1.add_action('choice', id='_x1')
    choice.label = 'Submit'
    choice.by = [logged_users_role().id]
    choice.status = st1.id
    wf.store()

    formdef.workflow = wf
    formdef.store()

    # comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp.form['comment'] = '<p>hello <i>world</i></p>'
    resp = resp.form.submit('button_x1').follow()
    assert resp.pyquery('div.comment').text() == 'hello world'
    assert '<p>hello <i>world</i></p>' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[-1].comment == '<p>hello <i>world</i></p>'

    resp.form['comment'] = '<p>hello <script>evil</script></p>'
    resp = resp.form.submit('button_x1').follow()
    assert '<p>hello evil</p>' in resp.text
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[-1].comment == '<p>hello evil</p>'

    resp.form['comment'] = '<p></p>'  # left empty
    resp = resp.form.submit('button_x1')
    assert resp.pyquery('.error').text() == 'required field'

    resp.form['comment'] = '<p>   </p>'  # left ~empty
    resp = resp.form.submit('button_x1')
    assert resp.pyquery('.error').text() == 'required field'

    # url to links
    resp.form['comment'] = '<p>Here is the address: https://example.net</p>'
    resp = resp.form.submit('button_x1').follow()
    assert (
        '<p>Here is the address: <a href="https://example.net" rel="nofollow">https://example.net</a></p>'
        in resp.text
    )

    # test paragraphs are converted to newlines in plain text view
    resp.form['comment'] = '<p>hello</p><p>world</p>'
    resp = resp.form.submit('button_x1').follow()
    formdata = formdef.data_class().select()[0]
    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_comment}}')
    assert tmpl.render(context) == 'hello\n\nworld'

    # test <br> are accepted and converted to single-newline in plain text view
    resp.form['comment'] = '<p>hello<br>world</p>'
    resp = resp.form.submit('button_x1').follow()
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[-1].comment == '<p>hello<br>world</p>'
    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_comment}}')
    assert tmpl.render(context) == 'hello\nworld'


def test_jumps_with_by_and_no_trigger(pub):
    FormDef.wipe()
    Workflow.wipe()
    pub.role_class.wipe()

    role = pub.role_class(name='xxx')
    role.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    jump = st1.add_action('jump')
    jump.status = 'st2'
    jump.by = [role.id]

    jump = st1.add_action('jump')
    jump.status = 'st3'
    jump.by = []

    workflow.add_status('Status2', 'st2')
    workflow.add_status('Status3', 'st3')
    workflow.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.workflow = workflow
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    # it jumps to st2, as jump.by is only related to triggers
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'


def test_user_filter_auto_custom_view(pub):
    user = create_user(pub)

    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.user_support = 'optional'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.user_id = user.id
        carddata.just_created()
        carddata.store()

    carddata.user_id = None  # don't associate latest (baz) with user
    carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemField(id='0', label='item', type='item', varname='foo', data_source=ds),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get(formdef.get_url())
    assert [x[2] for x in resp.form['f0'].options] == ['bar', 'baz', 'foo']

    formdef.fields[0].data_source['type'] = 'carddef:%s:_with_user_filter' % carddef.url_name
    formdef.store()
    resp = app.get(formdef.get_url())
    assert [x[2] for x in resp.form['f0'].options] == ['---']

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    assert [x[2] for x in resp.form['f0'].options] == ['bar', 'foo']


def test_go_to_backoffice(pub):
    formdef = create_formdef()
    app = get_app(pub)
    resp = app.get('/test/go-to-backoffice')
    assert resp.location.endswith('/backoffice/forms/%s/' % formdef.id)


def test_global_interactive_action(pub):
    user = create_user(pub)

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')

    display = action.add_action('displaymsg')
    display.message = 'This is a message'
    display.to = []

    form_action = action.add_action('form')
    form_action.varname = 'blah'
    form_action.formdef = WorkflowFormFieldsFormDef(item=form_action)
    form_action.formdef.fields.append(
        fields.StringField(id='1', label='Test', varname='test', type='string', required=True)
    )
    register_comment = action.add_action('register-comment')
    register_comment.comment = 'HELLO {{ form_workflow_form_blah_var_test }}'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.perform_workflow()
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')
    resp = resp.follow()  # -> error, empty action
    resp = resp.follow()  # -> back to form
    assert 'Error: empty action' in resp.text

    form_action.by = trigger.roles
    workflow.store()

    resp = app.get(formdata.get_url(backoffice=False))
    resp = resp.form.submit('button-action-1')
    resp = resp.follow()
    assert 'This is a message' in resp.text
    resp = resp.form.submit('submit')
    assert resp.pyquery('#form_error_fblah_1').text() == 'required field'
    resp.form['fblah_1'] = 'GLOBAL INTERACTIVE ACTION'
    resp = resp.form.submit('submit')
    assert resp.location == formdata.get_url(backoffice=False)
    resp = resp.follow()

    assert 'HELLO GLOBAL INTERACTIVE ACTION' in resp.text