wcs/tests/test_form_pages.py

# -*- coding: utf-8 -*-

import datetime
import json
import pytest
import hashlib
import os
import re
import time
import zipfile
import base64
from webtest import Upload, Hidden
import mock

try:
    from PIL import Image
except ImportError:
    Image = None

from django.utils.six import StringIO, BytesIO
from django.utils.six.moves.urllib import parse as urlparse

from quixote.http_request import Upload as QuixoteUpload
from django.utils.encoding import force_bytes, force_text

from wcs.qommon import force_str
from wcs.qommon.emails import docutils
from wcs.qommon.form import UploadedFile
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.carddef import CardDef
from wcs.formdef import FormDef
from wcs.workflows import (Workflow, EditableWorkflowStatusItem,
        DisplayMessageWorkflowStatusItem, WorkflowBackofficeFieldsFormDef,
        ChoiceWorkflowStatusItem, JumpOnSubmitWorkflowStatusItem,
        SendmailWorkflowStatusItem, CommentableWorkflowStatusItem)
from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
from wcs.wf.export_to_model import ExportToModel, transform_to_pdf
from wcs.wf.jump import JumpWorkflowStatusItem
from wcs.wf.attachment import AddAttachmentWorkflowStatusItem
from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
from wcs.wf.resubmit import ResubmitWorkflowStatusItem
from wcs.categories import Category
from wcs.roles import Role, logged_users_role
from wcs.tracking_code import TrackingCode
from wcs.data_sources import NamedDataSource
from wcs.wscalls import NamedWsCall
from wcs import fields
from wcs.logged_errors import LoggedError
from wcs.forms.root import PublicFormStatusPage

from utilities import get_app, login, create_temporary_pub, clean_temporary_pub


def assert_equal_zip(stream1, stream2):
    z1 = zipfile.ZipFile(stream1)
    z2 = zipfile.ZipFile(stream2)
    assert set(z1.namelist()) == set(z2.namelist())
    for name in z1.namelist():
        t1, t2 = z1.read(name), z2.read(name)
        assert t1 == t2, 'file "%s" differs' % name

def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates', 'pickle-lazy'], indirect=True)

@pytest.fixture
def pub(request, emails):
    pub = create_temporary_pub(
            sql_mode=bool('sql' in request.param),
            templates_mode=bool('templates' in request.param),
            lazy_mode=bool('lazy' in request.param),
            )
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    if Category.count() == 0:
        cat = Category(name='foobar')
        cat.store()

    return pub


def teardown_module(module):
    clean_temporary_pub()


def create_formdef():
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()
    return formdef

def create_user(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.name = 'User Name'
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()
    return user

def create_user_and_admin(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()

    admin = pub.user_class()
    admin.email = 'admin@localhost'
    admin.is_admin = True
    admin.store()
    account = PasswordAccount(id='admin')
    account.set_password('admin')
    account.user_id = admin.id
    account.store()
    return user, admin

def test_home(pub):
    create_formdef()
    home = get_app(pub).get('/')
    assert 'category-misc' in home.text
    assert '<a class="" href="test/">test</a>' in home.text

def test_home_with_user_forms(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.category_id = '1'
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    wf.store()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-st1'
    formdata.data = {}
    formdata.store()
    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="test/%s/"' % formdata.id in resp
    assert 'Draft' in resp
    assert '<a href="test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s' % draft.id, status=302)
    resp = resp.follow(status=200)

    # disable formdef: formdatas are still visible and accessible, drafts are not
    formdef.disabled = True
    formdef.store()
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="test/%s/"' % formdata.id in resp
    assert not 'Draft' in resp
    assert not '<a href="test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s' % draft.id, status=302)
    resp = resp.follow(status=403)

def test_home_category(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'category-foobar' in home.text
    assert not 'category-misc' in home.text
    assert '<a class="" href="foobar/test/">test</a>' in home.text

def test_home_two_categories(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'foobar'
    formdef2.fields = []
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text # 1st formdef
    assert 'category-misc' in resp.text   # 2nd formdef, fake category
    assert '<a class="" href="foobar/test/">test</a>' in resp.text

    cat2 = Category(name='barfoo')
    cat2.store()
    formdef2.category_id = cat2.id
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.text # 1st formdef
    assert 'category-barfoo' in resp.text # 2nd formdef
    assert not 'category-misc' in resp.text # no more "misc" category

def test_home_keywords(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.keywords = 'hello, world'
    formdef.store()
    home = get_app(pub).get('/')
    assert home.html.find('div', {'data-keywords': 'hello world'}) or home.html.find('div', {'data-keywords': 'world hello'})
    assert home.html.find('li', {'data-keywords': 'hello world'}) or home.html.find('li', {'data-keywords': 'world hello'})

def test_home_formdef_description(pub):
    formdef = create_formdef()
    formdef.description = 'HELLO WORLD'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'HELLO WORLD' in home.text
    assert '<a class="" href="test/">test</a>' in home.text

def test_home_disabled(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.store()
    home = get_app(pub).get('/')
    assert not '<a href="test/">test</a>' in home.text

    # check access is denied
    resp = get_app(pub).get('/test/', status=403)

def test_home_disabled_with_redirect(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.disabled_redirection = 'http://example.org'
    formdef.store()
    resp = get_app(pub).get('/')
    assert '<a class="redirection" href="test/">test</a>' in resp.text
    resp = resp.click('test')
    assert resp.location == 'http://example.org'

def test_home_inaccessible(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.store()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://example.net/login/?next=http%3A%2F%2Fexample.net%2F'

def test_home_always_advertise(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.always_advertise = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' in home.text
    assert '<a href="test/">test</a><span> (authentication required)</span>' in home.text

def test_home_redirect(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/'

def test_home_redirect_var(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/[site_lang]/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/en/'

def test_category_page(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    resp = get_app(pub).get('/foobar/')
    assert '<h2>foobar</h2>' in resp.text
    assert '<a class="" href="test/">test</a>' in resp.text

def test_category_page_redirect(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/'

def test_category_page_redirect_var(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/[site_lang]/[category_slug]/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/en/foobar/'

def test_form_access(pub):
    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    # check a formdef protected by a role cannot be accessed
    formdef.roles = [role.id]
    formdef.store()
    # an unlogged user will ge ta redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # while a logged-in user will get a 403
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=403)

    # unless the user has the right role
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check admin has access, even without specific roles
    user = create_user(pub)
    user.roles = []
    user.is_admin = True
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check special "logged users" role
    formdef.roles = [logged_users_role().id]
    formdef.store()
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)
    resp = get_app(pub).get('/test/', status=302) # redirect to login

    # check "receiver" can also access the formdef
    formdef = create_formdef()
    formdef.roles = [-2]
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

def test_form_access_auth_context(pub):
    user = create_user(pub)

    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'auth-contexts', 'fedict')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    formdef.required_authentication_contexts = ['fedict']
    formdef.roles = [logged_users_role().id]
    formdef.store()

    # an unlogged user will get a redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # a user logged in with a simple username/password tuple will get a page
    # to relogin with a stronger auth
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' in resp.text

    for session in pub.session_manager.values():
        session.saml_authn_context = 'urn:oasis:names:tc:SAML:2.0:ac:classes:SmartcardPKI'
        session.store()
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' not in resp.text
    assert resp.form

def test_form_submit(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert not 'None' in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text

def test_form_submit_no_confirmation(pub):
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1

def test_form_string_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.text
    next_page.forms[0]['f0'] = 'foobar'
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': 'foobar'}

def test_form_items_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.ItemsField(id='0', label='items', type='items',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Three']),]
    formdef.store()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.text
    next_page.forms[0]['f0$element0'].checked = True
    next_page.forms[0]['f0$element1'].checked = True
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar']
    assert data.data['0_display'] == 'Foo, Bar'

def test_form_string_with_invalid_xml_chars(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0'] = 'hello\x0b\x0cworld'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    data = formdef.data_class().select()[0]
    assert data.data == {'0': 'helloworld'}

def assert_current_page(resp, page_label):
    for li_tag in resp.html.findAll('li'):
        if 'current' in li_tag.attrs['class']:
            assert li_tag.find_all('span')[-1].text == page_label

def test_form_multi_page(pub):
    for initial_condition in (None, 'True'):
        formdef = create_formdef()
        formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
                fields.StringField(id='1', label='string'),
                fields.PageField(id='2', label='2nd page', type='page'),
                fields.StringField(id='3', label='string 2')]
        formdef.fields[0].condition = {'type': 'python', 'value': initial_condition}
        formdef.store()
        page = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        page.forms[0]['f1'] = 'foo'
        assert page.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
        next_page = page.forms[0].submit('submit')
        assert_current_page(next_page, '2nd page')
        assert next_page.forms[0]['previous']
        next_page.forms[0]['f3'] = 'bar'
        next_page = next_page.forms[0].submit('submit')
        assert_current_page(next_page, 'Validating')
        assert 'Check values then click submit.' in next_page.text
        next_page = next_page.forms[0].submit('submit')
        assert next_page.status_int == 302
        next_page = next_page.follow()
        assert 'The form has been recorded' in next_page.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        data = formdef.data_class().get(data_id)
        assert data.data == {'1': 'foo', '3': 'bar'}


def test_form_multi_page_title_and_subtitle_as_template(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.TitleField(id='4', label='<i>title of second page {{ form_var_foo }}</i>',
                              type='title'),
            fields.SubtitleField(id='5', label='<i>subtitle of second page {{ form_var_foo }}</i>',
                                 type='subtitle'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '35 < 42'
    resp = resp.form.submit('submit')
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    expected_label = '&lt;i&gt;title of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="title "><h3>%s</h3></div>' % expected_label in resp.text
    expected_label = '&lt;i&gt;subtitle of second page 35 &lt; 42&lt;/i&gt;'
    assert '<div class="subtitle "><h4>%s</h4></div>' % expected_label in resp.text


def test_form_multi_page_condition(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

def test_form_multi_page_condition_select(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar']),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_foo == "Foo"'}),
            fields.PageField(id='3', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_foo == "Bar"'}),
            fields.StringField(id='4', label='string 2')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not '2nd page' in resp.text
    assert not '3rd page' in resp.text
    assert resp.forms[0]['f1'].value == 'Foo' # preset
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert not '3rd page' in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert not '2nd page' in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')

def test_form_multi_page_condition_select_new_varname(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar']),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'form_var_foo == "Foo"'}),
            fields.PageField(id='3', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'form_var_foo == "Bar"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not '2nd page' in resp.text
    assert not '3rd page' in resp.text
    resp.forms[0]['f1'] = 'Foo'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.text
    assert not '3rd page' in resp.text
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert not '2nd page' in resp.text
    assert '3rd page' in resp.text
    assert_current_page(resp, '3rd page')

def test_form_multi_page_condition_checkbox(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'].checked = True
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']
    resp.forms[0]['f1'].checked = False
    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_json_check(pub):
    # make sure the json export has no value for fields from hidden pages
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2', varname='st2'),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "True"'}),
            fields.StringField(id='5', label='string 3', varname='st3'),
            ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit') # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit') # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit()

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None

def test_form_multi_page_condition_no_confirmation_json_check(pub):
    # same as above but without the confirmation page.
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2', varname='st2'),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "True"'}),
            fields.StringField(id='5', label='string 3', varname='st3'),
            fields.PageField(id='6', label='4th page', type='page'),
            fields.CommentField(id='7', label='Check values then click submit.',
                type='comment'),
            ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit') # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit') # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    resp = resp.form.submit('submit')

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None

def test_form_multi_page_condition_data_source(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    # add the named data source, empty
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_data_source_with_form_variable(pub):
    # this tries to recreate #8272 which is about a json datasource being
    # used in a page condition and taking a value from the given page to
    # filter its content.  It is emulated here with a Python datasource
    # being empty if a field was not set.
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='xxx',
                required=False),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    # add the named data source, related to a field on the first page
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': 'form_var_xxx and [form_var_xxx] or []'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'HELLO'
    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_on_first_page(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2'),
            fields.PageField(id='4', label='3rd page', type='page'),
            ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    # should be on second page already
    with pytest.raises(AssertionError):
        resp.form.get('previous')
    resp.form['f3'] = 'foo'
    assert_current_page(resp, '2nd page')
    resp = resp.form.submit('submit') # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('submit') # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert resp.form['previous']
    resp = resp.form.submit('previous') # -> 3rd page
    assert_current_page(resp, '3rd page')
    resp = resp.form.submit('previous') # -> 2nd page
    assert_current_page(resp, '2nd page')
    assert resp.form['f3']
    with pytest.raises(AssertionError):
        resp.form.get('previous')

def test_form_multi_page_condition_on_first_and_next(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page',
                condition={'type': 'python', 'value': 'True'}),
            fields.StringField(id='1', label='string', varname='val1'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'vars().get("form_var_val1") == "foo"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert resp.form['f3']
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo', '3': 'bar'}

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'xxx'
    resp = resp.form.submit('submit')
    with pytest.raises(AssertionError):
        assert resp.form['f3']
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert len(formdef.data_class().select()) == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data.get('1') == 'xxx'
    assert data.data.get('3') is None

def test_form_multi_page_condition_no_visible_page(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    get_app(pub).get('/test/', status=404)

def test_form_multi_page_many_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(id='0', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'True'}),
    ]

    formdef.store()
    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        resp = get_app(pub).get('/test/')
        call_count = invalidate_cache.call_count

    for i in range(30):
        formdef.fields.append(fields.PageField(
                id=str(i+2), label='page %s' % (i+2), type='page',
                condition={'type': 'python', 'value': 'True'}))
    formdef.store()

    # check the cache doesn't get invalidated for every page
    with mock.patch('wcs.qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        resp = get_app(pub).get('/test/')
        assert invalidate_cache.call_count == call_count

def test_form_multi_page_condition_stored_values(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'form_var_foo == "toto"'}),
            fields.StringField(id='3', label='string 2'),
            fields.PageField(id='4', label='3rd page', type='page'),
            ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert 'bar' in resp.text
    resp = resp.form.submit('previous')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')  # -> validation page
    assert 'Check values then click submit.' in resp.text
    assert not 'bar' in resp.text
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None

    # same without validation page
    formdef.confirmation = False
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'toto'
    resp = resp.form.submit('submit')  # -> page 2
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('previous')  # -> page 2
    resp = resp.form.submit('previous')  # -> page 1
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')  # -> page 3
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'blah'
    assert formdata.data.get('3') is None

def test_form_multi_page_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2'),
            fields.PageField(id='4', label='3rd page', type='page'),
            fields.StringField(id='5', label='string 3', varname='bar'),
            ]

    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foo'
    assert formdef.data_class().select()[0].data['5'] == 'bar'

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'False'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'global-errors' in resp.text
    assert 'You shall not pass.' in resp.text

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'form_var_foo == "foo"'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

    # check a post-condition raising an exception, they should always fail.
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': '1/0'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    # check a post-condition referring to a field on the same page
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'foo'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.text
    assert 'You shall not pass.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

def test_form_multi_page_conditions_and_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page',
            post_conditions=[{
                'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'},
                'error_message': 'You shall not pass.'}]),
        fields.StringField(id='1', label='string', varname='bar'),
        fields.PageField(id='3', label='2nd page', type='page'),
    ]

    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text

    # add a conditional page, this will cause pages to be evaluated first
    # (and would trigger #25197)
    formdef.fields.append(
        fields.PageField(id='4', label='3rd page', type='page',
            condition={'type': 'python', 'value': 'True'}))
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'You shall not pass.' in resp.text

def test_form_multi_page_page_name_as_title(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.TitleField(id='4', label='1st page', type='title'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2 # in steps and in main body

    # add a comment that will not be displayed and should therefore not be
    # considered.
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.CommentField(id='5', label='bla bla bla', type='comment'),
            fields.TitleField(id='4', label='1st page', type='title'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.text
    assert next_page.text.count('1st page') == 2 # in steps and in main body

def test_form_submit_with_user(pub, emails):
    create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    assert '<div class="section foldable folded" id="summary">' in next_page.text
    # check the user received a copy by email
    assert emails.emails.get('New form (test)')
    assert emails.emails.get('New form (test)')['email_rcpt'] == ['foo@localhost']

def test_form_titles(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.TitleField(id='4', label='1st page', type='title'),
        fields.SubtitleField(id='5', label='subtitle of 1st page', type='subtitle'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.TitleField(id='6', label='title of second page', type='title'),
        fields.StringField(id='3', label='string 2', required=False)]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert not '<h3 data-field-id="0">1st page/h3>' in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp = resp.form.submit('submit')  # -> validation page
    assert '<h3>1st page</h3>' in resp.text
    assert '<h4 data-field-id="5">subtitle of 1st page</h4>' in resp.text
    assert '<h3 data-field-id="6">title of second page</h3>' in resp.text
    resp.form['f3'] = 'foo'
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert not '<div class="title "><h3>1st page</h3></div>' in resp.text
    assert '<div class="subtitle "><h4>subtitle of 1st page</h4></div>' in resp.text
    assert '<div class="title "><h3>title of second page</h3></div>' in resp.text

def test_form_summary_empty_pages(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string', varname='toto'),
        fields.PageField(id='2', label='2nd page', type='page',
            condition={'type': 'python', 'value': 'form_var_toto == "foo"'}),
        fields.TitleField(id='6', label='title in second page', type='title'),
        fields.StringField(id='3', label='string'),
        fields.PageField(id='4', label='3rd page', type='page'),
        fields.StringField(id='5', label='string'),
        fields.PageField(id='7', label='4th page', type='page'),
        fields.CommentField(id='8', label='Bla bla bla', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit')
    formdata_id = resp.location.split('/')[-2]
    resp = resp.follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    resp = app.get('/test/')  # -> 1st page
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')  # -> 2nd page
    resp.form['f3'] = 'bar'
    resp = resp.form.submit('previous')  # -> 1st page
    resp.form['f1'] = 'baz'
    resp = resp.form.submit('submit')  # -> 3rd page
    resp.form['f5'] = 'baz'
    resp = resp.form.submit('submit')  # -> 4th page
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submit
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' not in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

    # change condition to have second page never displayed
    formdef.fields[2].condition['value'] = False
    formdef.store()
    formdata = formdef.data_class().get(formdata_id)
    resp = app.get(formdata.get_url())
    # it was filled by user, it should still appear (conditions should not be
    # replayed)
    assert '<h3>1st page</h3>' in resp.text
    assert '<h3>2nd page</h3>' in resp.text
    assert '<h3>3rd page</h3>' in resp.text
    assert '<h3>4th page</h3>' not in resp.text

def test_form_display_locations(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string1', display_locations=[]),
        fields.StringField(id='2', label='string2', display_locations=['validation']),
        fields.StringField(id='3', label='string3', display_locations=['summary']),
        fields.CommentField(id='4', label='Bla bla bla', type='comment',
            display_locations=['validation', 'summary']),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'plop1'
    resp.form['f2'] = 'plop2'
    resp.form['f3'] = 'plop3'
    resp = resp.form.submit('submit')  # -> validation
    pq = resp.pyquery.remove_namespaces()
    assert pq('div[style="display: none;"] [name=f1]')
    assert not pq('div[style="display: none;"] [name=f2]')
    assert pq('div[style="display: none;"] [name=f3]')
    assert 'Bla bla bla' in resp.text

    resp = resp.form.submit('submit').follow()  # -> submit
    assert formdef.data_class().select()[0].data['1'] == 'plop1'
    assert formdef.data_class().select()[0].data['2'] == 'plop2'
    assert formdef.data_class().select()[0].data['3'] == 'plop3'
    assert not 'plop1' in resp.text
    assert not 'plop2' in resp.text
    assert 'plop3' in resp.text
    assert 'Bla bla bla' in resp.text

def test_form_visit_existing(pub):
    user = create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.store()

    formdata_user = formdef.data_class()()
    formdata_user.user_id = user.id
    formdata_user.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = get_app(pub).get('/test/%s/' % formdata_user.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % formdata_user.id)
    assert 'The form has been recorded on' in resp

def test_form_auth(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/auth')
    assert resp.location == 'http://example.net/login/?ReturnUrl=http%3A//example.net/test/'

    resp = login(get_app(pub), username='foo', password='foo').get('/test/auth')
    assert resp.location == 'http://example.net/test/'

def test_form_tryauth(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/tryauth')
    assert resp.location == 'http://example.net/test/'

    app = login(get_app(pub), username='foo', password='foo')
    pub.cfg['identification'] = {'methods': ['idp']}
    pub.write_cfg()
    # if the user is logged in, the form should be presented
    resp = app.get('/test/tryauth')
    assert resp.location == 'http://example.net/test/'

    # if the user is unlogged, there should be a passive redirection to SSO
    resp = get_app(pub).get('/test/tryauth')
    assert 'IsPassive=true' in resp.location

    pub.cfg['identification'] = {'methods': ['password']}
    pub.write_cfg()

def test_form_forceauth(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/forceauth')
    assert resp.location == 'http://example.net/login/?ReturnUrl=http%3A//example.net/test/&forceAuthn=true'

def test_form_no_tracking_code(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert not '<h3>Tracking code</h3>' in resp.text

def test_form_no_tracking_code_variable(pub):
    create_user(pub)
    FormDef.wipe()
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.CommentField(id='3', label='<p>xxx{{form_tracking_code|default:""}}yyy</p>', type='comment'),
    ]
    formdef.store()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp.form['f1'] = 'foo'
    app.post('/test/autosave', params=resp.form.submit_fields())
    resp = resp.form.submit('submit')
    assert_current_page(resp, '2nd page')
    assert 'xxxyyy' in resp.text
    resp = resp.form.submit('submit')
    assert_current_page(resp, 'Validating')
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'foo'}
    assert data.tracking_code is None


def get_displayed_tracking_code(resp):
    tracking_code = None
    for a_tag in resp.html.findAll('a'):
        if 'code/' in a_tag['href']:
            tracking_code = a_tag.text
            break
    return tracking_code

def test_form_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()

    # check anonymous user can't get to it from the URL
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('http://example.net/test/%s' % formdata_id)
    assert resp.location.startswith('http://example.net/login')

    # or logged users that didn't enter the code:
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get(
            'http://example.net/test/%s' % formdata_id, status=403)

    # check we can also get to it as a logged user
    pub.session_manager.session_class.wipe()
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()

    # go back as anonymous
    pub.session_manager.session_class.wipe()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit') # -> confirmation page
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1 # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.text # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

    # check we can also use it with lowercase letters.
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code.lower()
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code.lower()
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()

def test_form_tracking_code_rate_limit(pub):
    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'rate-limit', '2/2s')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    # twice
    get_app(pub).get('/code/ABC/load', status=404)
    get_app(pub).get('/code/ABC/load', status=404)
    # and out
    get_app(pub).get('/code/ABC/load', status=403)
    get_app(pub).get('/code/ABC/load', status=403)
    # wait two second
    time.sleep(2)
    # and ok again
    get_app(pub).get('/code/ABC/load', status=404)

def test_form_tracking_code_as_user(pub, nocache):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit') # -> confirmation page
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.text
    assert formdef.data_class().count() == 1 # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert str(formdef.data_class().select()[0].user_id) == str(user.id)
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.text # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

    # and check we can also get back to it as anonymous
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.text # makes sure user is treated as submitter

    # and check a bot is not allowed to get it
    app = get_app(pub)
    resp = app.get('/code/%s/load' % tracking_code,
            headers={'User-agent': 'Googlebot'}, status=403)

def test_form_empty_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # check we get a 404 if we use the tracking code before it gets any data
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow(status=404)

def test_form_tracking_code_email(pub, emails, nocache):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string'),
                      fields.StringField(id='1', label='string2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'barfoo'
    # autosave will be made using javascript in real world
    app.post('/test/autosave', params=resp.form.submit_fields())

    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    resp = get_app(pub).get('/test/code/%s/' % tracking_code)
    assert '<h2>Keep your tracking code</h2>' in resp.text
    resp.forms[0]['email'] = 'foo@localhost'
    resp = resp.forms[0].submit()
    assert emails.emails.get('Tracking Code reminder')
    assert tracking_code in list(emails.emails.values())[0]['payload']
    assert resp.location == 'http://example.net/test/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    resp = resp.follow()
    assert resp.forms[1]['f0'].value == 'barfoo'

def test_form_tracking_code_remove_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '1'
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # visit page, check there's no remove draft button
    resp = app.get('/test/')
    assert '<h3>Tracking code</h3>' in resp.text
    assert not 'removedraft' in resp.text

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    resp = resp.forms[0].submit() # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0

def test_form_tracking_code_remove_empty_draft(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('previous')
    app.post('/test/autosave', params=resp.form.submit_fields())
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '0'

    # make draft empty
    formdata = formdef.data_class().select()[0]
    formdata.data = {}
    formdata.store()
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.text
    assert 'removedraft' in resp.text
    assert resp.forms[1]['f0'].value == ''

    resp = resp.forms[0].submit() # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0

def test_form_discard_draft(pub, nocache):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    formdef.data_class().wipe()

    # anonymous user, no tracking code (-> no draft)
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == []
    assert 'Cancel' in resp.text
    assert 'Discard' not in resp.text
    resp = resp.form.submit('cancel')

    # anonymous user, tracking code (-> draft)
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, no tracking code
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # logged-in user, tracking code
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    resp = resp.form.submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == []  # discarded

    # anonymous user, tracking code, recalled
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    assert 'Cancel' not in resp.text
    assert 'Discard' in resp.text
    tracking_code = get_displayed_tracking_code(resp)

    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()
    assert resp.forms[1]['f0'].value == 'foobar'
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']

    # logged-in user, no tracking code, recalled
    formdef.data_class().wipe()
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'foobar'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert [x.status for x in formdef.data_class().select()] == ['draft']
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.click('Continue with draft').follow()
    assert 'Cancel' in resp.text
    assert 'Discard Draft' in resp.text
    resp = resp.forms[1].submit('cancel')
    assert [x.status for x in formdef.data_class().select()] == ['draft']

def test_form_invalid_tracking_code(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    # create a secondary formdef, to always have the tracking code form
    # displayed on homepage
    formdef2 = FormDef()
    formdef2.name = 'test2'
    formdef2.fields = []
    formdef2.enable_tracking_codes = True
    formdef2.store()

    resp = get_app(pub).get('/')

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.store()

    # check we can go back to it
    formdef.data_class().wipe()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata.id
    resp = resp.follow()

    # check we get a not found error message on non-existent code
    fake_code = TrackingCode().get_new_id()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = fake_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % fake_code
    resp = resp.follow(status=404)

    # check we also get an error if tracking code access is disabled after the
    # fact
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow(status=404)

def test_form_tracking_code_as_variable(pub, nocache):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.CommentField(type='comment', id='3',
                label='!{{ form_tracking_code }}!')]
    formdef.enable_tracking_codes = True
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f1'] = 'foobar'
    resp = resp.form.submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    assert '!%s!' % tracking_code in resp.text

def test_form_draft_with_file(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert '<span class="filename">test.txt</span>' in resp.text

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit') # -> confirmation page
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()

    assert 'download="test.txt"' in resp.text
    assert resp.click('test.txt').follow().text == 'foobar'

def test_form_draft_with_file_direct_validation(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file', type='file')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0$file'] = Upload('test2.txt', b'foobar2', 'text/plain')
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert 'test2.txt' in resp.text

    # check submitted form keeps the file
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()

    assert 'download="test2.txt"' in resp.text
    assert resp.click('test2.txt').follow().text == 'foobar2'

def test_form_draft_with_date(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='date', type='date')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    tracking_code = get_displayed_tracking_code(resp)
    resp.forms[0]['f0'] = '2012-02-12'
    resp = resp.forms[0].submit('submit')

    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit().follow().follow().follow()
    assert '2012-02-12' in resp.text

    # check submitted form keeps the date
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()

    assert '2012-02-12' in resp.text

@pytest.mark.parametrize('tracking_code', [True, False])
def test_form_direct_draft_access(pub, tracking_code):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = tracking_code
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.status = 'draft'
    formdata.store()

    resp = get_app(pub).get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    formdata.user_id = user.id
    formdata.store()
    resp = get_app(pub).get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/test/?mt=')

    formdata.user_id = 1000
    formdata.store()
    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s' % formdata.id, status=403)

def form_password_field_submit(app, password):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [fields.PasswordField(id='0', label='password',
        formats=['sha1', 'md5', 'cleartext'])]
    formdef.store()
    page = app.get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.text
    next_page.forms[0]['f0$pwd1'] = password
    next_page.forms[0]['f0$pwd2'] = password
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': {
        'sha1': force_str(hashlib.sha1(force_bytes(password)).hexdigest()),
        'md5': force_str(hashlib.md5(force_bytes(password)).hexdigest()),
        'cleartext': force_str(password),
        }}

def test_form_password_field_submit(pub):
    user = create_user(pub)
    form_password_field_submit(get_app(pub), 'foobar')
    form_password_field_submit(get_app(pub), force_str(u'•	83003706'))
    form_password_field_submit(login(get_app(pub), username='foo', password='foo'), 'foobar\u00eb')

def test_form_multi_page_formdef_count_condition(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'form_objects.count > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text

    # add a formdata this will make the second page appear.
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should NOT go straight to validation
    assert 'Check values then click submit.' not in resp.text

def test_form_multi_page_post_edit(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    next_page.forms[0]['f3'] = 'barXYZ'
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.text

    data_id = formdef.data_class().select()[0].id

    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo'
    resp.forms[0]['f1'] = 'foo2'

    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1'].value == 'foo2'
    resp = resp.forms[0].submit('submit')
    assert 'Save Changes' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo2' in resp.text # modified value is there
    assert 'barXYZ' in resp.text # unchanged value is still there

    # modify workflow to jump to another status after the edition
    st2 = workflow.add_status('Status2', 'st2')
    editable.status = st2.id
    workflow.store()

    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.text
    assert 'barXYZ' in page.text

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo2'
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo3' in resp.text # modified value is there
    assert 'barXYZ' in resp.text # unchanged value is still there
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st2.id

    # jump to a nonexistent status == do not jump, but add a LoggedError
    LoggedError.wipe()
    assert LoggedError.count() == 0
    editable.status = 'deleted_status_id'
    workflow.store()
    # go back to st1
    formdata = formdef.data_class().get(data_id)
    formdata.status = 'wf-%s' % st1.id
    formdata.store()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    assert LoggedError.count() == 1
    logged_error = LoggedError.select()[0]
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.formdef_id == formdef.id
    assert logged_error.workflow_id == workflow.id
    assert logged_error.status_id == st1.id
    assert logged_error.status_item_id == editable.id
    assert logged_error.occurences_count == 1

    # do it again: increment logged_error.occurences_count
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    resp = page.forms[0].submit('button_editable')
    resp = resp.follow()
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id  # stay on st1
    assert LoggedError.count() == 1
    logged_error = LoggedError.select()[0]
    assert logged_error.occurences_count == 2

def test_form_count_dispatching(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.condition = {'type': 'python', 'value': 'form_objects.count_status_st2 < 1'}
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    st2 = workflow.add_status('Status2', 'st2')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 0
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.text # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 1

def test_preview_form(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = []
    formdef.disabled = True
    formdef.store()

    # check the preview page is not accessible to regular users
    get_app(pub).get('/preview/test/', status=403)

    # check it's accessible to admins
    user.is_admin = True
    user.store()
    page = login(get_app(pub), username='foo', password='foo').get('/preview/test/')

    # check no formdata gets stored
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.text
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 200
    assert 'This was only a preview: form was not actually submitted.' in next_page.text
    assert len([x for x in formdef.data_class().select() if not x.is_draft()]) == 0

def test_form_item_data_source_field_submit(pub):
    def submit_item_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0'] = '1'
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux')]),
    }
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'}])
    assert submit_item_data_source_field(ds) == {
            '0': '1', '0_display': 'un', '0_structured': {'id': '1', 'text': 'un', 'more': 'foo'}}

def test_form_items_data_source_field_submit(pub):
    def submit_items_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemsField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0$element1'].checked = True
        resp.forms[0]['f0$element3'].checked = True
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.text
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux'), ('3', 'trois')]),
    }
    assert submit_items_data_source_field(ds) == {'0': ['1', '3'], '0_display': 'un, trois'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'},
        {'id': '3', 'text': 'trois'}])
    assert submit_items_data_source_field(ds) == {'0': ['1', '3'], '0_display': 'un, trois'}

    ds['value'] = repr([
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'},
        {'id': '3', 'text': 'trois', 'more': 'baz'}])
    assert submit_items_data_source_field(ds) == {
            '0': ['1', '3'],
            '0_display': 'un, trois',
            '0_structured': [
                {'id': '1', 'more': 'foo', 'text': 'un'},
                {'id': '3', 'more': 'baz', 'text': 'trois'}]}

def test_form_page_string_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'string', 'value': 'HELLO WORLD'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text

def test_form_page_profile_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

def test_form_page_profile_first_name_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef
    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields = [
        fields.StringField(
            id='_first_name', label='first name',
            type='string', extra_css_class='autocomplete-given-name'),
        fields.StringField(
            id='_city', label='city',
            type='string', extra_css_class='autocomplete-address-level2'),
        fields.StringField(
            id='_plop', label='plop',
            type='string', extra_css_class='xxx')
    ]
    user_formdef.store()
    user.form_data = {'_first_name': 'plop', '_city': 'mytown'}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(
            id='0', label='string',
            prefill={'type': 'user', 'value': '_first_name'}),
        fields.StringField(
            id='1', label='string',
            prefill={'type': 'user', 'value': '_city'}),
        fields.StringField(
            id='2', label='string',
            prefill={'type': 'user', 'value': '_plop'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''
    assert resp.forms[0]['f0'].attrs['autocomplete'] == 'given-name'  # html5
    assert resp.forms[0]['f1'].value == ''
    assert resp.forms[0]['f1'].attrs['autocomplete'] == 'address-level2'  # html5

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'plop'
    assert resp.forms[0]['f1'].value == 'mytown'

def test_form_page_formula_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': repr('HELLO WORLD')})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.text

def test_form_page_template_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='string',
            prefill={'type': 'string', 'value': '{{session_user_display_name}}'})]
    formdef.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert resp.form['f0'].value == 'User Name'
    assert 'widget-prefilled' in resp.text

    # erroneous prefill
    formdef.fields = [
        fields.StringField(id='0', label='string',
            prefill={'type': 'string', 'value': '{{session_user_display_name|unknown}}'})]
    formdef.store()

    resp = app.get('/test/')
    assert resp.form['f0'].value == ''
    assert 'widget-prefilled' not in resp.text

def test_form_page_session_var_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_var_foo'})]
    formdef.store()

    # check it's empty if it doesn't exist
    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    # check it's not set if it's not whitelisted
    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urlparse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check it works
    open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w').write('''[options]
query_string_allowed_vars = foo,bar
''')

    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urlparse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello'

    # check it survives a login
    resp = get_app(pub).get('/?session_var_foo=hello2')
    assert urlparse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('Login')
    resp = resp.follow()
    resp.forms[0]['username'] = 'foo'
    resp.forms[0]['password'] = 'foo'
    resp = resp.forms[0].submit()
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello2'

    # check repeated options are ignored
    resp = get_app(pub).get('/?session_var_foo=hello&session_var_foo=hello2')
    assert urlparse.urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check extra query string parameters are not lost
    resp = get_app(pub).get('/?session_var_foo=hello&foo=bar')
    assert urlparse.urlparse(resp.location).path == '/'
    assert urlparse.urlparse(resp.location).query == 'foo=bar'

    os.unlink(os.path.join(pub.app_dir, 'site-options.cfg'))

def test_form_page_query_string_list_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='1', label='item',
        varname='item', required=False, data_source={'type': 'foobar'},
        prefill={'type': 'string', 'value': '{{request.GET.preselect}}'},
        )]
    formdef.store()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula',
            'value': repr([{'id': '1', 'text': 'un'},
                           {'id': '2', 'text': 'deux'},
                           {'id': '3', 'text': 'trois'},
                           {'id': '4', 'text': 'quatre'},
                           ])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f1'].value == '1'

    resp = get_app(pub).get('/test/?preselect=2')
    assert resp.form['f1'].value == '2'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert 'deux' in resp.text

def test_form_page_profile_prefill_list(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='item', type='item',
        items=['', 'bar@localhost', 'foo@localhost'], required=False,
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    # invalid value
    formdef.fields = [fields.ItemField(id='0', label='item', type='item',
        items=['', 'bar@localhost'], required=False,
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'invalid value selected' in resp.text
    assert resp.forms[0]['f0'].value == ''

def test_form_page_formula_prefill_items_field(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemsField(id='0', label='items',
        items=['foo', 'bar', 'baz'],
        prefill={'type': 'formula', 'value': '["foo", "baz"]'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0$element0'].checked
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked
    assert 'widget-prefilled' in resp.text

    resp.form['f0$element0'].checked = False
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert '>foo<' not in resp.text
    assert '>bar<' not in resp.text
    assert '>baz<' in resp.text

    # check with remote json
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemsField(id='0', label='items',
        data_source=ds, display_disabled_items=True,
        prefill={'type': 'formula', 'value': '["2"]'})]

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not resp.form['f0$element1'].checked
        assert resp.form['f0$element2'].checked


def test_form_captcha(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.store()

    # test authenticated users are not presented with a captcha
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert not 'form_captcha' in resp.text

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text

    session_id = list(app.cookies.values())[0].strip('"')
    session = pub.session_class.get(session_id)
    resp.form['captcha$q'] = session.get_captcha_token(resp.forms[0]['captcha$token'].value)['answer']
    resp = resp.form.submit('submit')
    assert resp.status_code == 302 # redirect when formdata is created

    # and check it gets it only once
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert not 'form_captcha' in resp.text

def test_form_captcha_and_no_validation_page(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.confirmation = False
    formdef.store()

    # test authenticated users are not stopped on a confirmation page
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert resp.status_code == 302 # redirect when formdata is created

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'form_captcha' in resp.text

def test_form_file_field_with_fargo(pub, fargo_url):
    user = create_user(pub)
    formdef = create_formdef()
    file_field = fields.FileField(id='0', label='file')
    assert file_field.allow_portfolio_picking is False
    file_field.allow_portfolio_picking = True
    formdef.fields = [file_field]
    formdef.store()
    formdef.data_class().wipe()

    assert file_field.allow_portfolio_picking == True

    resp = get_app(pub).get('/test/')
    assert 'f0$file' in resp.text
    assert 'fargo.js' not in resp.text
    assert 'use-file-from-fargo' not in resp.text

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.text
    assert 'fargo.js' in resp.text
    assert 'use-file-from-fargo' in resp.text

    file_field.allow_portfolio_picking = False
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.text
    assert 'fargo.js' not in resp.text
    assert 'use-file-from-fargo' not in resp.text

def test_form_file_field_without_fargo(pub):
    user = create_user(pub)
    formdef = create_formdef()
    file_field = fields.FileField(id='0', label='file')
    file_field.allow_portfolio_picking = True
    formdef.fields = [file_field]
    formdef.store()
    formdef.data_class().wipe()

    assert file_field.allow_portfolio_picking == True

    resp = get_app(pub).get('/test/')
    assert 'f0$file' in resp.text
    assert 'fargo.js' not in resp.text
    assert 'use-file-from-fargo' not in resp.text

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.text
    assert 'fargo.js' not in resp.text
    assert 'use-file-from-fargo' not in resp.text

def test_form_file_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    upload = Upload('test.txt', b'foobar', 'text/plain')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

def test_form_file_field_image_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    image_content = open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
    upload = Upload('test.jpg', image_content, 'image/jpeg')

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert '<img alt="" src="tempfile?' in resp.text
    tempfile_id = resp.text[resp.text.index('tempfile?'):].split('&', 1)[0].split('=')[1]

    resp_tempfile = app.get('/test/tempfile?t=%s' % tempfile_id)
    assert resp_tempfile.body == image_content

    if Image:
        # check thumbnailing of image in validation page
        resp_thumbnail = app.get('/test/tempfile?t=%s&thumbnail=1' % tempfile_id)
        assert resp_thumbnail.content_type == 'image/png'

    resp = resp.form.submit('submit').follow()
    assert '<img ' in resp.text
    assert 'download?f=0&thumbnail=1' in resp.text
    resp = resp.goto('download?f=0&thumbnail=1')
    assert '/thumbnail/' in resp.location
    resp = resp.follow()
    if Image:
        # check thumbnailing of image in submitted form
        assert resp.content_type == 'image/png'

        # check a fake image is not sent back
        upload = Upload('test.jpg', b'<script>evil javascript</script>', 'image/jpeg')
        app = get_app(pub)
        resp = app.get('/test/')
        resp.forms[0]['f0$file'] = upload
        resp = resp.forms[0].submit('submit')
        assert not '<img alt="" src="tempfile?' in resp.text

def test_form_file_field_submit_wrong_mimetype(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    upload = Upload('test.txt', b'foobar', 'application/force-download')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

    upload = Upload('test.pdf', b'%PDF-1.4 ...', 'application/force-download')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    resp = resp.click('test.pdf')
    assert resp.location.endswith('/test.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    assert resp.text == '%PDF-1.4 ...'

def test_form_table_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [fields.TableField(id='0', label='table', type='table',
        rows=[force_str(u'à'), 'b'],
        columns=['c', 'd', force_str(u'e')], required=False)]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': [['', '', ''], ['', '', '']]}
    formdef.data_class().wipe()

    formdef.fields = [fields.TableField(id='0', label='table', type='table',
        rows=['a', 'b'], columns=['c', 'd', 'e'], required=True)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert not 'Check values then click submit.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$c-0-0'] = 'a'
    resp.form['f0$c-1-0'] = 'b'
    resp.form['f0$c-0-1'] = 'c'
    resp.form['f0$c-1-1'] = 'd'
    resp.form['f0$c-0-2'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'c', 'e'], ['b', 'd', '']]}

    # check table is present in received email (via form_details).
    user = create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$c-0-0'] = 'àà'  # would trigger column length bug (#23072)
    resp.form['f0$c-1-0'] = 'bb'
    resp.form['f0$c-0-1'] = 'cc'
    resp.form['f0$c-1-1'] = 'dd'
    resp.form['f0$c-0-2'] = 'ee'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    # check rst2html didn't fail
    assert 'ee' in emails.emails['New form (test)']['msg'].get_payload()[1].get_payload()

def test_form_table_rows_field_submit(pub, emails):
    formdef = create_formdef()
    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=False)]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': []}
    formdef.data_class().wipe()

    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=True)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert not 'Check values then click submit.' in resp.text

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = 'b'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = 'd'
    resp.form['f0$element2$col0'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().select()[0].data == {'0': [['a', 'b'], ['c', 'd'], ['e', '']]}

    formdef.data_class().wipe()

    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=True, total_row=True)]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert '37.00' in resp.text

    # check table is present in received email (via form_details).
    user = create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0$element0$col0'] = 'àà'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'ee'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert 'ee' in emails.emails['New form (test)']['msg'].get_payload()[1].get_payload()

def test_form_new_table_rows_field_draft_recall(pub):
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='string'),
        fields.PageField(id='2', label='2nd page', type='page'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'test'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # add new table rows field to formdef
    formdef.fields.append(
        fields.TableRowsField(id='3', label='table', type='tablerows',
            columns=['a', 'b'], required=False)
    )
    formdef.store()

    # restore form on validation page
    resp = get_app(pub).get('/')
    resp.form['code'] = tracking_code
    resp = resp.form.submit().follow().follow().follow()

    # validate form
    resp = resp.forms[1].submit()
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'test'
    assert formdef.data_class().select()[0].data['3'] is None

def test_formdata_attachment_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow() # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

def test_formdata_attachment_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.varname = 'attached_doc'
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow() # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    attachment_variable = attachments.attached_doc

    resp = login(get_app(pub), username='admin', password='admin').get(
        attachment_variable.url).follow()
    assert attachment_variable.content == resp.body
    assert attachment_variable.b64_content == base64.b64encode(resp.body)
    assert attachment_variable.content_type == resp._headers['content-type'].split(';')[0]
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(attachment_variable.filename)

def test_formdata_attachment_download_to_backoffice_file_field(pub):
    create_user(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
    ]
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    attach.backoffice_filefield_id = 'bo1'
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    assert attach.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    # backoffice file field is set
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'bo1' in formdata.data
    bo1 = formdata.data['bo1']
    assert bo1.base_filename == 'test.txt'
    assert bo1.content_type == 'text/plain'
    assert bo1.get_content() == b'foobar'

    # and file is in history, too
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

def test_formdata_attachment_download_to_backoffice_file_field_only(pub):
    create_user(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
    ]
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    attach.backoffice_filefield_id = 'bo1'
    attach.attach_to_history = False  # store only in backoffice field
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    assert attach.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    # backoffice file field is set
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'bo1' in formdata.data
    bo1 = formdata.data['bo1']
    assert bo1.base_filename == 'test.txt'
    assert bo1.content_type == 'text/plain'
    assert bo1.get_content() == b'foobar'

    # but nothing in history
    for evo in formdata.evolution:
        assert not evo.parts

def test_formdata_generated_document_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    assert resp.text == 'HELLO WORLD'

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('test.rtf')
    assert resp.location.endswith('/test.rtf')
    resp = resp.follow()
    assert resp.content_type == 'application/rtf'
    assert resp.text == 'HELLO WORLD'

    # change export model to now be a RTF file, do the action again on the same form and
    # check that both the old .odt file and the new .rtf file are there and valid.
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    assert resp.click('test.rtf', index=0).follow().text == 'HELLO WORLD'
    assert resp.click('test.rtf', index=1).follow().text == 'HELLO NEW WORLD'

    # use substitution variables on rtf: only ezt format is accepted
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO {{DJANGO}} WORLD [form_name]')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow()

    assert resp.click('test.rtf', index=2).follow().text == 'HELLO {{DJANGO}} WORLD {\\uc1{test}}'


@pytest.fixture(params=['template.odt', 'template-django.odt'])
def odt_template(request):
    return request.param

def test_formdata_generated_document_odt_download(pub, odt_template):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    template_filename = os.path.join(os.path.dirname(__file__), odt_template)
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/' + odt_template, content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        assert_equal_zip(BytesIO(resp.body), f)

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    resp = resp.follow() # $form/$id/create_doc
    with mock.patch('wcs.wf.export_to_model.get_formdata_template_context') as get_context_1:
        with mock.patch('wcs.workflows.get_formdata_template_context') as get_context_never:
            get_context_1.return_value = {}
            get_context_never.return_value = {}
            resp = resp.follow() # $form/$id/create_doc/
            # substitution variables are computed only one :
            assert get_context_1.call_count == 1
            assert get_context_never.call_count == 0

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click(odt_template)
    assert resp.location.endswith('/' + odt_template)
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        assert_equal_zip(BytesIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        body = resp.click(odt_template, index=0).follow().body
        assert_equal_zip(BytesIO(body), f)
    assert resp.click('test.rtf', index=0).follow().body == b'HELLO NEW WORLD'

def test_formdata_generated_document_odt_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        assert_equal_zip(BytesIO(resp.body), f)

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    response1 = resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        assert_equal_zip(BytesIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        body = resp.click('template.odt', index=0).follow().body
        assert_equal_zip(BytesIO(body), f)
    response2 = resp.click('test.rtf', index=0).follow()
    assert response2.body == b'HELLO NEW WORLD'
    # Test attachment substitution variables
    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    file1 = attachments.created_doc
    assert file1.content == response2.body
    assert file1.b64_content == base64.b64encode(response2.body)
    assert file1.content_type == response2._headers['content-type']
    content_disposition = response2._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response2.request.environ['PATH_INFO'].endswith(file1.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file1.url).follow()
    assert file1.content == resp.body
    assert file1.b64_content == base64.b64encode(resp.body)
    assert file1.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file1.filename)

    file2 = attachments.created_doc[1]
    assert file2.content == response1.body
    assert file2.b64_content == base64.b64encode(response1.body)
    assert file2.content_type == response1._headers['content-type']
    content_disposition = response1._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response1.request.environ['PATH_INFO'].endswith(file2.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file2.url).follow()
    assert file2.content == resp.body
    assert file2.b64_content == base64.b64encode(resp.body)
    assert file2.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file2.filename)

@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'inline'
    assert resp.body.startswith(b'%PDF-')

@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(pub, fargo_url,
                                                                           fargo_secret, caplog):
    user = create_user(pub)
    user.name = 'Foo Baré'
    user.store()

    pub.cfg['debug'] = {'logger': True}
    pub.write_cfg()
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    export_to.push_to_portfolio = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert ("file 'template.pdf' pushed to portfolio of 'Foo Bar\\xc3\\xa9'"
                == caplog.records[-1].message)

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 400, 'null', None  # fail
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert ("file 'template.pdf' failed to be pushed to portfolio of 'Foo Bar\\xc3\\xa9'"
                == caplog.records[-1].message)

    # failed to push to portfolio, but document is here
    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert http_post_request.call_args[0][0].startswith('http://fargo.example.net/api/documents/push/')
        payload = json.loads(http_post_request.call_args[0][1])
        assert payload['file_name'] == 'template.pdf'
        assert payload['user_email'] == 'foo@localhost'
        assert payload['origin'] == 'example.net'
        assert payload['file_b64_content'].decode('base64').startswith('%PDF')
        assert ("file 'template.pdf' pushed to portfolio of 'Foo Bar\\xc3\\xa9'"
                == caplog.records[-1].message)
    assert resp.location == form_location
    resp = resp.follow()  # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    assert resp.body.startswith(b'%PDF-')

def test_formdata_generated_document_non_interactive(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    st1.items.append(export_to)
    export_to.parent = st1

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    st2 = wf.add_status('Status2', 'st2')

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
        assert_equal_zip(BytesIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'

def test_formdata_generated_document_to_backoffice_field(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
        fields.StringField(id='bo2', label='bo field 2', type='string'),
    ]

    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    export_to.backoffice_filefield_id = 'bo1'
    st1.items.append(export_to)
    export_to.parent = st1

    assert export_to.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    wf.add_status('Status2', 'st2')
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    # get the two generated files from backoffice: in backoffice fields
    # (export_to.backoffice_filefield_id), and in history (export_to.attach_to_history)
    for index in (0, 1):
        resp = login(get_app(pub), username='admin', password='admin').get(
                '/backoffice/management/test/1/')
        resp = resp.click('template.odt', index=index)
        assert resp.location.endswith('/template.odt')
        resp = resp.follow()
        assert resp.content_type == 'application/octet-stream'
        with open(os.path.join(os.path.dirname(__file__), 'template-out.odt'), 'rb') as f:
            assert_equal_zip(BytesIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'

def test_formdata_generated_document_in_private_history(pub):
    user = create_user(pub)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    user.roles = [role.id]
    user.store()

    wf = Workflow(name='status')
    st0 = wf.add_status('Status0', 'st0')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = BytesIO()
    upload.fp.write(b'HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.attach_to_history = True
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1

    st2 = wf.add_status('Status2', 'st2')

    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_receiver']
    jump1.status = st1.id
    jump1.parent = st0
    st0.items.append(jump1)

    jump2 = ChoiceWorkflowStatusItem()
    jump2.id = '_jump2'
    jump2.label = 'Jump 2'
    jump2.by = ['_receiver']
    jump2.status = st2.id
    jump2.parent = st1
    st1.items.append(jump2)

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_jump1')
    resp = resp.follow()

    resp = resp.form.submit('button_export_to')
    resp = resp.follow()
    assert 'Form exported in a model' in resp.text

    resp = resp.form.submit('button_jump2')
    resp = resp.follow()

    # limit visibility of status with document
    st1.visibility = ['_receiver']
    wf.store()

    formdata = formdef.data_class().select()[0]
    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
    assert not 'Form exported in a model' in resp.text

    # check status is visible in backoffice
    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=True))
    assert 'visibility-off' in resp.text
    assert 'Form exported in a model' in resp.text

def test_formdata_form_file_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.FileField(id='1', label='File',
        type='file', varname='yyy'))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    assert 'qommon.fileupload.js' in resp.text
    resp.forms[0]['f1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'xxx_var_yyy_raw' in formdata.workflow_data

    download = resp.test_app.get(resp.location + 'files/form-xxx-yyy/test.txt')
    assert download.content_type == 'text/plain'
    assert download.body == b'foobar'

    # go back to the status page, this will exercise the substitution variables
    # codepath.
    resp = resp.follow()

def test_formdata_workflow_form_prefill(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.StringField(id='1', label='blah',
        type='string', varname='yyy', prefill={'type': 'user', 'value': 'email'}))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert resp.forms[0]['f1'].value == 'foo@localhost'

def test_formdata_workflow_form_prefill_conditional_field(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.StringField(id='1', label='blah1',
        type='string', varname='yyy1', prefill={'type': 'user', 'value': 'email'},
        condition={'type': 'django', 'value': '0'}))
    display_form.formdef.fields.append(fields.StringField(id='2', label='blah2',
        type='string', varname='yyy2', prefill={'type': 'user', 'value': 'email'},
        condition={'type': 'django', 'value': '1'}))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert resp.forms[0]['f2'].value == 'foo@localhost'


def test_form_map_field_back_and_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.StringField(id='1', label='street', required=True,
            prefill={'type': 'geolocation', 'value': 'road'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.text
    assert 'qommon.geolocation.js' in resp.text
    # with a real user interaction this would get set by javascript
    resp.forms[0]['f0$latlng'].value = '1.234;-1.234'
    assert 'data-geolocation="road"' in resp.text

    # check required field
    resp = resp.forms[0].submit('submit')
    assert not 'Check values then click submit.' in resp.text
    assert 'data-geolocation="road"' in resp.text
    resp.forms[0]['f1'].value = 'bla'

    # check summary page
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-init-lng="-1.234"' in resp.text
    assert 'data-init-lat="1.234"' in resp.text

    # get back to the map field
    resp = resp.forms[0].submit('previous')
    # check the field is still marked as holding the road
    assert 'data-geolocation="road"' in resp.text
    assert resp.forms[0]['f0$latlng'].value == '1.234;-1.234'

    # back to summary page
    resp = resp.forms[0].submit('submit')

    # and submitting the form
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'bla', '0': '1.234;-1.234'}

def test_form_map_field_prefill_address(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='address', required=True, varname='address'),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.MapField(id='3', label='map',
            prefill={'type': 'string', 'value': '{{ form_var_address }}'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'] = '169 rue du chateau, paris'
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (None, 200,
                json.dumps([{'lat':'48.8337085','lon':'2.3233693'}]), None)
        resp = resp.form.submit('submit')
        assert resp.form['f3$latlng'].value == '48.8337085;2.3233693'
        assert 'chateau' in http_get_page.call_args[0][0]

def test_form_map_multi_page(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.MapField(id='1', label='map'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1$latlng'] = '1.234;-1.234'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f1$latlng'].value == '1.234;-1.234'
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': '1.234;-1.234', '3': 'bar'}

def test_form_middle_session_change(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    app.cookiejar.clear()
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/'
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    app.cookiejar.clear()
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.text

def test_form_autocomplete_variadic_url(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.PageField(id='3', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'True'}),
            fields.ItemField(id='1', label='string', type='item',
                varname='foo', items=['Foo', 'Bar']),
            fields.StringField(id='2', label='string2', required=True,
                data_source={'type': 'jsonp', 'value': '[var_foo]'}),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'True'})
            ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit') # next
    # test javascript will be used to compute the full URL
    assert 'options.wcs_base_url' in resp.text
    assert 'jquery-ui.min.js' in resp.text

    # test going forward (will error out), check it's still a variadic URL (#9786)
    resp.form['f1'] = 'Foo'
    resp = resp.form.submit('submit')
    assert 'options.wcs_base_url' in resp.text

def test_form_page_formula_prefill_user_name(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'form_user_email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

def test_form_page_formula_prefill_session_user(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_user_email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    app = login(get_app(pub), username='foo', password='foo')
    for session in pub.session_manager.values():
        session.extra_user_variables = {'foo': 'bar'}
        session.store()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_var_user_foo'})]
    formdef.store()
    resp = app.get('/test/')
    assert resp.forms[0]['f0'].value == 'bar'

def test_form_date_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='string', type='date',
        required=False)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f0'] = '2015-01-01'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert time.strftime('%Y-%m-%d', data.data['0']) == '2015-01-01'

    # without filling the field
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

def test_form_jsonp_item_field(http_requests, pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='string', type='item',
                data_source={'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'}),
            ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert 'data-select2-url="http://remote.example.net/jsonp"' in resp.text
    assert 'select2.min.js' in resp.text

def test_form_string_regex_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string',
        validation={'type': 'regex', 'value': r'\d{5}$'}, required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # with an invalid input
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    assert 'invalid value' in resp.text

def test_form_text_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TextField(id='0', label='string', type='text', required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # check max length
    formdef.fields = [fields.TextField(id='0', label='string', type='text', maxlength=10)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'x' * 11
    resp = resp.forms[0].submit('submit')
    assert 'too many characters (limit is 10)' in resp.text
    # check it counts characters, not bytes
    resp.forms[0]['f0'] = u'☭' * 10
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text

def test_unknown_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.StringField(id='1', label='string',
                varname='string', required=False, data_source={'type': 'foobar'}),
        fields.ItemField(id='2', label='item',
                varname='item', required=False, data_source={'type': 'foobar'}),
        fields.ItemsField(id='3', label='items',
                varname='items', required=False, data_source={'type': 'foobar'}),
    ]

    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text

def test_form_items_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [fields.ItemsField(id='1', label='items',
        varname='items', required=False, data_source={'type': 'foobar'})]
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()

    # add the named data source
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.text
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert 'f1$elementun' in resp.form.fields
    assert 'f1$elementdeux' in resp.form.fields
    resp.form['f1$elementun'].checked = True
    resp.form['f1$elementdeux'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {'1': ['un', 'deux'], '1_display': 'un, deux'}

    data_source.data_source = {'type': 'formula',
            'value': repr([{'id': '1', 'text': 'un'},
                           {'id': '2', 'text': 'deux'}])}
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {'1': ['1', '2'], '1_display': 'un, deux'}

    data_source.data_source = {'type': 'formula',
            'value': repr([{'id': '1', 'text': 'un', 'foo': 'bar1'},
                           {'id': '2', 'text': 'deux', 'foo': 'bar2'}])}
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert data_class.select()[0].data == {
            '1': ['1', '2'],
            '1_structured': [{'text': 'un', 'foo': 'bar1', 'id': '1'},
                             {'text': 'deux', 'foo': 'bar2', 'id': '2'}],
            '1_display': 'un, deux'}
    # along the way, check substitution variables
    substvars = data_class.select()[0].get_substitution_variables()
    assert substvars['form_var_items'] == 'un, deux'
    assert substvars['form_var_items_raw'] == ['1', '2']
    assert substvars['form_var_items_0_foo'] == 'bar1'
    assert substvars['form_var_items_1_foo'] == 'bar2'

def test_form_ranked_items_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.RankedItemsField(
        id='0', label='ranked items',
        type='ranked-items', required=False,
        items=['foo', 'bar', 'baz'])]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0$element0'] = '1'
    resp.form['f0$element1'] = '2'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == {'bar': 2, 'foo': 1}

def test_form_ranked_items_randomize_order(pub):
    formdef = create_formdef()
    formdef.fields = [fields.RankedItemsField(
        id='0', label='ranked items',
        type='ranked-items', required=False,
        randomize_items=True,
        items=['foo', 'bar', 'baz'])]
    formdef.store()
    orders = {}
    for i in range(10):
        resp = get_app(pub).get('/test/')
        orders['%s-%s-%s' % (resp.text.index('foo'), resp.text.index('bar'), resp.text.index('baz'))] = True
    assert len(orders.keys()) > 1

def test_form_autosave(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'foobar'

    resp.form['f1'] = 'foobar2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar2'

    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    resp.form['f3'] = 'xxx'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx'

    resp.form['f3'] = 'xxx2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx2'

    resp.form['f3'] = 'xxx3'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'foobar3' in resp.text
    assert 'xxx3' in resp.text

    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx3'

    # make sure autosave() doesn't destroy data that would have been submitted
    # in the meantime
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'
    autosave_fields = resp.form.submit_fields()
    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    # post content with 'foobar' as value, it should not be saved
    ajax_resp = app.post('/test/autosave', params=autosave_fields)
    assert json.loads(ajax_resp.text)['result'] == 'error'
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

def test_form_autosave_with_items_field(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.ItemsField(id='3', label='items', type='items',
                items=[force_str(x) for x in (u'pomme', u'poire', u'pêche', u'abricot')]),
    ]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') == None

    resp = resp.forms[0].submit('submit')
    resp.form['f3$element1'].checked = True
    resp.form['f3$element3'].checked = True
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == ['poire', 'abricot']

def test_form_autosave_with_parameterized_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.ItemField(id='3', label='item', type='item',
                data_source={'type': 'formula',
                             'value': '''[('1', form_var_foo*2)]'''})]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') == None

    resp = resp.forms[0].submit('submit')
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == '1'
    assert formdef.data_class().select()[0].data['3_display'] == 'barbar'

def test_form_string_field_autocomplete(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string', required=False)]
    formdef.fields[0].data_source = {'type': 'jsonp'}
    formdef.store()

    # not filled completed, no call to .autocomplete
    resp = get_app(pub).get('/test/')
    assert not ').autocomplete({' in resp.text

    # straight URL
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': 'http://example.org'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.org' in resp.text

    # URL from variable
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': '[site_url]'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.text
    assert 'http://example.net' in resp.text

def test_form_workflow_trigger(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    jump2 = JumpWorkflowStatusItem()
    jump2.trigger = 'YYY'
    jump2.status = 'st3'
    jump2.set_marker_on_status = True
    st1.items.append(jump2)
    jump2.parent = st1

    st2 = workflow.add_status('Status2', 'st2')
    st3 = workflow.add_status('Status3', 'st3')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    resp = login(app, username='foo', password='foo').get('/')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    jump.by = [role.id]
    workflow.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    user.roles = [role.id]
    user.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st2'

    formdata.status = 'wf-st1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY', status=403)
    jump2.by = [role.id]
    workflow.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st3'
    assert formdata.workflow_data.get('_markers_stack') == [{'status_id': 'st1'}]

    formdata.status = 'wf-st1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY',
            params=json.dumps({'data': {'foo': 'bar'}}),
            content_type='application/json')
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.workflow_data.get('data') == {'foo': 'bar'}

def test_form_worklow_multiple_identical_status(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    st1.extra_css_class = 'CSS-STATUS1'
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st1'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)

    Role.wipe()
    role = Role(name='xxx')
    role.allows_backoffice_access = False
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    assert len(formdef.data_class().get(formdata.id).evolution) == 1
    assert formdef.data_class().get(formdata.id).evolution[0].last_jump_datetime is None

    login(app, username='foo', password='foo')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    # status is not changed: no new evolution, only a new last_jump_datetime
    assert len(formdata.evolution) == 1
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].last_jump_datetime is not None
    assert formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed'] is False
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # add a comment to last evolution, forcing create a new one
    formdata.evolution[-1].comment = 'new-evolution-1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 2
    assert formdata.status == 'wf-st1'
    assert formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed'] is False
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    # again
    formdata.evolution[-1].comment = 'new-evolution-2'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # last evolution is empty, this last trigger does not create a new one
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    # finally, 3 evolutions: new-evolution-1, new-evolution-2, empty
    formdata = formdef.data_class().get(formdata.id)
    assert len(formdata.evolution) == 3
    assert formdata.status == 'wf-st1'
    assert formdata.evolution[0].comment == 'new-evolution-1'
    assert formdata.evolution[1].comment == 'new-evolution-2'
    assert formdata.evolution[2].comment is None

    resp = app.get(formdata.get_url())
    assert resp.text.count('Status1') == 3 # once in summary and two in journal
    assert resp.text.count('CSS-STATUS1') == 2
    assert resp.text.count('new-evolution-1') == 1
    assert resp.text.count('new-evolution-2') == 1

def test_form_worklow_comments_on_same_status(pub):
    pub.session_manager.session_class.wipe()
    user = create_user(pub)

    role = Role(name='xxx')
    role.store()
    user.roles = [role.id]
    user.store()

    formdef = create_formdef()
    formdef.fields = []
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    workflow = Workflow.get_default_workflow()
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.perform_workflow()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-new'

    app = get_app(pub)

    assert formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed'] is True
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True
    login(app, username='foo', password='foo')
    resp = app.get(formdata.get_url()).follow()
    resp.form['comment'] = 'TEST COMMENT'
    resp = resp.form.submit('button_commentable')
    assert formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed'] is False
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is False

    resp = app.get(formdata.get_url()).follow()
    resp = resp.form.submit('button_accept')
    assert formdef.data_class().get(formdata.id).get_static_substitution_variables()['form_status_changed'] is True
    assert formdef.data_class().get(formdata.id).get_substitution_variables()['form_status_changed'] is True


def test_display_message(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    display2 = DisplayMessageWorkflowStatusItem()
    display2.message = 'message-to-submitter'
    display2.to = ['_submitter']
    st1.items.append(display2)
    display2.parent = st1

    display3 = DisplayMessageWorkflowStatusItem()
    display3.message = 'message-to-nobody'
    display3.to = ['xxx']
    st1.items.append(display3)
    display3.parent = st1

    display4 = DisplayMessageWorkflowStatusItem()
    display4.message = 'message-to-xxx-and-submitter'
    display4.to = ['_submitter', 'xxx']
    st1.items.append(display4)
    display4.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()

    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-nobody' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') < page.text.index('message-to-xxx-and-submitter')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    # actions alert vs top alert
    display2.position = 'actions'
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # add an action, so display2 will appear again
    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_submitter']
    jump1.status = st1.id
    jump1.parent = st1
    st1.items.append(jump1)
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    jump1.by = ['xxx']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' not in page.text
    assert 'message-to-xxx-and-submitter' in page.text

    # change to always display at the bottom
    display2.position = 'bottom'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text
    assert 'message-to-submitter' in page.text
    assert 'message-to-xxx-and-submitter' in page.text
    assert page.text.index('message-to-submitter') > page.text.index('message-to-xxx-and-submitter')

    # set a level
    display2.level = 'warning'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' in page.text

def test_workflow_condition_on_message(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()
    assert 'message-to-all' in page.text

    formdata = formdef.data_class().select()[0]
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.text

    display1.condition = {'type': 'django', 'value': 'xxx'}
    workflow.store()
    page = app.get(formdata.get_url())
    assert not 'message-to-all' in page.text

def test_session_cookie_flags(pub):
    formdef = create_formdef()
    app = get_app(pub)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('wcs-')
    assert 'httponly' in resp.headers['Set-Cookie']
    assert not 'secure' in resp.headers['Set-Cookie']

    app = get_app(pub, https=True)
    resp = app.get('/test/', status=200)
    assert resp.headers['Set-Cookie'].startswith('wcs-')
    assert 'httponly' in resp.headers['Set-Cookie']
    assert 'secure' in resp.headers['Set-Cookie']

def test_form_page_profile_verified_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert not 'readonly' in resp.form['f0'].attrs
    resp.form['f0'].value = 'Hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'Hello'

    user.verified_fields = ['email']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'readonly' in resp.form['f0'].attrs

    resp.form['f0'].value = 'Hello' # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'foo@localhost'  # it is reverted

    resp.form['f0'].value = 'Hello' # try again changing the value
    resp = resp.form.submit('submit')

    formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
    assert len(formdatas) == 1
    assert formdatas[0].data['0'] == 'foo@localhost'

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp.form['f0'].value = 'Hello' # try changing
    resp = resp.form.submit('previous')
    assert 'readonly' in resp.form['f0'].attrs
    assert not 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'foo@localhost'

def test_form_page_profile_verified_date_prefill(pub):
    user = create_user(pub)

    from wcs.admin.settings import UserFieldsFormDef
    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.DateField(id='_date', label='date', type='date'))
    user_formdef.store()
    user.form_data = {'_date': time.strptime('2018-09-27', '%Y-%m-%d')}
    user.set_attributes_from_formdata(user.form_data)
    user.store()

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.DateField(id='0', label='date',
        prefill={'type': 'user', 'value': '_date'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    assert not 'readonly' in resp.form['f0'].attrs
    resp.form['f0'].value = '2018-09-27'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'

    user.verified_fields = ['_date']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == '2018-09-27'
    assert 'readonly' in resp.form['f0'].attrs

    resp.form['f0'].value = '2018-09-24' # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == '2018-09-27'  # it is reverted

    resp.form['f0'].value = '2018-09-24' # try again changing the value
    resp = resp.form.submit('submit')

    formdatas = [x for x in formdef.data_class().select() if not x.is_draft()]
    assert len(formdatas) == 1
    assert time.strftime('%Y-%m-%d', formdatas[0].data['0']) == '2018-09-27'

def test_form_page_profile_verified_radio_item_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='item', type='item',
        items=['bar@localhost', 'foo@localhost', 'baz@localhost'],
        display_mode='radio',
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value is None

    user.verified_fields = ['email']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'disabled' in resp.form['f0'].attrs
    for radio in resp.html.findAll('input'):
        if radio['name'] == 'f0':
            if radio['value'] == 'foo@localhost':
                assert radio.attrs.get('checked')
                assert not radio.attrs.get('disabled')
            else:
                assert not radio.attrs.get('checked')
                assert radio.attrs.get('disabled')

    resp.form['f0'].value = 'baz@localhost' # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert resp.form['f0'].value == 'foo@localhost'  # it is reverted

def test_item_field_from_cards(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_template = '{{form_var_name}}'
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef.fields = [fields.ItemField(id='0', label='string', type='item',
        data_source=ds, display_disabled_items=True)]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].options == [
        (u'2', False, u'bar'),
        (u'3', False, u'baz'),
        (u'1', False, u'foo'),
    ]
    resp.form['f0'] = '2'
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submit
    assert formdef.data_class().select()[0].data['0'] == '2'
    assert formdef.data_class().select()[0].data['0_display'] == 'bar'
    assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'

def test_item_field_with_disabled_items(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemField(id='0', label='string', type='item',
        data_source=ds, display_disabled_items=True)]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert pq('option[disabled=disabled][value="1"]').text() == 'hello'
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit') # -> validation page
        assert 'There were errors processing the form' in resp.text

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='string', type='item', data_source=ds,
        display_disabled_items=False)]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('option[disabled=disabled][value="1"]')) == 0
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='string', type='item',
        data_source=ds, display_mode='radio', display_disabled_items=True)]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        pq = resp.pyquery.remove_namespaces()
        assert len(pq('input[name="f0"][disabled=disabled][value="1"]')) == 1
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit') # -> validation page
        assert 'There were errors processing the form' in resp.text

def test_items_field_with_disabled_items(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemsField(id='0', label='string', type='items',
        data_source=ds, display_disabled_items=True)]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['1', '2']
        assert formdef.data_class().select()[0].data['0_display'] == 'hello, world'

    formdef.data_class().wipe()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'disabled' in resp.form['f0$element1'].attrs
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemsField(id='0', label='string', type='items',
        data_source=ds, display_disabled_items=False)]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not 'f0$element1' in resp.form.fields
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

def test_item_field_autocomplete_json_source(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://remote.example.net/json'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'},
                         {'id': '2', 'text': 'world', 'extra': 'bar'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert resp.form['f0'].value == '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][1]

    # check hint is displayed within
    formdef.fields[0].hint = 'help text'
    formdef.store()
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'},
                         {'id': '2', 'text': 'world', 'extra': 'bar'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert 'data-autocomplete="true"' in resp.text
        assert 'data-hint="help text"' in resp.text
        assert resp.form['f0'].value == ''

    formdef.fields[0].hint = ''
    formdef.store()

    # check with possibility of remote query
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    formdef.data_class().wipe()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'},
                         {'id': '2', 'text': 'world', 'extra': 'bar'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?q=hell'
        assert resp2.json == data

        # check unauthorized access
        resp2 = get_app(pub).get(select2_url + '?q=hell', status=403)

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = resp.form.submit('submit') # -> validation page
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?id=1'
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = resp.form.submit('submit') # -> submit
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0] == 'http://remote.example.net/json?id=1'
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # same thing with signed URLs
    formdef.data_class().wipe()
    open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w').write('''\
[wscall-secrets]
remote.example.net = 1234
''')

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'},
                         {'id': '2', 'text': 'world', 'extra': 'bar'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp2 = app.get(select2_url + '?q=hell')
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?q=hell&orig=example.net&')
        assert resp2.json == data

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
    resp.form['f0'].force_value('1')
    resp.form.fields['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = resp.form.submit('submit') # -> validation page
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = resp.form.submit('submit') # -> submit
        assert urlopen.call_count == 1
        assert urlopen.call_args[0][0].startswith('http://remote.example.net/json?id=1&orig=example.net&')
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        assert formdef.data_class().select()[0].data['0_structured'] == data['data'][0]

    # check with optional field
    formdef.data_class().wipe()
    formdef.fields[0].required = False
    formdef.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'},
                         {'id': '2', 'text': 'world', 'extra': 'bar'}]}
        urlopen.side_effect = lambda *args: StringIO(json.dumps(data))
        resp = app.get('/test/')
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']

    resp = resp.form.submit('submit') # -> validation page
    assert resp.form['f0'].value == ''
    assert resp.form['f0_label'].value == ''

    resp = resp.form.submit('submit') # -> submit
    assert formdef.data_class().select()[0].data['0'] is None

def test_item_field_autocomplete_jsonp_source(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'}
    data_source.store()

    formdef.fields = [
        fields.ItemField(id='0', label='string', type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
        ),
    ]
    formdef.store()

    app = get_app(pub)
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = app.get('/test/')
        assert urlopen.call_count == 0
        pq = resp.pyquery.remove_namespaces()
        select2_url = pq('select').attr['data-select2-url']
        assert select2_url == 'http://remote.example.net/jsonp'

    # simulate select2 mode, with qommon.forms.js adding an extra hidden widget
    resp.form.fields['f0_display'] = [Hidden(form=resp.form, tag='input', name='f0_display', pos=10)]
    resp.form.field_order.append(('f0_display', resp.form.fields['f0_display'][0]))
    resp.form['f0'].force_value('1')
    resp.form['f0_display'].force_value('hello')

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit') # -> validation page
        assert urlopen.call_count == 0
        assert resp.form['f0'].value == '1'
        assert resp.form['f0_label'].value == 'hello'

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        resp = resp.form.submit('submit') # -> submit
        assert urlopen.call_count == 0
        assert formdef.data_class().select()[0].data['0'] == '1'
        assert formdef.data_class().select()[0].data['0_display'] == 'hello'
        # no _structured data for pure jsonp sources
        assert '0_structured' not in formdef.data_class().select()[0].data

def test_form_data_keywords(pub):
    formdef = create_formdef()
    formdef.keywords = 'hello,world'
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'data-keywords="hello world"' in resp.text
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert formdef.data_class().count() == 1

def test_logged_errors(pub):
    Workflow.wipe()
    workflow = Workflow.get_default_workflow()
    workflow.id = '12'
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = 'rejected'
    jump.condition = {'type': 'python', 'value': '1//0'}  # ZeroDivisionError
    st1 = workflow.possible_status[0]
    st1.items.insert(0, jump)
    jump.parent = st1
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.id = '34'
    formdef.workflow = workflow
    formdef.name = 'test'
    formdef.confirmation = False
    formdef.fields = []
    formdef.store()

    LoggedError.wipe()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    error = LoggedError.get_on_index(
        '34-12-just_submitted-_jump-failed-to-evaluate-condition-ZeroDivisionError-integer-division-or-modulo-by-zero',
        'tech_id')
    assert error.occurences_count == 2

    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', '34')) == 1
    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', 'X')) == 0

    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', '12')) == 1
    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', 'X')) == 0

def test_formdata_named_wscall(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    comment = RegisterCommenterWorkflowStatusItem()
    comment.id = '_comment'
    comment.comment = 'Hello [webservice.hello_world.foo] World'
    st1.items.append(comment)
    comment.parent = st1

    display = DisplayMessageWorkflowStatusItem()
    display.message = 'The form has been recorded and: X[webservice.hello_world.foo]Y'
    display.to = []
    st1.items.append(display)
    display.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'

    # check with publisher variable in named webservice call
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': '[example_url]json'}
    wscall.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'

def test_formdata_named_wscall_in_conditions(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json', 'method': 'GET'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(id='1', label='2nd page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'}),
        fields.PageField(id='1', label='3rd page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] != "bar"'}),
        fields.PageField(id='1', label='4th page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'}),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '>1st page<' in resp.text
    assert '>2nd page<' in resp.text
    assert '>3rd page<' not in resp.text
    assert '>4th page<' in resp.text
    assert len(http_requests.requests) == 1

def test_resubmit(pub):
    user = create_user(pub)

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='string', varname='toto')]
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'form title bis'
    formdef2.enable_tracking_codes = True
    formdef2.fields = [fields.StringField(id='1', label='string', varname='titi'),
                       fields.StringField(id='2', label='string', varname='toto')]
    formdef2.store()

    wf = Workflow(name='resubmit')
    st1 = wf.add_status('Status1')
    st2 = wf.add_status('Status2')

    resubmit = ResubmitWorkflowStatusItem()
    resubmit.id = '_resubmit'
    resubmit.by = ['_submitter']
    resubmit.formdef_slug = formdef2.url_name
    st1.items.append(resubmit)
    resubmit.parent = st1

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1

    register_comment = RegisterCommenterWorkflowStatusItem()
    register_comment.id = '_register'
    register_comment.comment = '<p><a href="[resubmit_formdata_draft_url]">new draft</a></p>'
    st2.items.append(register_comment)
    register_comment.parent = st2

    wf.store()

    formdef.workflow_id = wf.id
    formdef.store()

    formdef2.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.data = {'1': 'XXX'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'XXX'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'XXX'

    # anonymous
    formdef2.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/form-title/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit') # -> validation
    resp = resp.form.submit('submit') # -> submission
    resp = resp.follow()
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'foo'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'foo'

def test_form_custom_select_template(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Baz'])]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not 'TEST TEMPLATE' in resp.text
    formdef.fields[0].extra_css_class = 'template-test'
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' in resp.text
    # make sure request is available in context
    assert '<!-- backoffice: False -->' in resp.text
    assert '<!-- backoffice compat: False -->' in resp.text

    # test for substitution variables being available
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)
    resp = get_app(pub).get('/test/')
    assert 'substitution variable: http://remote.example.net/' in resp.text

def test_form_status_appearance_keywords(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Baz'])]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    root = PublicFormStatusPage(formdef, formdata, register_workflow_subdirs=False)
    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == root.status_templates

    formdef.appearance_keywords = 'foobar plop'
    formdef.store()

    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == [
        'wcs/front/appearance-foobar/formdata_status.html',
        'wcs/front/appearance-plop/formdata_status.html',
        'wcs/front/formdata_status.html',
        'wcs/appearance-foobar/formdata_status.html',
        'wcs/appearance-plop/formdata_status.html',
        'wcs/formdata_status.html']

    resp = get_app(pub).get('/test/')
    assert 'class="quixote foobar plop"' in resp.text

def test_user_global_action(pub):
    user = create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert not 'button-action-1' in resp.text

    trigger.roles = ['_submitter']
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'

def test_user_global_action_same_status_store(pub):
    user = create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'

    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string'),
    ]
    action = workflow.add_global_action('FOOBAR')
    jump = action.append_item('jump')
    jump.status = 'new'
    trigger = action.triggers[0]
    trigger.roles = ['_submitter']

    new_status = workflow.possible_status[1]

    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = new_status
    setbo.fields = [{'field_id': 'bo1', 'value': '123'}]
    new_status.items = [setbo] + new_status.items

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['bo1'] == '123'

    # change global action
    setbo.fields = [{'field_id': 'bo1', 'value': '321'}]
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')  # click global action

    # check status actions are not rerun
    resp = app.get(formdata.get_url())
    assert formdef.data_class().get(formdata.id).status == 'wf-new'
    assert formdef.data_class().get(formdata.id).data['bo1'] == '123'

def test_condition_on_action(pub, emails):
    user = create_user(pub)

    workflow = Workflow.get_default_workflow()
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.id = '2'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test condition on action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form (test condition on action)')  # no receiver
    assert emails.get('New form2 (test condition on action)')  # submitter

    emails.empty()

    workflow.possible_status[0].items[1].condition = {'type': 'python', 'value': 'False'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with a condition on field data
    formdef.fields = [fields.StringField(id='0', label='string', varname='foobar')]
    formdef.store()
    workflow.possible_status[0].items[1].condition = {'type': 'django', 'value': 'form_var_foobar'}
    workflow.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = ''
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert not emails.get('New form2 (test condition on action)')

    # check with condition evaluating positively
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp.form['f0'] = 'toto'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    assert emails.get('New form2 (test condition on action)')

def test_email_actions(pub, emails):
    user = create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    # change email subjects to differentiate them
    workflow.possible_status[0].items[0].subject = 'New form ([name])'
    workflow.possible_status[0].items[1].subject = 'New form2 ([name])'
    workflow.possible_status[0].items[1].body = 'Hello; {% action_button "do-accept" label="Accepté!" %} Adiós.'
    workflow.possible_status[1].items[1].identifier = 'do-accept'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'test email action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    assert '/actions/' in action_url
    if docutils:
        assert len(email_data['payloads']) == 2
        assert action_url in email_data['payloads'][1]

    app = get_app(pub)
    resp = app.get(action_url)
    assert u'Accepté!' in resp.text
    resp = resp.form.submit()
    assert 'The action has been confirmed.' in resp.text
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'wf-accepted'

    # action token has been used, it will now return a custom 404
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # check against independently changed status, it should also return a
    # custom 404.
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    action_url = re.findall(r'http.* ', email_data['payload'])[0].strip()
    formdata = formdef.data_class().select()[0]
    formdata.jump_status('rejected')
    app = get_app(pub)
    resp = app.get(action_url, status=404)
    assert 'This action link has already been used or has expired.' in resp.text

    # two buttons on the same line, two urls
    workflow.possible_status[0].items[1].body = '{% action_button "ok" label="OK" %}{% action_button "ko" label="KO" %} '
    workflow.store()
    emails.empty()
    formdef.data_class().wipe()
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')
    email_data = emails.get('New form2 (test email action)')
    assert len(re.findall(r'http.*? ', email_data['payload'])) == 2

def test_manager_public_access(pub):
    user, manager = create_user_and_admin(pub)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    manager.is_admin = False
    manager.roles = [role.id]
    manager.store()
    assert manager.can_go_in_backoffice()

    formdef = create_formdef()
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    # user access to own formdata
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to formdata
    app = login(get_app(pub), username='admin', password='admin')
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    # agent access to an unauthorized formdata
    formdef.workflow_roles = {'_receiver': None}
    formdef.store()
    resp = app.get(formdata.get_url(), status=403)

    # agent access via a tracking code (stays in frontoffice)
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.enable_tracking_codes = True
    formdef.store()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp = app.get('/code/%s/load' % code.id)
    resp = resp.follow() # -> /test/1
    assert not 'backoffice' in resp.location
    resp = resp.follow() # -> /test/1/
    assert 'The form has been recorded' in resp.text

    # authorized access but not backoffice access
    app = login(get_app(pub), username='admin', password='admin')  # reset session
    resp = app.get(formdata.get_url())
    assert resp.location == formdata.get_url(backoffice=True)  # check tracking code is no longer effective
    role.allows_backoffice_access = False
    role.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

    # agent access to own formdata (stays in frontoffice)
    formdata = formdef.data_class()()
    formdata.user_id = manager.id
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    resp = app.get(formdata.get_url())
    assert 'The form has been recorded' in resp.text

def test_form_and_category_same_slug(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.store()

    # check we get to the form, not the category
    resp = get_app(pub).get('/foobar/')
    assert resp.form

def test_field_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True,
            condition={'type': 'django', 'value': '1'}),
        fields.StringField(type='string', id='2', label='Foo', size='40',
            required=True,
            condition={'type': 'django', 'value': '0'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' not in resp.form.fields
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' not in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' not in resp.text

def test_field_unicode_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(type='string', id='4', label='Baz', size='40',
            required=True, varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar == "éléphant"'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert not 'f4' in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields

def test_field_unicode_condition_contains_in_list(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(type='string', id='4', label='Baz', size='40',
            required=True, varname='baz',
            condition={'type': 'django', 'value': 'form_var_bar in "éléphant"|split'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert not 'f4' in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields

def test_field_unicode_condition_contains_in_string(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(type='string', id='4', label='Baz', size='40',
            required=True, varname='baz',
            condition={'type': 'django', 'value': '"éléphant" in form_var_bar'}),
    ]
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert not 'f4' in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1'] = 'éléphant'
    resp = resp.form.submit('submit')
    assert 'f4' in resp.form.fields

def test_field_unicode_condition_in_array(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemsField(id='1', label='items', type='items',
                required=True,
                varname='foo', items=['Pomme', 'Poire', 'Pêche', 'Abricot']),
        fields.PageField(id='2', label='2nd page', type='page'),
        fields.StringField(type='string', id='3', label='Baz', size='40',
            required=True, varname='baz',
            condition={'type': 'django', 'value': '"Pêche" in form_var_foo'}),
        fields.CommentField(type='comment', id='4', label='{{form_var_foo}}'),
        fields.CommentField(type='comment', id='5', label='{% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'),
    ]

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'Message {% if "Pêche" in form_var_foo %}CHECK OK{% endif %}'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1
    workflow.store()

    formdef.workflow = workflow
    formdef.store()

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp = resp.form.submit('submit')
    assert not 'f3' in resp.form.fields

    resp = get_app(pub).get('/foo/')
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'f3' in resp.form.fields  # check it's ok in field condition
    resp.form['f3'] = 'hop'
    assert u'>Poire, Pêche<' in resp.text  # check it's displayed correctly
    assert 'CHECK OK' in resp.text  # check it's ok in template condition
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.form.submit('submit').follow()
    assert '<p>Message CHECK OK</p>' in resp.text  # check it's ok in workflow template

def test_field_live_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Foo', size='40',
            required=True, varname='foo',
            condition={'type': 'django', 'value': 'form_var_bar == "bye"'}),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'bye'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' not in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' not in resp.text

    resp = get_app(pub).get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    resp.form['f1'] = 'bye'
    resp = resp.form.submit('submit')
    assert 'There were errors' in resp.text
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') is None
    resp.form['f2'] = 'bye'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' in resp.text

def test_field_live_items_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.ItemsField(type='items', id='1', label='Bar',
            items=['a', 'b'], varname='bar'),
        fields.StringField(type='string', id='2', label='Foo', size='40',
            required=True, varname='foo',
            condition={'type': 'django', 'value': '"b" in form_var_bar'}),
    ]
    formdef.store()

    user = create_user(pub)
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/foo/')
    assert 'f1$element0' in resp.form.fields
    assert 'f1$element1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1$element0'].checked = True
    app.post('/foo/autosave', params=resp.form.submit_fields())
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']
    resp.form['f1$element1'].checked = True
    app.post('/foo/autosave', params=resp.form.submit_fields())
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp.form['f1$element0'].checked = False
    resp.form['f1$element1'].checked = False
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']

def test_field_condition_on_required_field(pub):
    # from https://dev.entrouvert.org/issues/27247
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.ItemField(type='item', id='1', label='Bar',
            items=['oui', 'non'],
            display_mode='radio',
            required=True, varname='bar'),
        fields.ItemField(type='item', id='2', label='Foo', size='40',
            required=True,
            hint='---',
            items=['plop'],
            condition={'type': 'django', 'value': 'form_var_bar == "oui"'}),
        fields.PageField(id='3', label='1st page', type='page',
            condition={'type': 'python', 'value': 'True'}),
        fields.CommentField(type='comment', id='4', label='HELLO!'),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1'] = 'non'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']
    resp = resp.form.submit('submit')
    assert 'HELLO' in resp.text
    resp = resp.form.submit('previous')
    resp.form['f1'] = 'oui'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp = resp.form.submit('submit')
    assert '<div class="error">required field</div>' in resp.text
    assert 'HELLO' not in resp.text

def test_field_live_condition_multipages(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Foo', size='40',
            required=True, varname='foo',
            condition={'type': 'django', 'value': 'form_var_bar == "bye"'}),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(type='string', id='4', label='Baz', size='40',
            required=True, varname='baz'),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'bye'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'bye'
    resp.form['f2'] = 'bye'
    resp = resp.form.submit('submit')
    resp = resp.form.submit('previous')
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') is None
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp = resp.form.submit('submit')
    resp.form['f4'] = 'plop'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' in resp.text
    assert 'name="f4"' in resp.text
    resp = resp.form.submit('submit')

def test_field_live_select_content(pub, http_requests):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Bar2', size='40',
            required=True, varname='bar2'),
        fields.ItemField(type='item', id='3', label='Foo',
            data_source={
                'type': 'json',
                'value': '{% if form_var_bar2 %}http://remote.example.net/json-list?plop={{form_var_bar2}}{% endif %}'
            }),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '2'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '3'}).find('select')
    resp.form['f1'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    assert live_resp.json['result']['3']['visible']
    assert not 'items' in live_resp.json['result']['3']
    resp.form['f2'] = 'plop'
    live_resp = app.post('/foo/live?modified_field_id=2', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    assert live_resp.json['result']['3']['visible']
    assert 'items' in live_resp.json['result']['3']
    resp.form['f3'].options = []
    for item in live_resp.json['result']['3']['items']:
        # simulate javascript filling the <select>
        resp.form['f3'].options.append((item['id'], False, item['text']))
    resp.form['f3'] = 'a'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' in resp.text
    assert 'name="f3"' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'hello'
    assert formdata.data['2'] == 'plop'
    assert formdata.data['3'] == 'a'
    assert formdata.data['3_display'] == 'b'

def test_field_live_select_content_on_workflow_form(pub, http_requests):
    create_user(pub)
    wf = Workflow(name='wf-title')
    st1 = wf.add_status('Status1', 'st1')

    # form displayed into the workflow
    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Bar2', size='40',
            required=True, varname='bar2'),
        fields.ItemField(type='item', id='3', label='Foo',
            required=False, varname='foo',
            data_source={
                'type': 'json',
                'value': '{% if xxx_var_bar2 %}http://remote.example.net/json-list?plop={{xxx_var_bar2}}{% endif %}'
            }),
    ]
    st1.items.append(display_form)
    display_form.parent = st1
    wf.store()

    # initial empty form
    formdef = create_formdef()
    formdef.fields = []
    formdef.confirmation = False
    formdef.workflow_id = wf.id
    formdef.store()
    formdef.data_class().wipe()

    app = get_app(pub)
    resp = login(app, username='foo', password='foo').get('/test/')
    assert 'Forms - test' in resp.text

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert 'data-live-url' in resp.html.find('form').attrs
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '2'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '3'}).find('select')
    resp = resp.form.submit('submit')  # submit with error, to check <form> attributes
    assert 'data-live-url' in resp.html.find('form').attrs
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '2'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '3'}).find('select')
    resp.form['f1'] = 'hello'
    live_resp = app.post('/test/1/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    assert live_resp.json['result']['3']['visible']
    assert not 'items' in live_resp.json['result']['3']
    resp.form['f2'] = 'plop'
    live_resp = app.post('/test/1/live?modified_field_id=2', params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    assert live_resp.json['result']['3']['visible']
    assert 'items' in live_resp.json['result']['3']
    assert len(live_resp.json['result']['3']['items']) > 0

    resp.form['f3'].options = []
    for item in live_resp.json['result']['3']['items']:
        # simulate javascript filling the <select>
        resp.form['f3'].options.append((item['id'], False, item['text']))
    resp.form['f3'] = 'a'

    resp = resp.form.submit('submit')
    assert 'invalid value selected' not in resp
    resp = resp.follow()
    formdata = formdef.data_class().select()[0]
    assert formdata.workflow_data['xxx_var_bar'] == 'hello'
    assert formdata.workflow_data['xxx_var_bar2'] == 'plop'
    assert formdata.workflow_data['xxx_var_foo_raw'] == 'a'
    assert formdata.workflow_data['xxx_var_foo'] == 'b'

def test_field_live_comment_content(pub, http_requests):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Baz', size='40'),
        fields.CommentField(id='5', label='bla {{form_var_bar}} bla', type='comment'),
        fields.StringField(type='string', id='6', label='Bar2', size='40',
            required=True, varname='bar2'),
        fields.CommentField(id='7', label='bla {{form_var_bar2}} bla', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    resp.form['f1'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['5']['content'] == '<p>bla hello bla</p>'
    resp.form['f1'] = 'toto'
    live_resp = app.post('/foo/live?modified_field_id=1', params=resp.form.submit_fields())
    assert live_resp.json['result']['5']['content'] == '<p>bla toto bla</p>'
    live_resp = app.post('/foo/live?modified_field_id=2', params=resp.form.submit_fields())
    assert live_resp.json['result']['5']['content'] == '<p>bla toto bla</p>'

    # check evaluation of later fields
    # <https://dev.entrouvert.org/issues/31922>
    resp = app.get('/foo/')
    resp.form['f1'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['5']['content'] == '<p>bla hello bla</p>'
    resp.form['f6'] = 'hello'
    live_resp = app.post('/foo/live', params=resp.form.submit_fields())
    assert live_resp.json['result']['7']['content'] == '<p>bla hello bla</p>'

def test_form_edit_and_backoffice_field_change(pub):
    create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page')
            ]
    formdef.store()
    formdef.data_class().wipe()

    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='bo field 1', type='string',
            varname='plop'),
    ]
    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_foo'}]
    setbo2 = SetBackofficeFieldsWorkflowStatusItem()
    setbo2.parent = st1
    setbo2.fields = [{'field_id': 'bo1', 'value': '="foo" + form_var_plop'}]
    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items = [setbo, setbo2, jump]

    st2 = workflow.add_status('Status2', 'st2')

    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter']
    st2.items.append(editable)
    editable.parent = st2
    editable.status = st1.id
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f1'] = 'bar'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit')  # -> validation
    resp = resp.form.submit('submit').follow()  # -> submitted
    assert 'The form has been recorded' in resp.text

    data_id = formdef.data_class().select()[0].id
    assert formdef.data_class().get(data_id).data['bo1'] == 'foobar'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % data_id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable')
    resp = resp.follow()
    assert resp.form['f1'].value == 'bar'
    resp.form['f1'].value = 'baz'
    resp = resp.form.submit('submit')  # -> page 2
    resp = resp.form.submit('submit').follow()  # -> saved

    assert formdef.data_class().get(data_id).data['bo1'] == 'foobaz'


def test_field_live_condition_unknown_page_id(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'Foo'
    formdef.fields = [
        fields.PageField(id='0', label='2nd page', type='page'),
        fields.StringField(type='string', id='1', label='Bar', size='40',
                           required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Foo', size='40',
                           required=True, varname='foo',
                           condition={'type': 'django', 'value': 'form_var_bar == "bye"'}),
        fields.PageField(id='3', label='1st page', type='page'),
        fields.StringField(type='string', id='4', label='Baz', size='40',
                           required=True, varname='baz'),
    ]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/foo/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1'] = 'hello'
    params = resp.form.submit_fields()
    params = [(key, value if key != 'page_id' else 'eiuiu') for key, value in params]
    app.post('/foo/live', params=params)

def test_backoffice_fields_just_after_conditional_form_submit(pub):
    """
    simulate selection of a structured list via condition on form,
    followed by an evaluation on workflow in order to get structured value
    from the selected list.
    ie: test unfeed on ConditionVars
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(
            id='bo1', label='first text', type='string', varname='both_text'),
        fields.StringField(
            id='bo2', label='first more', type='string', varname='both_more')
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [
        {'field_id': 'bo1',
         'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2',
         'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'}
         ]
    st1.items.append(setbo)
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more':'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more':'moreB1'},
               {'id': '2', 'text': 'B2', 'more':'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'],
                         type='item', label='list to choice'),
        fields.ItemField(id='2', varname='listA', type='item', label='list A',
                         data_source={'type': 'formula', 'value': str(items_A)},
                         condition={'type': 'python',
                                    'value': 'form_var_choice_raw == "A"'}),
        fields.ItemField(id='3', varname='listB', type='item', label='list B',
                         data_source={'type': 'formula', 'value': str(items_B)},
                         condition={'type': 'python',
                                    'value': 'form_var_choice_raw == "B"'})
        ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user_and_admin(pub)
    resp = get_app(pub).get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'

def test_backoffice_fields_just_after_conditional_form_edit_action(pub):
    """
    test unfeed on ConditionVars within edit context
    """
    Workflow.wipe()
    workflow = Workflow(name='test')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(
            id='bo1', label='both text', type='string', varname='both_text'),
        fields.StringField(
            id='bo2', label='both more', type='string', varname='both_more')
    ]

    st1 = workflow.add_status('Status1', 'st1')
    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [
        {'field_id': 'bo1',
         'value': '{{ form_var_listA }} vs {{ form_var_listB }}'},
        {'field_id': 'bo2',
         'value': '{{ form_var_listA_more }} vs {{ form_var_listB_more }}'}
         ]
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter']
    editable.parent = st1
    editable.status = st1.id
    st1.items = [setbo, editable]
    workflow.store()

    items_A = [{'id': '1', 'text': 'A1', 'more':'moreA1'}]
    items_B = [{'id': '1', 'text': 'B1', 'more':'moreB1'},
               {'id': '2', 'text': 'B2', 'more':'moreB2'}]
    formdef = create_formdef()
    formdef.fields = [
        fields.ItemField(id='1', varname='choice', items=['A', 'B'],
                         type='item', label='list to choice'),
        fields.ItemField(id='2', varname='listA', type='item', label='list A',
                         data_source={'type': 'formula', 'value': str(items_A)},
                         condition={'type': 'python',
                                    'value': 'form_var_choice_raw == "A"'}),
        fields.ItemField(id='3', varname='listB', type='item', label='list B',
                         data_source={'type': 'formula', 'value': str(items_B)},
                         condition={'type': 'python',
                                    'value': 'form_var_choice_raw == "B"'})
        ]
    formdef.confirmation = False
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    create_user(pub)
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')

    resp.form['f1'].value = 'B'
    resp.form['f2'].value = '1'
    resp.form['f3'].value = '2'

    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'B'
    assert formdata.data.get('2') is None
    assert formdata.data['3'] == '2'

    # check unfeed on FormPage::submitted()
    assert formdata.data['bo1'] == 'None vs B2'
    assert formdata.data['bo2'] == ' vs moreB2'

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/%s/' % formdata.id)
    assert 'button_editable-button' in resp.text

    resp = resp.form.submit('button_editable').follow()
    assert resp.form['f1'].value == 'B'
    resp.form['f1'].value = 'A'
    resp = resp.form.submit('submit').follow()  # -> saved
    assert 'The form has been recorded' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.data['1'] == 'A'
    assert formdata.data['2'] == '1'
    assert formdata.data.get('3') is None

    # check unfeed on FormPage::submitted_existing()
    assert formdata.data['bo1'] == 'A1 vs None'
    assert formdata.data['bo2'] == 'moreA1 vs '

def test_form_recall_draft(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert not 'You already started to fill this form.' in resp.text

    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s"' % draft.id in resp.text

    draft2 = formdef.data_class()()
    draft2.user_id = user.id
    draft2.status = 'draft'
    draft2.data = {}
    draft2.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You already started to fill this form.' in resp.text
    assert 'href="%s"' % draft.id in resp.text
    assert 'href="%s"' % draft2.id in resp.text

def test_frontoffice_workflow_form_with_conditions(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2', type='string', required=True),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    # check with static condition
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': '0'}),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields
    assert 'f2' not in resp.form.fields

    # check condition based on formdata
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'form_var_plop'}),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'form_var_plop != "xxx"'}),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    # check with live conditions
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'blah_var_str == "xxx"'}),
    ]
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['f1'] = ''
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']

    resp.form['f1'] = 'xxx'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']

    # check submit doesn't work
    resp = resp.form.submit('submit')
    assert 'There were errors processing your form.' in resp.text

    resp.form['f1'] = 'xxx2'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']

    # check submit does work when second field is hidden
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().get(formdata.id).workflow_data == {'blah_var_str': 'xxx2', 'blah_var_str2': None}

def test_frontoffice_workflow_form_with_dynamic_comment(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.CommentField(id='2', label='value is {{blah_var_str}}', type='comment'),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='0', label='string', varname='plop')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.data = {'0': 'plop'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'f1' in resp.form.fields

    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['f1'] = 'test'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['2']['visible']
    assert live_resp.json['result']['2']['content'] == '<p>value is test</p>'

def test_frontoffice_workflow_form_with_impossible_condition(pub):
    user = create_user(pub)
    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = ['_submitter']
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string',
                condition={'type': 'django', 'value': '0 == 1'}),
            fields.StringField(id='2', label='Test2', type='string',
                condition={'type': 'django', 'value': 'blah_var_str == "toto"'}),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-new'
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url(backoffice=False))
    assert 'f1' not in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'

def test_choice_button_ignore_form_errors(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.roles = [logged_users_role().id]
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    st2 = wf.add_status('Status2', 'st2')

    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = [logged_users_role().id]
    commentable.required = True
    st1.items.append(commentable)
    commentable.parent = st1

    choice = ChoiceWorkflowStatusItem()
    choice.label = 'Submit'
    choice.by = [logged_users_role().id]
    choice.id = '_x1'
    choice.status = st2.id
    st1.items.append(choice)
    choice.parent = st1

    choice2 = ChoiceWorkflowStatusItem()
    choice2.label = 'Submit no check'
    choice2.by = [logged_users_role().id]
    choice2.id = '_x2'
    choice2.status = st2.id
    choice2.ignore_form_errors = True
    st1.items.append(choice2)
    choice2.parent = st1

    wf.store()

    formdef.workflow = wf
    formdef.store()

    # no comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x1')
    assert 'There were errors processing your form.' in resp.text

    # comment
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp.form['comment'] = 'plop'
    resp = resp.form.submit('button_x1').follow()
    assert '<p>plop</p>' in resp.text
    assert '<span class="status">Status2' in resp.text

    # no comment but no check
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')  # -> validation page
    resp = resp.form.submit('submit')  # -> submission
    resp = resp.follow()

    resp = resp.form.submit('button_x2').follow()
    assert '<span class="status">Status2' in resp.text