wcs/tests/test_form_pages.py

# -*- coding: utf-8 -*-

import json
import pytest
import hashlib
import os
import re
import StringIO
import time
from urlparse import urlparse
import zipfile
import base64
from webtest import Upload
import mock

try:
    from PIL import Image
except ImportError:
    Image = None

from quixote.http_request import Upload as QuixoteUpload
from wcs.qommon.form import UploadedFile
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.formdef import FormDef
from wcs.workflows import (Workflow, EditableWorkflowStatusItem,
        DisplayMessageWorkflowStatusItem, WorkflowBackofficeFieldsFormDef,
        ChoiceWorkflowStatusItem, JumpOnSubmitWorkflowStatusItem)
from wcs.wf.export_to_model import ExportToModel, transform_to_pdf
from wcs.wf.jump import JumpWorkflowStatusItem
from wcs.wf.attachment import AddAttachmentWorkflowStatusItem
from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
from wcs.wf.resubmit import ResubmitWorkflowStatusItem
from wcs.categories import Category
from wcs.roles import Role, logged_users_role
from wcs.tracking_code import TrackingCode
from wcs.data_sources import NamedDataSource
from wcs.wscalls import NamedWsCall
from wcs import fields
from wcs.sessions import BasicSession
from wcs.logged_errors import LoggedError
from wcs.forms.root import PublicFormStatusPage

from utilities import get_app, login, create_temporary_pub, clean_temporary_pub


def assert_equal_zip(stream1, stream2):
    z1 = zipfile.ZipFile(stream1)
    z2 = zipfile.ZipFile(stream2)
    assert set(z1.namelist()) == set(z2.namelist())
    for name in z1.namelist():
        t1, t2 = z1.read(name), z2.read(name)
        assert t1 == t2, 'file "%s" differs' % name


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql'], indirect=True)

@pytest.fixture
def pub(request, emails):
    pub = create_temporary_pub(sql_mode=(request.param == 'sql'))
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    if Category.count() == 0:
        cat = Category(name='foobar')
        cat.store()

    return pub


def teardown_module(module):
    clean_temporary_pub()


def create_formdef():
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()
    return formdef

def create_user(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()
    return user

def create_user_and_admin(pub):
    pub.user_class.wipe()
    PasswordAccount.wipe()

    user = pub.user_class()
    user.email = 'foo@localhost'
    user.store()
    account = PasswordAccount(id='foo')
    account.set_password('foo')
    account.user_id = user.id
    account.store()

    admin = pub.user_class()
    admin.email = 'admin@localhost'
    admin.is_admin = True
    admin.store()
    account = PasswordAccount(id='admin')
    account.set_password('admin')
    account.user_id = admin.id
    account.store()
    return user, admin

def test_home(pub):
    create_formdef()
    home = get_app(pub).get('/')
    assert 'category-misc' in home.body
    assert '<a class="" href="test/">test</a>' in home.body

def test_home_with_user_forms(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.enable_tracking_codes = True
    formdef.category_id = '1'
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    wf.store()
    formdef.workflow_id = wf.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.status = 'wf-st1'
    formdata.data = {}
    formdata.store()
    draft = formdef.data_class()()
    draft.user_id = user.id
    draft.status = 'draft'
    draft.data = {}
    draft.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="test/%s/"' % formdata.id in resp
    assert 'Draft' in resp
    assert '<a href="test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s' % draft.id, status=302)
    resp = resp.follow(status=200)

    # disable formdef: formdatas are still visible and accessible, drafts are not
    formdef.disabled = True
    formdef.store()
    resp = app.get('/')
    assert 'Status1' in resp
    assert '<a href="test/%s/"' % formdata.id in resp
    assert not 'Draft' in resp
    assert not '<a href="test/%s"' % draft.id in resp
    resp = app.get('/test/%s' % formdata.id)
    resp.status_int = 200
    resp = app.get('/test/%s' % draft.id, status=302)
    resp = resp.follow(status=403)

def test_home_category(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'category-foobar' in home.body
    assert not 'category-misc' in home.body
    assert '<a class="" href="foobar/test/">test</a>' in home.body

def test_home_two_categories(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'foobar'
    formdef2.fields = []
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.body # 1st formdef
    assert 'category-misc' in resp.body   # 2nd formdef, fake category
    assert '<a class="" href="foobar/test/">test</a>' in resp.body

    cat2 = Category(name='barfoo')
    cat2.store()
    formdef2.category_id = cat2.id
    formdef2.store()

    resp = get_app(pub).get('/')
    assert 'category-foobar' in resp.body # 1st formdef
    assert 'category-barfoo' in resp.body # 2nd formdef
    assert not 'category-misc' in resp.body # no more "misc" category

def test_home_keywords(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.keywords = 'hello, world'
    formdef.store()
    home = get_app(pub).get('/')
    assert '<div data-keywords="hello ' in home.body or '<div data-keywords="world ' in home.body
    assert '<li data-keywords="hello ' in home.body or '<li data-keywords="world ' in home.body

def test_home_formdef_description(pub):
    formdef = create_formdef()
    formdef.description = 'HELLO WORLD'
    formdef.store()
    home = get_app(pub).get('/')
    assert 'HELLO WORLD' in home.body
    assert '<a class="" href="test/">test</a>' in home.body

def test_home_disabled(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.store()
    home = get_app(pub).get('/')
    assert not '<a href="test/">test</a>' in home.body

def test_home_disabled_with_redirect(pub):
    formdef = create_formdef()
    formdef.disabled = True
    formdef.disabled_redirection = 'http://example.org'
    formdef.store()
    resp = get_app(pub).get('/')
    assert '<a class="redirection" href="test/">test</a>' in resp.body
    resp = resp.click('test')
    assert resp.location == 'http://example.org'

def test_home_inaccessible(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.store()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://example.net/login/?next=http%3A%2F%2Fexample.net%2F'

def test_home_always_advertise(pub):
    formdef = create_formdef()
    formdef.roles = ['xxx']
    formdef.always_advertise = True
    formdef.store()
    home = get_app(pub).get('/')
    assert '<a href="test/">test</a>' in home.body
    assert '<a href="test/">test</a><span> (authentication required)</span>' in home.body

def test_home_redirect(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/'

def test_home_redirect_var(pub):
    pub.cfg['misc']['homepage-redirect-url'] = 'http://www.example.com/[site_lang]/'
    pub.write_cfg()
    create_formdef()
    home = get_app(pub).get('/')
    assert home.status_int == 302
    assert home.location == 'http://www.example.com/en/'

def test_category_page(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    resp = get_app(pub).get('/foobar/')
    assert '<h2>foobar</h2>' in resp.body
    assert '<a class="" href="test/">test</a>' in resp.body

def test_category_page_redirect(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/'

def test_category_page_redirect_var(pub):
    formdef = create_formdef()
    formdef.category_id = '1'
    formdef.store()
    cat = Category.get(1)
    cat.redirect_url = 'http://www.example.com/[site_lang]/[category_slug]/'
    cat.store()
    resp = get_app(pub).get('/foobar/')
    assert resp.status_int == 302
    assert resp.location == 'http://www.example.com/en/foobar/'

def test_form_access(pub):
    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    # check a formdef protected by a role cannot be accessed
    formdef.roles = [role.id]
    formdef.store()
    # an unlogged user will ge ta redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # while a logged-in user will get a 403
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=403)

    # unless the user has the right role
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check admin has access, even without specific roles
    user = create_user(pub)
    user.roles = []
    user.is_admin = True
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

    # check special "logged users" role
    formdef.roles = [logged_users_role().id]
    formdef.store()
    user = create_user(pub)
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)
    resp = get_app(pub).get('/test/', status=302) # redirect to login

    # check "receiver" can also access the formdef
    formdef = create_formdef()
    formdef.roles = [-2]
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    user = create_user(pub)
    user.roles = [role.id]
    user.store()
    login(get_app(pub), username='foo', password='foo').get('/test/', status=200)

def test_form_access_auth_context(pub):
    user = create_user(pub)

    pub.load_site_options()
    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
    pub.site_options.set('options', 'auth-contexts', 'fedict')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    formdef = create_formdef()
    get_app(pub).get('/test/', status=200)

    formdef.required_authentication_contexts = ['fedict']
    formdef.roles = [logged_users_role().id]
    formdef.store()

    # an unlogged user will get a redirect to login
    resp = get_app(pub).get('/test/', status=302)
    assert '/login' in resp.location

    # a user logged in with a simple username/password tuple will get a page
    # to relogin with a stronger auth
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' in resp.body

    for session in pub.session_manager.values():
        session.saml_authn_context = 'urn:oasis:names:tc:SAML:2.0:ac:classes:SmartcardPKI'
        session.store()
    resp = app.get('/test/')
    assert 'You need a stronger authentication level to fill this form.' not in resp.body
    assert resp.form

def test_form_submit(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert not 'None' in next_page.body
    assert formdef.data_class().count() == 1

def test_form_submit_no_confirmation(pub):
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert formdef.data_class().count() == 1

def test_form_string_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.body
    next_page.forms[0]['f0'] = 'foobar'
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': 'foobar'}

def test_form_items_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.ItemsField(id='0', label='items', type='items',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Three']),]
    formdef.store()
    formdef.data_class().wipe()
    page = get_app(pub).get('/test/')
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.body
    next_page.forms[0]['f0$element0'].checked = True
    next_page.forms[0]['f0$element1'].checked = True
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == ['Foo', 'Bar']
    assert data.data['0_display'] == 'Foo, Bar'

def assert_current_page(resp, page_label):
    assert re.findall('<li class=".*?current.*?">.*?<span class="label">(.*?)</span></li>',
            resp.body)[0] == page_label

def test_form_multi_page(pub):
    for initial_condition in (None, 'True'):
        formdef = create_formdef()
        formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
                fields.StringField(id='1', label='string'),
                fields.PageField(id='2', label='2nd page', type='page'),
                fields.StringField(id='3', label='string 2')]
        formdef.fields[0].condition = {'type': 'python', 'value': initial_condition}
        formdef.store()
        page = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        page.forms[0]['f1'] = 'foo'
        assert page.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
        next_page = page.forms[0].submit('submit')
        assert_current_page(next_page, '2nd page')
        assert next_page.forms[0]['previous']
        next_page.forms[0]['f3'] = 'bar'
        next_page = next_page.forms[0].submit('submit')
        assert_current_page(next_page, 'Validating')
        assert 'Check values then click submit.' in next_page.body
        next_page = next_page.forms[0].submit('submit')
        assert next_page.status_int == 302
        next_page = next_page.follow()
        assert 'The form has been recorded' in next_page.body
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        data = formdef.data_class().get(data_id)
        assert data.data == {'1': 'foo', '3': 'bar'}

def test_form_multi_page_condition(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

def test_form_multi_page_condition_select(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar']),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_foo == "Foo"'}),
            fields.PageField(id='3', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_foo == "Bar"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not '2nd page' in resp.body
    assert not '3rd page' in resp.body
    assert resp.forms[0]['f1'].value == 'Foo' # preset
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.body
    assert not '3rd page' in resp.body
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert not '2nd page' in resp.body
    assert '3rd page' in resp.body
    assert_current_page(resp, '3rd page')

def test_form_multi_page_condition_select_new_varname(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar']),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'form_var_foo == "Foo"'}),
            fields.PageField(id='3', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'form_var_foo == "Bar"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not '2nd page' in resp.body
    assert not '3rd page' in resp.body
    resp.forms[0]['f1'] = 'Foo'
    resp = resp.forms[0].submit('submit')
    assert '2nd page' in resp.body
    assert not '3rd page' in resp.body
    assert_current_page(resp, '2nd page')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'Bar'
    resp = resp.forms[0].submit('submit')
    assert not '2nd page' in resp.body
    assert '3rd page' in resp.body
    assert_current_page(resp, '3rd page')

def test_form_multi_page_condition_checkbox(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'].checked = True
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']
    resp.forms[0]['f1'].checked = False
    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_json_check(pub):
    # make sure the json export has no value for fields from hidden pages
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2', varname='st2'),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "True"'}),
            fields.StringField(id='5', label='string 3', varname='st3'),
            ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit') # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit') # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body

    resp = resp.form.submit()

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None

def test_form_multi_page_condition_no_confirmation_json_check(pub):
    # same as above but without the confirmation page.
    formdef = create_formdef()
    formdef.confirmation = False
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "False"'}),
            fields.StringField(id='3', label='string 2', varname='st2'),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'var_checkbox == "True"'}),
            fields.StringField(id='5', label='string 3', varname='st3'),
            fields.PageField(id='6', label='4th page', type='page'),
            fields.CommentField(id='7', label='Check values then click submit.',
                type='comment'),
            ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.form['f1'].checked = True
    resp = resp.form.submit('submit') # should go straight to 3rd page
    assert 'f5' in resp.form.fields
    resp.form['f5'] = 'VALUE F5'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    resp = resp.form.submit('previous')

    # back to first page
    assert 'f1' in resp.form.fields
    resp.form['f1'].checked = False
    resp = resp.form.submit('submit') # should go to second page
    assert 'f3' in resp.form.fields
    resp.form['f3'] = 'VALUE F3'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body

    resp = resp.form.submit('submit')

    assert len(formdef.data_class().select()) == 1
    json_dict = formdef.data_class().select()[0].get_json_export_dict()
    assert json_dict['fields']['st2'] == 'VALUE F3'
    assert json_dict['fields']['st3'] is None

def test_form_multi_page_condition_data_source(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.BoolField(id='1', label='checkbox', varname='checkbox'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    # add the named data source, empty
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1']

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_data_source_with_form_variable(pub):
    # this tries to recreate #8272 which is about a json datasource being
    # used in a page condition and taking a value from the given page to
    # filter its content.  It is emulated here with a Python datasource
    # being empty if a field was not set.
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='xxx',
                required=False),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'len(data_source.foobar) > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    # add the named data source, related to a field on the first page
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': 'form_var_xxx and [form_var_xxx] or []'}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'HELLO'
    resp = resp.forms[0].submit('submit') # should go to second page
    assert 'f3' in resp.forms[0].fields

def test_form_multi_page_condition_on_first_page(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page',
                condition={'type': 'condition', 'value': 'False'}),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    # should be on second page already
    with pytest.raises(AssertionError):
        resp.form.get('previous')
    resp.form['f3'] = 'foo'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert resp.form['previous']
    resp = resp.form.submit('previous')
    assert resp.form['f3']
    with pytest.raises(AssertionError):
        resp.form.get('previous')

def test_form_multi_page_condition_no_visible_page(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    get_app(pub).get('/test/', status=404)

def test_form_multi_page_many_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(id='0', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'True'}),
    ]

    formdef.store()
    formdef.data_class().wipe()

    with mock.patch('qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        resp = get_app(pub).get('/test/')
        call_count = invalidate_cache.call_count

    for i in range(30):
        formdef.fields.append(fields.PageField(
                id=str(i+2), label='page %s' % (i+2), type='page',
                condition={'type': 'python', 'value': 'True'}))
    formdef.store()

    # check the cache doesn't get invalidated for every page
    with mock.patch('qommon.publisher.Substitutions.invalidate_cache') as invalidate_cache:
        resp = get_app(pub).get('/test/')
        assert invalidate_cache.call_count == call_count

def test_form_multi_page_post_conditions(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'False'}),
            fields.StringField(id='3', label='string 2'),
            fields.PageField(id='4', label='3rd page', type='page'),
            fields.StringField(id='5', label='string 3', varname='bar'),
            ]

    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foo'
    assert formdef.data_class().select()[0].data['5'] == 'bar'

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'False'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.body
    assert 'global-errors' in resp.body
    assert 'You shall not pass.' in resp.body

    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'form_var_foo == "foo"'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.body
    assert 'You shall not pass.' in resp.body

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body

    # check a post-condition raising an exception, they should always fail.
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': '1/0'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.body
    assert 'You shall not pass.' in resp.body

    # check a post-condition referring to a field on the same page
    formdef.fields[4].post_conditions = [
        {'condition': {'type': 'python', 'value': 'form_var_bar == "bar"'}, 'error_message': 'You shall not pass.'},
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'foo'
    resp = resp.forms[0].submit('submit')
    assert 'errornotice' in resp.body
    assert 'You shall not pass.' in resp.body

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'bar'
    resp = resp.forms[0].submit('submit')
    resp.forms[0]['f5'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body

def test_form_multi_page_page_name_as_title(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.TitleField(id='4', label='1st page', type='title'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    assert_current_page(next_page, '2nd page')
    assert next_page.forms[0]['previous']
    next_page.forms[0]['f3'] = 'bar'
    next_page = next_page.forms[0].submit('submit')
    assert_current_page(next_page, 'Validating')
    assert 'Check values then click submit.' in next_page.body
    assert next_page.body.count('1st page') == 2 # in steps and in main body

def test_form_submit_with_user(pub, emails):
    create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert formdef.data_class().count() == 1
    # check the user received a copy by email
    assert emails.emails.get('New form (test)')
    assert emails.emails.get('New form (test)')['email_rcpt'] == ['foo@localhost']

def test_form_visit_existing(pub):
    user = create_user(pub)
    formdef = create_formdef()
    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.store()

    formdata_user = formdef.data_class()()
    formdata_user.user_id = user.id
    formdata_user.store()

    resp = get_app(pub).get('/test/%s/' % formdata.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = get_app(pub).get('/test/%s/' % formdata_user.id)
    assert resp.location.startswith('http://example.net/login/?next=')

    resp = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % formdata_user.id)
    assert 'The form has been recorded on' in resp

def test_form_auth(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/auth')
    assert resp.location == 'http://example.net/login/?ReturnUrl=http%3A//example.net/test/'

    resp = login(get_app(pub), username='foo', password='foo').get('/test/auth')
    assert resp.location == 'http://example.net/test/'

def test_form_tryauth(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/tryauth')
    assert resp.location == 'http://example.net/test/'

    app = login(get_app(pub), username='foo', password='foo')
    pub.cfg['identification'] = {'methods': ['idp']}
    pub.write_cfg()
    # if the user is logged in, the form should be presented
    resp = app.get('/test/tryauth')
    assert resp.location == 'http://example.net/test/'

    # if the user is unlogged, there should be a passive redirection to SSO
    resp = get_app(pub).get('/test/tryauth')
    assert 'IsPassive=true' in resp.location

    pub.cfg['identification'] = {'methods': ['password']}
    pub.write_cfg()

def test_form_no_tracking_code(pub):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert not '<h3>Tracking code</h3>' in resp.body

def get_displayed_tracking_code(resp):
    tracking_code = None
    for a_tag in resp.html.findAll('a'):
        if 'code/' in a_tag['href']:
            tracking_code = a_tag.text
            break
    return tracking_code

def test_form_tracking_code(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.body
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit') # -> confirmation page
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.body
    assert formdef.data_class().count() == 1 # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.body # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

def test_form_tracking_code_as_user(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.body
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code_2 = get_displayed_tracking_code(resp)
    assert tracking_code == tracking_code_2

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    formdata_id = formdef.data_class().select()[0].id

    # check we can load the formdata as a draft
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    # check submitted form keeps the tracking code
    resp.forms[1]['f0'] = 'barfoo'
    resp = resp.forms[1].submit('submit') # -> confirmation page
    resp = resp.forms[1].submit('submit') # -> done
    resp = resp.follow()
    assert 'barfoo' in resp.body
    assert formdef.data_class().count() == 1 # check the draft one has been removed
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert str(formdef.data_class().select()[0].user_id) == str(user.id)
    assert formdef.data_class().select()[0].status == 'wf-new'
    assert formdef.data_class().select()[0].data['0'] == 'barfoo'
    formdata_id = formdef.data_class().select()[0].id

    # check we can still go back to it
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.body # makes sure user is treated as submitter
    resp.forms[0]['comment'] = 'hello world'
    resp = resp.forms[0].submit()
    assert formdef.data_class().get(formdata_id).evolution[-1].comment == 'hello world'

    # and check we can also get back to it as anonymous
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    resp = resp.follow()
    assert 'form_comment' in resp.body # makes sure user is treated as submitter

    # and check a bot is not allowed to get it
    app = get_app(pub)
    resp = app.get('/code/%s/load' % tracking_code,
            headers={'User-agent': 'Googlebot'}, status=403)

def test_form_empty_tracking_code(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.body
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    # check we get a 404 if we use the tracking code before it gets any data
    app = get_app(pub)
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow(status=404)

def test_form_tracking_code_email(pub, emails):
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.tracking_code = 'ABCDEF'
    formdata.store()

    resp = get_app(pub).get('/test/code/ABCDEF/')
    assert '<h2>Keep your tracking code</h2>' in resp.body
    resp.forms[0]['email'] = 'foo@localhost'
    resp = resp.forms[0].submit()
    assert emails.emails.get('Tracking Code reminder')
    assert 'ABCDEF' in emails.emails.values()[0]['payload']
    assert resp.location == 'http://example.net/test/code/ABCDEF/load'

def test_form_tracking_code_remove_draft(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.body
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '1'
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # visit page, check there's no remove draft button
    resp = app.get('/test/')
    assert '<h3>Tracking code</h3>' in resp.body
    assert not 'removedraft' in resp.body

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.body
    assert 'removedraft' in resp.body
    resp = resp.forms[1].submit('previous')
    assert resp.forms[1]['f0'].value == 'foobar'

    resp = resp.forms[0].submit() # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0

def test_form_tracking_code_remove_empty_draft(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()
    app = get_app(pub)
    resp = app.get('/test/')
    formdef.data_class().wipe()
    assert '<h3>Tracking code</h3>' in resp.body
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    resp = resp.forms[0].submit('previous')
    app.post('/test/autosave', params=resp.form.submit_fields())
    tracking_code = get_displayed_tracking_code(resp)
    assert tracking_code is not None

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].is_draft()
    assert formdef.data_class().select()[0].tracking_code == tracking_code
    assert formdef.data_class().select()[0].data['0'] == 'foobar'
    assert str(formdef.data_class().select()[0].page_no) == '0'

    # make draft empty
    formdata = formdef.data_class().select()[0]
    formdata.data = {}
    formdata.store()
    formdata_id = formdef.data_class().select()[0].id

    app = get_app(pub)

    # check we can load the formdata as a draft
    resp = app.get('/')
    resp.forms[0]['code'] = tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % tracking_code
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata_id
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/test/?mt=')
    resp = resp.follow()
    assert '<h3>Tracking code</h3>' in resp.body
    assert 'removedraft' in resp.body
    assert resp.forms[1]['f0'].value == ''

    resp = resp.forms[0].submit() # remove draft
    assert resp.location == 'http://example.net/'
    assert formdef.data_class().count() == 0

def test_form_invalid_tracking_code(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    # create a secondary formdef, to always have the tracking code form
    # displayed on homepage
    formdef2 = FormDef()
    formdef2.name = 'test2'
    formdef2.fields = []
    formdef2.enable_tracking_codes = True
    formdef2.store()

    resp = get_app(pub).get('/')

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.store()

    # check we can go back to it
    formdef.data_class().wipe()

    code = pub.tracking_code_class()
    code.formdata = formdata
    code.store()

    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow()
    assert resp.location == 'http://example.net/test/%s' % formdata.id
    resp = resp.follow()

    # check we get a not found error message on non-existent code
    fake_code = TrackingCode().get_new_id()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = fake_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % fake_code
    resp = resp.follow(status=404)

    # check we also get an error if tracking code access is disabled after the
    # fact
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = get_app(pub).get('/')
    resp.forms[0]['code'] = code.id
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/code/%s/load' % code.id
    resp = resp.follow(status=404)

def test_form_direct_draft_access(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'foobar'}
    formdata.status = 'draft'
    formdata.store()

    resp = get_app(pub).get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    formdata.user_id = user.id
    formdata.store()
    resp = get_app(pub).get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/login')

    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s' % formdata.id, status=302)
    assert resp.location.startswith('http://example.net/test/?mt=')

    formdata.user_id = 1000
    formdata.store()
    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s' % formdata.id, status=403)

    formdata.user_id = user.id
    formdata.store()
    formdef.enable_tracking_codes = False
    formdef.store()
    resp = login(get_app(pub), 'foo', 'foo').get('/test/%s' % formdata.id, status=403)

def form_password_field_submit(pub, password):
    password = unicode(password).encode(pub.site_charset)
    formdef = create_formdef()
    formdef.fields = [fields.PasswordField(id='0', label='password',
        formats=['sha1', 'md5', 'cleartext'])]
    formdef.store()
    page = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    next_page = page.forms[0].submit('submit') # but the field is required
    assert '<div class="error">required field</div>' in next_page.body
    next_page.forms[0]['f0$pwd1'] = password
    next_page.forms[0]['f0$pwd2'] = password
    next_page = next_page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'0': {
        'sha1': hashlib.sha1(password).hexdigest(),
        'md5': hashlib.md5(password).hexdigest(),
        'cleartext': unicode(password, 'utf-8'),
        }}

def test_form_password_field_submit(pub):
    form_password_field_submit(pub, 'foobar')
    form_password_field_submit(pub, u'foobar\u00eb')

def test_form_multi_page_formdef_count_condition(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'form_objects.count > 0'}),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body

    # add a formdata this will make the second page appear.
    formdef.data_class()().store()

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f1'] = 'foo'
    resp = resp.forms[0].submit('submit') # should NOT go straight to validation
    assert 'Check values then click submit.' not in resp.body

def test_form_multi_page_post_edit(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    editable = EditableWorkflowStatusItem()
    editable.id = '_editable'
    editable.by = ['_submitter', '_receiver']
    st1.items.append(editable)
    editable.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page.forms[0]['f1'] = 'foo'
    next_page = page.forms[0].submit('submit')
    next_page.forms[0]['f3'] = 'barXYZ'
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.forms[0].submit('submit')
    next_page = next_page.follow()
    assert 'The form has been recorded' in next_page.body

    data_id = formdef.data_class().select()[0].id

    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.body
    assert 'barXYZ' in page.body

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo'
    resp.forms[0]['f1'] = 'foo2'

    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('previous')
    assert resp.forms[0]['f1'].value == 'foo2'
    resp = resp.forms[0].submit('submit')
    assert 'Save Changes' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo2' in resp.body # modified value is there
    assert 'barXYZ' in resp.body # unchanged value is still there

    # modify workflow to jump to another status after the edition
    st2 = workflow.add_status('Status2', 'st2')
    editable.status = st2.id
    workflow.store()

    assert formdef.data_class().get(data_id).status == 'wf-%s' % st1.id
    page = login(get_app(pub), username='foo', password='foo').get('/test/%s/' % data_id)
    assert 'button_editable-button' in page.body
    assert 'barXYZ' in page.body

    resp = page.forms[0].submit('button_editable')
    assert resp.location.startswith('http://example.net/test/%s/wfedit-' % data_id)
    resp = resp.follow()
    assert resp.forms[0]['f1'].value == 'foo2'
    resp.forms[0]['f1'] = 'foo3'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f3'].value == 'barXYZ'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/%s/' % data_id
    resp = resp.follow()
    assert 'foo3' in resp.body # modified value is there
    assert 'barXYZ' in resp.body # unchanged value is still there
    assert formdef.data_class().get(data_id).status == 'wf-%s' % st2.id

def test_form_count_dispatching(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.condition = {'type': 'python', 'value': 'form_objects.count_status_st2 < 1'}
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    st2 = workflow.add_status('Status2', 'st2')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.body # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 0
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1

    page = login(get_app(pub), username='foo', password='foo').get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()
    assert 'The form has been recorded' in page.body # success

    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st2')) == 1
    assert len(formdef.data_class().select(clause=lambda x: x.status == 'wf-st1')) == 1

def test_preview_form(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = []
    formdef.disabled = True
    formdef.store()

    # check the preview page is not accessible to regular users
    get_app(pub).get('/preview/test/', status=403)

    # check it's accessible to admins
    user.is_admin = True
    user.store()
    page = login(get_app(pub), username='foo', password='foo').get('/preview/test/')

    # check no formdata gets stored
    next_page = page.forms[0].submit('submit')
    assert 'Check values then click submit.' in next_page.body
    next_page = next_page.forms[0].submit('submit')
    assert next_page.status_int == 302
    assert next_page.location == 'http://example.net/preview/test/'
    assert formdef.data_class().count() == 0

def test_form_item_data_source_field_submit(pub):
    def submit_item_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0'] = '1'
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.body
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.body
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux')]),
    }
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])
    assert submit_item_data_source_field(ds) == {'0': '1', '0_display': 'un'}

    ds['value'] = repr([
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'}])
    assert submit_item_data_source_field(ds) == {
            '0': '1', '0_display': 'un', '0_structured': {'id': '1', 'text': 'un', 'more': 'foo'}}

def test_form_items_data_source_field_submit(pub):
    def submit_items_data_source_field(ds):
        formdef = create_formdef()
        formdef.fields = [fields.ItemsField(id='0', label='string', data_source=ds)]
        formdef.store()
        resp = get_app(pub).get('/test/')
        formdef.data_class().wipe()
        resp.forms[0]['f0$element1'].checked = True
        resp.forms[0]['f0$element3'].checked = True
        resp = resp.forms[0].submit('submit')
        assert 'Check values then click submit.' in resp.body
        resp = resp.forms[0].submit('submit')
        assert resp.status_int == 302
        resp = resp.follow()
        assert 'The form has been recorded' in resp.body
        assert formdef.data_class().count() == 1
        data_id = formdef.data_class().select()[0].id
        return formdef.data_class().get(data_id).data

    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux'), ('3', 'trois')]),
    }
    assert submit_items_data_source_field(ds) == {'0': ['1', '3'], '0_display': 'un, trois'}

    ds['value'] = repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'},
        {'id': '3', 'text': 'trois'}])
    assert submit_items_data_source_field(ds) == {'0': ['1', '3'], '0_display': 'un, trois'}

    ds['value'] = repr([
        {'id': '1', 'text': 'un', 'more': 'foo'},
        {'id': '2', 'text': 'deux', 'more': 'bar'},
        {'id': '3', 'text': 'trois', 'more': 'baz'}])
    assert submit_items_data_source_field(ds) == {
            '0': ['1', '3'],
            '0_display': 'un, trois',
            '0_structured': [
                {'id': '1', 'more': 'foo', 'text': 'un'},
                {'id': '3', 'more': 'baz', 'text': 'trois'}]}

def test_form_page_string_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'string', 'value': 'HELLO WORLD'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.body
    assert not 'Value has been automatically prefilled.' in resp.body

def test_form_page_profile_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

def test_form_page_formula_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': repr('HELLO WORLD')})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == 'HELLO WORLD'
    assert 'widget-prefilled' in resp.body
    assert 'Value has been automatically prefilled.' in resp.body

def test_form_page_query_string_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_var_foo'})]
    formdef.store()

    # check it's empty if it doesn't exist
    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    # check it's not set if it's not whitelisted
    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check it works
    open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w').write('''[options]
query_string_allowed_vars = foo,bar
''')

    resp = get_app(pub).get('/?session_var_foo=hello')
    assert urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello'

    # check it survives a login
    resp = get_app(pub).get('/?session_var_foo=hello2')
    assert urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('Login')
    resp = resp.follow()
    resp.forms[0]['username'] = 'foo'
    resp.forms[0]['password'] = 'foo'
    resp = resp.forms[0].submit()
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == 'hello2'

    # check repeated options are ignored
    resp = get_app(pub).get('/?session_var_foo=hello&session_var_foo=hello2')
    assert urlparse(resp.location).path == '/'
    resp = resp.follow()
    resp = resp.click('test')
    assert resp.forms[0]['f0'].value == ''

    # check extra query string parameters are not lost
    resp = get_app(pub).get('/?session_var_foo=hello&foo=bar')
    assert urlparse(resp.location).path == '/'
    assert urlparse(resp.location).query == 'foo=bar'

    os.unlink(os.path.join(pub.app_dir, 'site-options.cfg'))

def test_form_page_profile_prefill_list(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='item', type='item',
        items=['', 'bar@localhost', 'foo@localhost'], required=False,
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    # invalid value
    formdef.fields = [fields.ItemField(id='0', label='item', type='item',
        items=['', 'bar@localhost'], required=False,
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()
    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'invalid value selected' in resp.body
    assert resp.forms[0]['f0'].value == ''

def test_form_page_formula_prefill_items_field(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.ItemsField(id='0', label='items',
        items=['foo', 'bar', 'baz'],
        prefill={'type': 'formula', 'value': '["foo", "baz"]'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0$element0'].checked
    assert not resp.form['f0$element1'].checked
    assert resp.form['f0$element2'].checked
    assert 'widget-prefilled' in resp.body
    assert 'Value has been automatically prefilled.' in resp.body

    resp.form['f0$element0'].checked = False
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit').follow()
    assert '>foo<' not in resp.body
    assert '>bar<' not in resp.body
    assert '>baz<' in resp.body

    # check with remote json
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemsField(id='0', label='items',
        data_source=ds, display_disabled_items=True,
        prefill={'type': 'formula', 'value': '["2"]'})]

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not resp.form['f0$element1'].checked
        assert resp.form['f0$element2'].checked


def test_form_captcha(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.store()

    # test authenticated users are not presented with a captcha
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert not 'form_captcha' in resp.body

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert 'form_captcha' in resp.body

    session_id = app.cookies.values()[0].strip('"')
    session = BasicSession.get(session_id)
    resp.form['captcha$q'] = session.get_captcha_token(resp.forms[0]['captcha$token'].value)['answer']
    resp = resp.form.submit('submit')
    assert resp.status_code == 302 # redirect when formdata is created

    # and check it gets it only once
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert not 'form_captcha' in resp.body

def test_form_captcha_and_no_validation_page(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='Some field')]
    formdef.has_captcha = True
    formdef.enable_tracking_codes = True
    formdef.confirmation = False
    formdef.store()

    # test authenticated users are not stopped on a confirmation page
    resp = login(get_app(pub), username='foo', password='foo').get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert resp.status_code == 302 # redirect when formdata is created

    # check anonymous user gets the captcha
    app = get_app(pub)
    resp = app.get('/')
    resp = resp.click('test')
    resp.form['f0'] = 'test'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert 'form_captcha' in resp.body

def test_form_file_field_with_fargo(pub, fargo_url):
    user = create_user(pub)
    formdef = create_formdef()
    file_field = fields.FileField(id='0', label='file')
    formdef.fields = [file_field]
    formdef.store()
    formdef.data_class().wipe()

    assert file_field.allow_portfolio_picking == True

    resp = get_app(pub).get('/test/')
    assert 'f0$file' in resp.body
    assert 'fargo.js' not in resp.body
    assert 'use-file-from-fargo' not in resp.body

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.body
    assert 'fargo.js' in resp.body
    assert 'use-file-from-fargo' in resp.body

    file_field.allow_portfolio_picking = False
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.body
    assert 'fargo.js' not in resp.body
    assert 'use-file-from-fargo' not in resp.body

def test_form_file_field_without_fargo(pub):
    user = create_user(pub)
    formdef = create_formdef()
    file_field = fields.FileField(id='0', label='file')
    formdef.fields = [file_field]
    formdef.store()
    formdef.data_class().wipe()

    assert file_field.allow_portfolio_picking == True

    resp = get_app(pub).get('/test/')
    assert 'f0$file' in resp.body
    assert 'fargo.js' not in resp.body
    assert 'use-file-from-fargo' not in resp.body

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert 'f0$file' in resp.body
    assert 'fargo.js' not in resp.body
    assert 'use-file-from-fargo' not in resp.body

def test_form_file_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    upload = Upload('test.txt', 'foobar', 'text/plain')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.body == 'foobar'

def test_form_file_field_image_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    image_content = open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg')).read()
    upload = Upload('test.jpg', image_content, 'image/jpeg')

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert '<img alt="" src="tempfile?' in resp.body
    tempfile_id = resp.body[resp.body.index('tempfile?'):].split('&', 1)[0].split('=')[1]

    resp_tempfile = app.get('/test/tempfile?t=%s' % tempfile_id)
    assert resp_tempfile.body == image_content

    if Image:
        # check thumbnailing of image in validation page
        resp_thumbnail = app.get('/test/tempfile?t=%s&thumbnail=1' % tempfile_id)
        assert resp_thumbnail.content_type == 'image/png'

    resp = resp.form.submit('submit').follow()
    assert '<img ' in resp.body
    assert 'download?f=0&thumbnail=1' in resp.body
    resp = resp.goto('download?f=0&thumbnail=1')
    assert '/thumbnail/' in resp.location
    resp = resp.follow()
    if Image:
        # check thumbnailing of image in submitted form
        assert resp.content_type == 'image/png'

        # check a fake image is not sent back
        upload = Upload('test.jpg', '<script>evil javascript</script>', 'image/jpeg')
        app = get_app(pub)
        resp = app.get('/test/')
        resp.forms[0]['f0$file'] = upload
        resp = resp.forms[0].submit('submit')
        assert not '<img alt="" src="tempfile?' in resp.body

def test_form_file_field_submit_wrong_mimetype(pub):
    formdef = create_formdef()
    formdef.fields = [fields.FileField(id='0', label='file')]
    formdef.store()
    formdef.data_class().wipe()

    upload = Upload('test.txt', 'foobar', 'application/force-download')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.body == 'foobar'

    upload = Upload('test.pdf', '%PDF-1.4 ...', 'application/force-download')

    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0$file'] = upload
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    resp = resp.click('test.pdf')
    assert resp.location.endswith('/test.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    assert resp.body == '%PDF-1.4 ...'

def test_form_table_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TableField(id='0', label='table', type='table',
        rows=['a', 'b'], columns=['c', 'd', 'e'], required=False)]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': [['', '', ''], ['', '', '']]}
    formdef.data_class().wipe()

    formdef.fields = [fields.TableField(id='0', label='table', type='table',
        rows=['a', 'b'], columns=['c', 'd', 'e'], required=True)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert not 'Check values then click submit.' in resp.body

    resp = get_app(pub).get('/test/')
    resp.form['f0$c-0-0'] = 'a'
    resp.form['f0$c-1-0'] = 'b'
    resp.form['f0$c-0-1'] = 'c'
    resp.form['f0$c-1-1'] = 'd'
    resp.form['f0$c-0-2'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().select()[0].data == {'0': [['a', 'c', 'e'], ['b', 'd', '']]}

def test_form_table_rows_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=False)]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data == {'0': []}
    formdef.data_class().wipe()

    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=True)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit')
    assert not 'Check values then click submit.' in resp.body

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = 'b'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = 'd'
    resp.form['f0$element2$col0'] = 'e'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().select()[0].data == {'0': [['a', 'b'], ['c', 'd'], ['e', '']]}

    formdef.data_class().wipe()

    formdef.fields = [fields.TableRowsField(id='0', label='table', type='tablerows',
        columns=['a', 'b'], required=True, total_row=True)]
    formdef.store()

    resp = get_app(pub).get('/test/')
    resp.form['f0$element0$col0'] = 'a'
    resp.form['f0$element0$col1'] = '14'
    resp.form['f0$element1$col0'] = 'c'
    resp.form['f0$element1$col1'] = '23'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert '37.00' in resp.body

def test_formdata_attachment_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp.forms[0]['attachment_attach'] = Upload('test.txt', 'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow() # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.body == 'foobar'

def test_formdata_attachment_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.varname = 'attached_doc'
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp.forms[0]['attachment_attach'] = Upload('test.txt', 'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow() # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.body == 'foobar'

    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    attachment_variable = attachments.attached_doc

    resp = login(get_app(pub), username='admin', password='admin').get(
        attachment_variable.url).follow()
    assert attachment_variable.content == resp.body
    assert attachment_variable.b64_content == base64.b64encode(resp.body)
    assert attachment_variable.content_type == resp._headers['content-type'].split(';')[0]
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(attachment_variable.filename)

def test_formdata_generated_document_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    assert resp.body == 'HELLO WORLD'

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('test.rtf')
    assert resp.location.endswith('/test.rtf')
    resp = resp.follow()
    assert resp.content_type == 'application/rtf'
    assert resp.body == 'HELLO WORLD'

    # change export model to now be a RTF file, do the action again on the same form and
    # check that both the old .odt file and the new .rtf file are there and valid.
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    assert resp.click('test.rtf', index=0).follow().body == 'HELLO WORLD'
    assert resp.click('test.rtf', index=1).follow().body == 'HELLO NEW WORLD'

    # use substitution variables on rtf: only ezt format is accepted
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO {{DJANGO}} WORLD [form_name]')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow()

    assert resp.click('test.rtf', index=2).follow().body == 'HELLO {{DJANGO}} WORLD {\\uc1{test}}'


@pytest.fixture(params=['template.odt', 'template-django.odt'])
def odt_template(request):
    return request.param

def test_formdata_generated_document_odt_download(pub, odt_template):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    template_filename = os.path.join(os.path.dirname(__file__), odt_template)
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/' + odt_template, content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        assert_equal_zip(StringIO.StringIO(resp.body), f)

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    resp = resp.follow() # $form/$id/create_doc
    with mock.patch('wcs.wf.export_to_model.get_formdata_template_context') as get_context_1:
        with mock.patch('wcs.workflows.get_formdata_template_context') as get_context_never:
            get_context_1.return_value = {}
            get_context_never.return_value = {}
            resp = resp.follow() # $form/$id/create_doc/
            # substitution variables are computed only one :
            assert get_context_1.call_count == 1
            assert get_context_never.call_count == 0

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click(odt_template)
    assert resp.location.endswith('/' + odt_template)
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        assert_equal_zip(StringIO.StringIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        body = resp.click(odt_template, index=0).follow().body
        assert_equal_zip(StringIO.StringIO(body), f)
    assert resp.click('test.rtf', index=0).follow().body == 'HELLO NEW WORLD'

def test_formdata_generated_document_odt_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        assert_equal_zip(StringIO.StringIO(resp.body), f)

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    response1 = resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        assert_equal_zip(StringIO.StringIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        body = resp.click('template.odt', index=0).follow().body
        assert_equal_zip(StringIO.StringIO(body), f)
    response2 = resp.click('test.rtf', index=0).follow()
    assert response2.body == 'HELLO NEW WORLD'
    # Test attachment substitution variables
    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    file1 = attachments.created_doc
    assert file1.content == response2.body
    assert file1.b64_content == base64.b64encode(response2.body)
    assert file1.content_type == response2._headers['content-type']
    content_disposition = response2._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response2.request.environ['PATH_INFO'].endswith(file1.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file1.url).follow()
    assert file1.content == resp.body
    assert file1.b64_content == base64.b64encode(resp.body)
    assert file1.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file1.filename)

    file2 = attachments.created_doc[1]
    assert file2.content == response1.body
    assert file2.b64_content == base64.b64encode(response1.body)
    assert file2.content_type == response1._headers['content-type']
    content_disposition = response1._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response1.request.environ['PATH_INFO'].endswith(file2.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file2.url).follow()
    assert file2.content == resp.body
    assert file2.b64_content == base64.b64encode(resp.body)
    assert file2.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file2.filename)

@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.form.submit('button_export_to')

    resp = resp.follow() # $form/$id/create_doc
    resp = resp.follow() # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert 'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location
    resp = resp.follow() # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'inline'
    assert resp.body.startswith('%PDF-')

@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(pub, fargo_url,
                                                                           fargo_secret, caplog):
    create_user(pub)
    pub.cfg['debug'] = {'logger': True}
    pub.write_cfg()
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    export_to.push_to_portfolio = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert ('file template.pdf pushed to portfolio of foo@localhost'
                == caplog.records[-1].message)

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert 'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert ('file template.pdf pushed to portfolio of foo@localhost'
                == caplog.records[-1].message)
    assert resp.location == form_location
    resp = resp.follow()  # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    assert resp.body.startswith('%PDF-')

def test_formdata_generated_document_non_interactive(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    st1.items.append(export_to)
    export_to.parent = st1

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    st2 = wf.add_status('Status2', 'st2')

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
        assert_equal_zip(StringIO.StringIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'

def test_formdata_generated_document_to_backoffice_field(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
        fields.StringField(id='bo2', label='bo field 2', type='string'),
    ]

    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), 'template.odt')
    template = open(template_filename).read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = StringIO.StringIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    export_to.backoffice_filefield_id = 'bo1'
    st1.items.append(export_to)
    export_to.parent = st1

    assert export_to.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    wf.add_status('Status2', 'st2')
    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    # get the two generated files from backoffice: in backoffice fields
    # (export_to.backoffice_filefield_id), and in history (export_to.attach_to_history)
    for index in (0, 1):
        resp = login(get_app(pub), username='admin', password='admin').get(
                '/backoffice/management/test/1/')
        resp = resp.click('template.odt', index=index)
        assert resp.location.endswith('/template.odt')
        resp = resp.follow()
        assert resp.content_type == 'application/octet-stream'
        with open(os.path.join(os.path.dirname(__file__), 'template-out.odt')) as f:
            assert_equal_zip(StringIO.StringIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'

def test_formdata_generated_document_in_private_history(pub):
    user = create_user(pub)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    user.roles = [role.id]
    user.store()

    wf = Workflow(name='status')
    st0 = wf.add_status('Status0', 'st0')
    st1 = wf.add_status('Status1', 'st1')
    st1.visibility = ['_receiver']
    export_to = ExportToModel()
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = StringIO.StringIO()
    upload.fp.write('HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.attach_to_history = True
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1

    st2 = wf.add_status('Status2', 'st2')

    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_receiver']
    jump1.status = st1.id
    jump1.parent = st0
    st0.items.append(jump1)

    jump2 = ChoiceWorkflowStatusItem()
    jump2.id = '_jump2'
    jump2.label = 'Jump 2'
    jump2.by = ['_receiver']
    jump2.status = st2.id
    jump2.parent = st1
    st1.items.append(jump2)

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp = resp.form.submit('button_jump1')
    resp = resp.follow()

    resp = resp.form.submit('button_export_to')
    resp = resp.follow()
    assert 'Form exported in a model' in resp.body

    resp = resp.form.submit('button_jump2')
    resp = resp.follow()


    # change formdef receiver so the hidden status should not longer be visible
    role2 = Role(name='yyy')
    role2.store()
    formdef.workflow_roles = {'_receiver': role2.id}
    formdef.store()

    resp = login(get_app(pub), username='foo', password='foo').get(resp.request.url)
    assert not 'Form exported in a model' in resp.body

def test_formdata_form_file_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.FileField(id='1', label='File',
        type='file', varname='yyy'))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body

    resp.forms[0]['f1$file'] = Upload('test.txt', 'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'xxx_var_yyy_raw' in formdata.workflow_data

    download = resp.test_app.get(resp.location + 'files/form-xxx-yyy/test.txt')
    assert download.content_type == 'text/plain'
    assert download.body == 'foobar'

    # go back to the status page, this will exercise the substitution variables
    # codepath.
    resp = resp.follow()

def test_formdata_workflow_form_prefill(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.StringField(id='1', label='blah',
        type='string', varname='yyy', prefill={'type': 'user', 'value': 'email'}))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert resp.forms[0]['f1'].value == 'foo@localhost'

def test_form_map_field_back_and_submit(pub):
    formdef = create_formdef()
    formdef.fields = [
        fields.MapField(id='0', label='map'),
        fields.StringField(id='1', label='street', required=True,
            prefill={'type': 'geolocation', 'value': 'road'}),
    ]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    assert 'qommon.map.js' in resp.body
    assert 'qommon.geolocation.js' in resp.body
    # with a real user interaction this would get set by javascript
    resp.forms[0]['f0$latlng'].value = '1.234;-1.234'
    assert 'data-geolocation="road"' in resp.body

    # check required field
    resp = resp.forms[0].submit('submit')
    assert not 'Check values then click submit.' in resp.body
    assert 'data-geolocation="road"' in resp.body
    resp.forms[0]['f1'].value = 'bla'

    # check summary page
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert 'data-init-lng="-1.234"' in resp.body
    assert 'data-init-lat="1.234"' in resp.body

    # get back to the map field
    resp = resp.forms[0].submit('previous')
    # check the field is still marked as holding the road
    assert 'data-geolocation="road"' in resp.body
    assert resp.forms[0]['f0$latlng'].value == '1.234;-1.234'

    # back to summary page
    resp = resp.forms[0].submit('submit')

    # and submitting the form
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': 'bla', '0': '1.234;-1.234'}

def test_form_map_multi_page(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.MapField(id='1', label='map'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f1$latlng'] = '1.234;-1.234'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['f1$latlng'].value == '1.234;-1.234'
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data == {'1': '1.234;-1.234', '3': 'bar'}

def test_form_middle_session_change(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    app.cookiejar.clear()
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert resp.location == 'http://example.net/test/'
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.body

    app = get_app(pub)
    resp = app.get('/test/')
    resp.forms[0]['f1'] = 'foo'
    assert resp.forms[0].fields['submit'][0].value_if_submitted() == 'Next'
    resp = resp.forms[0].submit('submit')
    assert resp.forms[0]['previous']
    resp.forms[0]['f3'] = 'bar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    app.cookiejar.clear()
    resp = resp.forms[0].submit('submit')
    resp = resp.follow()
    assert 'Sorry, your session have been lost.' in resp.body

def test_form_autocomplete_variadic_url(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.PageField(id='3', label='2nd page', type='page',
                condition={'type': 'python', 'value': 'True'}),
            fields.ItemField(id='1', label='string', type='item',
                varname='foo', items=['Foo', 'Bar']),
            fields.StringField(id='2', label='string2', required=True,
                data_source={'type': 'jsonp', 'value': '[var_foo]'}),
            fields.PageField(id='4', label='3rd page', type='page',
                condition={'type': 'python', 'value': 'True'})
            ]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.form.submit('submit') # next
    # test javascript will be used to compute the full URL
    assert 'options.wcs_base_url' in resp.body

    # test going forward (will error out), check it's still a variadic URL (#9786)
    resp.form['f1'] = 'Foo'
    resp = resp.form.submit('submit')
    assert 'options.wcs_base_url' in resp.body

def test_form_page_formula_prefill_user_name(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'form_user_email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

def test_form_page_formula_prefill_session_user(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_user_email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.forms[0]['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.forms[0]['f0'].value == 'foo@localhost'

    app = login(get_app(pub), username='foo', password='foo')
    for session in pub.session_manager.values():
        session.extra_user_variables = {'foo': 'bar'}
        session.store()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'formula', 'value': 'session_var_user_foo'})]
    formdef.store()
    resp = app.get('/test/')
    assert resp.forms[0]['f0'].value == 'bar'

def test_form_date_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.DateField(id='0', label='string', type='date',
        required=False)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp.forms[0]['f0'] = '2015-01-01'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert time.strftime('%Y-%m-%d', data.data['0']) == '2015-01-01'

    # without filling the field
    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

def test_form_jsonp_item_field(http_requests, pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='string', type='item',
                data_source={'type': 'jsonp', 'value': 'http://remote.example.net/jsonp'}),
            ]
    formdef.store()
    formdef.data_class().wipe()

    resp = get_app(pub).get('/test/')
    assert 'url: "http://remote.example.net/jsonp"' in resp.body

    formdef.fields = [
            fields.ItemField(id='1', label='string', type='item',
                data_source={'type': 'jsonp', 'value': '[var_XXX]'}),
            ]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert 'url: function(' in resp.body
    assert "wcs_base_url = '[var_XXX]';" in resp.body

def test_form_string_regex_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string',
        validation=r'\d{5}$', required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # with an invalid input
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    assert 'wrong format' in resp.body

def test_form_text_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.TextField(id='0', label='string', type='text', required=False)]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = '12345'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == '12345'

    # without filling the field
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] is None

    # check max length
    formdef.fields = [fields.TextField(id='0', label='string', type='text', maxlength=10)]
    formdef.store()
    resp = get_app(pub).get('/test/')
    resp.forms[0]['f0'] = 'x' * 11
    resp = resp.forms[0].submit('submit')
    assert 'too many characters (limit is 10)' in resp.body
    # check it counts characters, not bytes
    resp.forms[0]['f0'] = u'☭' * 10
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body

def test_form_items_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [fields.ItemsField(id='1', label='items',
        varname='items', required=False, data_source={'type': 'foobar'})]
    formdef.store()

    data_class = formdef.data_class()
    data_class.wipe()

    # add the named data source
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr([])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    formdef.data_class().wipe()
    resp = resp.forms[0].submit('submit') # should go straight to validation
    assert 'Check values then click submit.' in resp.body
    assert resp.forms[0]['previous']
    resp = resp.forms[0].submit('previous')

    # replace the named data source with one with items
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'formula', 'value': repr(['un', 'deux'])}
    data_source.store()

    resp = get_app(pub).get('/test/')
    assert 'f1$elementun' in resp.form.fields
    assert 'f1$elementdeux' in resp.form.fields
    resp.form['f1$elementun'].checked = True
    resp.form['f1$elementdeux'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {'1': ['un', 'deux'], '1_display': 'un, deux'}

    data_source.data_source = {'type': 'formula',
            'value': repr([{'id': '1', 'text': 'un'},
                           {'id': '2', 'text': 'deux'}])}
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()

    assert data_class.select()[0].data == {'1': ['1', '2'], '1_display': 'un, deux'}

    data_source.data_source = {'type': 'formula',
            'value': repr([{'id': '1', 'text': 'un', 'foo': 'bar1'},
                           {'id': '2', 'text': 'deux', 'foo': 'bar2'}])}
    data_source.store()

    data_class.wipe()
    resp = get_app(pub).get('/test/')
    assert 'f1$element1' in resp.form.fields
    assert 'f1$element2' in resp.form.fields
    resp.form['f1$element1'].checked = True
    resp.form['f1$element2'].checked = True
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert data_class.select()[0].data == {
            '1': ['1', '2'],
            '1_structured': [{'text': 'un', 'foo': 'bar1', 'id': '1'},
                             {'text': 'deux', 'foo': 'bar2', 'id': '2'}],
            '1_display': 'un, deux'}
    # along the way, check substitution variables
    substvars = data_class.select()[0].get_substitution_variables()
    assert substvars['form_var_items'] == 'un, deux'
    assert substvars['form_var_items_raw'] == ['1', '2']
    assert substvars['form_var_items_0_foo'] == 'bar1'
    assert substvars['form_var_items_1_foo'] == 'bar2'

def test_form_ranked_items_field_submit(pub):
    formdef = create_formdef()
    formdef.fields = [fields.RankedItemsField(
        id='0', label='ranked items',
        type='ranked-items', required=False,
        items=['foo', 'bar', 'baz'])]
    formdef.store()

    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    resp.form['f0$element0'] = '1'
    resp.form['f0$element1'] = '2'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1
    data_id = formdef.data_class().select()[0].id
    data = formdef.data_class().get(data_id)
    assert data.data['0'] == {'bar': 2, 'foo': 1}

def test_form_ranked_items_randomize_order(pub):
    formdef = create_formdef()
    formdef.fields = [fields.RankedItemsField(
        id='0', label='ranked items',
        type='ranked-items', required=False,
        randomize_items=True,
        items=['foo', 'bar', 'baz'])]
    formdef.store()
    orders = {}
    for i in range(10):
        resp = get_app(pub).get('/test/')
        orders['%s-%s-%s' % (resp.body.index('foo'), resp.body.index('bar'), resp.body.index('baz'))] = True
    assert len(orders.keys()) > 1

def test_form_autosave(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.StringField(id='3', label='string 2')]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'foobar'

    resp.form['f1'] = 'foobar2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar2'

    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    resp.form['f3'] = 'xxx'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx'

    resp.form['f3'] = 'xxx2'
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx2'

    resp.form['f3'] = 'xxx3'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert 'foobar3' in resp.body
    assert 'xxx3' in resp.body

    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
    assert formdef.data_class().select()[0].data['3'] == 'xxx3'

    # make sure autosave() doesn't destroy data that would have been submitted
    # in the meantime
    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'
    autosave_fields = resp.form.submit_fields()
    resp.form['f1'] = 'foobar3'
    resp = resp.forms[0].submit('submit')
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

    # post content with 'foobar' as value, it should not be saved
    ajax_resp = app.post('/test/autosave', params=autosave_fields)
    assert json.loads(ajax_resp.body)['result'] == 'error'
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'

def test_form_autosave_with_parameterized_datasource(pub):
    formdef = create_formdef()
    formdef.fields = [fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='string', varname='foo'),
            fields.PageField(id='2', label='2nd page', type='page'),
            fields.ItemField(id='3', label='item', type='item',
                data_source={'type': 'formula',
                             'value': '''[('1', form_var_foo*2)]'''})]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    app = get_app(pub)
    resp = app.get('/test/')
    resp.form['f1'] = 'bar'

    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.status == 'draft'
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data.get('3') == None

    resp = resp.forms[0].submit('submit')
    app.post('/test/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['1'] == 'bar'
    assert formdef.data_class().select()[0].data['3'] == '1'
    assert formdef.data_class().select()[0].data['3_display'] == 'barbar'

def test_form_string_field_autocomplete(pub):
    formdef = create_formdef()
    formdef.fields = [fields.StringField(id='0', label='string', type='string', required=False)]
    formdef.fields[0].data_source = {'type': 'jsonp'}
    formdef.store()

    # not filled completed, no call to .autocomplete
    resp = get_app(pub).get('/test/')
    assert not ').autocomplete({' in resp.body

    # straight URL
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': 'http://example.org'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.body
    assert 'http://example.org' in resp.body

    # URL from variable
    formdef.fields[0].data_source = {'type': 'jsonp', 'value': '[site_url]'}
    formdef.store()
    resp = get_app(pub).get('/test/')
    assert ').autocomplete({' in resp.body
    assert 'http://example.net' in resp.body

def test_form_workflow_trigger(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    jump2 = JumpWorkflowStatusItem()
    jump2.trigger = 'YYY'
    jump2.status = 'st3'
    st1.items.append(jump2)
    jump2.parent = st1

    st2 = workflow.add_status('Status2', 'st2')
    st3 = workflow.add_status('Status3', 'st3')
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)
    resp = login(app, username='foo', password='foo').get('/')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    jump.by = [role.id]
    workflow.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=403)

    user.roles = [role.id]
    user.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st2'

    formdata.status = 'wf-st1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY', status=403)
    jump2.by = [role.id]
    workflow.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY', status=302)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.status == 'wf-st3'

    formdata.status = 'wf-st1'
    formdata.store()
    resp = app.post(formdata.get_url() + 'jump/trigger/YYY',
            params=json.dumps({'data': {'foo': 'bar'}}),
            content_type='application/json')
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.workflow_data == {'data': {'foo': 'bar'}}

def test_form_worklow_multiple_identical_status(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')
    st1.extra_css_class = 'CSS-STATUS1'
    jump = JumpWorkflowStatusItem()
    jump.trigger = 'XXX'
    jump.status = 'st1'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    app = get_app(pub)

    Role.wipe()
    role = Role(name='xxx')
    role.store()

    jump.by = [role.id]
    workflow.store()
    user.roles = [role.id]
    user.store()

    login(app, username='foo', password='foo')
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)
    resp = app.post(formdata.get_url() + 'jump/trigger/XXX', status=302)

    assert len(formdef.data_class().get(formdata.id).evolution) == 4
    assert formdef.data_class().get(formdata.id).status == 'wf-st1'

    resp = app.get(formdata.get_url())
    assert resp.body.count('Status1') == 2 # once in summary and once in journal
    assert resp.body.count('CSS-STATUS1') == 1

def test_display_message(pub):
    user = create_user(pub)

    formdef = create_formdef()
    formdef.fields = []
    formdef.store()

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    display2 = DisplayMessageWorkflowStatusItem()
    display2.message = 'message-to-submitter'
    display2.to = ['_submitter']
    st1.items.append(display2)
    display2.parent = st1

    display3 = DisplayMessageWorkflowStatusItem()
    display3.message = 'message-to-nobody'
    display3.to = ['xxx']
    st1.items.append(display3)
    display3.parent = st1

    display4 = DisplayMessageWorkflowStatusItem()
    display4.message = 'message-to-xxx-and-submitter'
    display4.to = ['_submitter', 'xxx']
    st1.items.append(display4)
    display4.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    page = app.get('/test/')
    page = page.forms[0].submit('submit') # form page
    page = page.forms[0].submit('submit') # confirmation page
    page = page.follow()

    assert 'message-to-all' in page.body
    assert 'message-to-submitter' in page.body
    assert 'message-to-nobody' not in page.body
    assert 'message-to-xxx-and-submitter' in page.body
    assert page.body.index('message-to-submitter') < page.body.index('message-to-xxx-and-submitter')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    # actions alert vs top alert
    display2.position = 'actions'
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.body
    assert 'message-to-submitter' not in page.body
    assert 'message-to-xxx-and-submitter' in page.body

    # add an action, so display2 will appear again
    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_submitter']
    jump1.status = st1.id
    jump1.parent = st1
    st1.items.append(jump1)
    workflow.store()

    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.body
    assert 'message-to-submitter' in page.body
    assert 'message-to-xxx-and-submitter' in page.body
    assert page.body.index('message-to-submitter') > page.body.index('message-to-xxx-and-submitter')

    jump1.by = ['xxx']
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.body
    assert 'message-to-submitter' not in page.body
    assert 'message-to-xxx-and-submitter' in page.body

    # change to always display at the bottom
    display2.position = 'bottom'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'message-to-all' in page.body
    assert 'message-to-submitter' in page.body
    assert 'message-to-xxx-and-submitter' in page.body
    assert page.body.index('message-to-submitter') > page.body.index('message-to-xxx-and-submitter')

    # set a level
    display2.level = 'warning'
    workflow.store()
    page = app.get(formdata.get_url())
    assert 'warningnotice' in page.body

def test_session_cookie_flags(pub):
    formdef = create_formdef()
    app = get_app(pub)
    resp = app.get('/test/', status=200)
    resp = resp.form.submit('submit')
    assert resp.headers['Set-Cookie'].startswith('wcs-')
    assert 'httponly' in resp.headers['Set-Cookie']
    assert not 'secure' in resp.headers['Set-Cookie']

    app = get_app(pub, https=True)
    resp = app.get('/test/', status=200)
    resp = resp.form.submit('submit')
    assert resp.headers['Set-Cookie'].startswith('wcs-')
    assert 'httponly' in resp.headers['Set-Cookie']
    assert 'secure' in resp.headers['Set-Cookie']

def test_form_page_profile_verified_prefill(pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string',
        prefill={'type': 'user', 'value': 'email'})]
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert resp.form['f0'].value == ''

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert not 'readonly' in resp.form['f0'].attrs
    resp.form['f0'].value = 'Hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert resp.form['f0'].value == 'Hello'

    user.verified_fields = ['email']
    user.store()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    assert 'readonly' in resp.form['f0'].attrs

    resp.form['f0'].value = 'Hello' # try changing the value
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert resp.form['f0'].value == 'foo@localhost'  # it is reverted

    resp.form['f0'].value = 'Hello' # try again changing the value
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].data['0'] == 'foo@localhost'

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert resp.form['f0'].value == 'foo@localhost'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp.form['f0'].value = 'Hello' # try changing
    resp = resp.form.submit('previous')
    assert 'readonly' in resp.form['f0'].attrs
    assert not 'Check values then click submit.' in resp.body
    assert resp.form['f0'].value == 'foo@localhost'

def test_item_field_with_disabled_items(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemField(id='0', label='string', type='item',
        data_source=ds, display_disabled_items=True)]
    formdef.store()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert '<option disabled="disabled" value="1">hello</option>' in resp.body
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        assert '<option disabled="disabled" value="1">hello</option>' in resp.body
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit') # -> validation page
        assert 'There were errors processing the form' in resp.body

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='string', type='item', data_source=ds,
        display_disabled_items=False)]
    formdef.store()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not '<option disabled="disabled" value="1">hello</option>' in resp.body
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemField(id='0', label='string', type='item',
        data_source=ds, show_as_radio=True, display_disabled_items=True)]
    formdef.store()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert '<input disabled="disabled" type="radio" name="f0" value="1" />' in resp.body
        resp.form['f0'] = '1'
        resp.form['f0'] = '2'
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == '2'
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

        resp = get_app(pub).get('/test/')
        assert '<input disabled="disabled" type="radio" name="f0" value="1" />' in resp.body
        resp.form['f0'] = '1'
        resp = resp.form.submit('submit') # -> validation page
        assert 'There were errors processing the form' in resp.body

def test_items_field_with_disabled_items(http_requests, pub):
    user = create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()
    ds = {'type': 'json', 'value': 'http://remote.example.net/json'}
    formdef.fields = [fields.ItemsField(id='0', label='string', type='items',
        data_source=ds, display_disabled_items=True)]
    formdef.store()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello'}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['1', '2']
        assert formdef.data_class().select()[0].data['0_display'] == 'hello, world'

    formdef.data_class().wipe()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert resp.form['f0$element1'].attrs['disabled']
        resp.form['f0$element1'].checked = True
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

    formdef.data_class().wipe()
    formdef.fields = [fields.ItemsField(id='0', label='string', type='items',
        data_source=ds, display_disabled_items=False)]
    formdef.store()

    with mock.patch('qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'disabled': True}, {'id': '2', 'text': 'world'}]}
        urlopen.side_effect = lambda *args: StringIO.StringIO(json.dumps(data))
        resp = get_app(pub).get('/test/')
        assert not 'f0$element1' in resp.form.fields
        resp.form['f0$element2'].checked = True
        resp = resp.form.submit('submit') # -> validation page
        resp = resp.form.submit('submit') # -> submit
        assert formdef.data_class().select()[0].data['0'] == ['2']
        assert formdef.data_class().select()[0].data['0_display'] == 'world'

def test_form_data_keywords(pub):
    formdef = create_formdef()
    formdef.keywords = 'hello,world'
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert 'data-keywords="hello world"' in resp.body
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.body
    assert 'data-keywords="hello world"' in resp.body
    resp = resp.form.submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.body
    assert formdef.data_class().count() == 1

def test_logged_errors(pub):
    Workflow.wipe()
    workflow = Workflow.get_default_workflow()
    workflow.id = '12'
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = 'rejected'
    jump.condition = {'type': 'python', 'value': '1/0'}  # ZeroDivisionError
    st1 = workflow.possible_status[0]
    st1.items.insert(0, jump)
    jump.parent = st1
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.id = '34'
    formdef.workflow = workflow
    formdef.name = 'test'
    formdef.confirmation = False
    formdef.fields = []
    formdef.store()

    LoggedError.wipe()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    error = LoggedError.get_on_index(
            '34-12-zerodivisionerror-integer-division-or-modulo-by-zero', 'tech_id')
    assert error.occurences_count == 2

    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', '34')) == 1
    assert len(LoggedError.get_ids_with_indexed_value('formdef_id', 'X')) == 0

    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', '12')) == 1
    assert len(LoggedError.get_ids_with_indexed_value('workflow_id', 'X')) == 0

def test_formdata_named_wscall(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    comment = RegisterCommenterWorkflowStatusItem()
    comment.id = '_comment'
    comment.comment = 'Hello [webservice.hello_world.foo] World'
    st1.items.append(comment)
    comment.parent = st1

    display = DisplayMessageWorkflowStatusItem()
    display.message = 'The form has been recorded and: X[webservice.hello_world.foo]Y'
    display.to = []
    st1.items.append(display)
    display.parent = st1

    wf.store()

    formdef = create_formdef()
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.body

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'

    # check with publisher variable in named webservice call
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': '[example_url]json'}
    wscall.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.body
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.body

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'

def test_formdata_named_wscall_in_conditions(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json', 'method': 'GET'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    formdef = create_formdef()
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(id='1', label='2nd page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'}),
        fields.PageField(id='1', label='3rd page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] != "bar"'}),
        fields.PageField(id='1', label='4th page', type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'}),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '>1st page<' in resp.body
    assert '>2nd page<' in resp.body
    assert '>3rd page<' not in resp.body
    assert '>4th page<' in resp.body
    assert len(http_requests.requests) == 1

def test_resubmit(pub):
    user = create_user(pub)

    formdef = FormDef()
    formdef.name = 'form title'
    formdef.fields = [fields.StringField(id='1', label='string', varname='toto')]
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'form title bis'
    formdef2.enable_tracking_codes = True
    formdef2.fields = [fields.StringField(id='1', label='string', varname='titi'),
                       fields.StringField(id='2', label='string', varname='toto')]
    formdef2.store()

    wf = Workflow(name='resubmit')
    st1 = wf.add_status('Status1')
    st2 = wf.add_status('Status2')

    resubmit = ResubmitWorkflowStatusItem()
    resubmit.id = '_resubmit'
    resubmit.by = ['_submitter']
    resubmit.formdef_slug = formdef2.url_name
    st1.items.append(resubmit)
    resubmit.parent = st1

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1

    register_comment = RegisterCommenterWorkflowStatusItem()
    register_comment.id = '_register'
    register_comment.comment = '<p><a href="[resubmit_formdata_draft_url]">new draft</a></p>'
    st2.items.append(register_comment)
    register_comment.parent = st2

    wf.store()

    formdef.workflow_id = wf.id
    formdef.store()

    formdef2.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.just_created()
    formdata.data = {'1': 'XXX'}
    formdata.store()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdata.get_url())
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'new draft' in resp.body
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data.get('1') is None
    assert formdef2.data_class().select()[0].data.get('2') == 'XXX'
    resp = resp.click('new draft')
    resp = resp.follow()
    assert resp.forms[1]['f2'].value == 'XXX'

def test_form_custom_select_template(pub):
    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Baz'])]
    formdef.store()
    formdef.data_class().wipe()
    resp = get_app(pub).get('/test/')
    assert not 'TEST TEMPLATE' in resp.body
    formdef.fields[0].extra_css_class = 'template-test'
    formdef.store()

    resp = get_app(pub).get('/test/')
    assert 'TEST TEMPLATE' in resp.body
    # make sure request is available in context
    assert '<!-- backoffice: False -->' in resp.body

def test_form_status_appearance_keywords(pub):
    create_user(pub)
    formdef = create_formdef()
    formdef.data_class().wipe()

    formdef = create_formdef()
    formdef.fields = [
            fields.ItemField(id='1', label='select', type='item',
                required=True,
                varname='foo', items=['Foo', 'Bar', 'Baz'])]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    root = PublicFormStatusPage(formdef, formdata, register_workflow_subdirs=False)
    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == root.status_templates

    formdef.appearance_keywords = 'foobar plop'
    formdef.store()

    template_names = root.get_formdef_template_variants(root.status_templates)
    assert list(template_names) == [
        'wcs/front/appearance-foobar/formdata_status.html',
        'wcs/front/appearance-plop/formdata_status.html',
        'wcs/front/formdata_status.html',
        'wcs/appearance-foobar/formdata_status.html',
        'wcs/appearance-plop/formdata_status.html',
        'wcs/formdata_status.html']

    resp = get_app(pub).get('/test/')
    assert 'class="quixote foobar plop"' in resp.body

def test_user_global_action(pub):
    user = create_user(pub)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]

    workflow.store()

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {}
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get(formdef.get_url())
    resp = resp.form.submit('submit')
    resp = resp.form.submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]

    resp = app.get(formdata.get_url())
    assert not 'button-action-1' in resp.body

    trigger.roles = ['_submitter']
    workflow.store()

    resp = app.get(formdata.get_url())
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get(formdata.get_url())
    assert 'HELLO WORLD GLOBAL ACTION' in resp.body
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'