wcs/tests/form_pages/test_formdata.py

# -*- coding: utf-8 -*-

import base64
import io
import json
import locale
import os
import urllib.parse
import xml.etree.ElementTree as ET
import zipfile

import mock
import pytest
from django.utils.encoding import force_bytes
from quixote.http_request import Upload as QuixoteUpload
from utilities import clean_temporary_pub, create_temporary_pub, get_app, login
from webtest import Hidden, Upload

from wcs import fields
from wcs.data_sources import NamedDataSource
from wcs.formdef import FormDef
from wcs.qommon.form import UploadedFile
from wcs.wf.attachment import AddAttachmentWorkflowStatusItem
from wcs.wf.export_to_model import ExportToModel, transform_to_pdf
from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
from wcs.wf.jump import JumpWorkflowStatusItem
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
from wcs.workflows import (
    ChoiceWorkflowStatusItem,
    DisplayMessageWorkflowStatusItem,
    Workflow,
    WorkflowBackofficeFieldsFormDef,
)
from wcs.wscalls import NamedWsCall

from .test_all import create_user, create_user_and_admin


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates', 'pickle-lazy'], indirect=True)


@pytest.fixture
def pub(request, emails):
    pub = create_temporary_pub(
        sql_mode=bool('sql' in request.param),
        templates_mode=bool('templates' in request.param),
        lazy_mode=bool('lazy' in request.param),
    )
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    return pub


def teardown_module(module):
    clean_temporary_pub()


def assert_equal_zip(stream1, stream2):
    z1 = zipfile.ZipFile(stream1)
    z2 = zipfile.ZipFile(stream2)
    assert set(z1.namelist()) == set(z2.namelist())
    for name in z1.namelist():
        if name == 'styles.xml':
            continue
        if name in ['content.xml', 'meta.xml']:
            t1, t2 = ET.tostring(ET.XML(z1.read(name))), ET.tostring(ET.XML(z2.read(name)))
            try:
                # >= python 3.8: tostring preserves attribute order; use canonicalize to sort them
                t1, t2 = ET.canonicalize(t1), ET.canonicalize(t2)
            except AttributeError:
                pass
        else:
            t1, t2 = z1.read(name), z2.read(name)
        assert t1 == t2, 'file "%s" differs' % name


def test_formdata_attachment_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow()  # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'


def test_formdata_attachment_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.varname = 'attached_doc'
    attach.id = '_attach'
    attach.by = ['_submitter']
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'

    resp = resp.follow()  # back to form page
    resp = resp.click('test.txt')
    assert resp.location.endswith('/test.txt')
    resp = resp.follow()
    assert resp.content_type == 'text/plain'
    assert resp.text == 'foobar'

    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    attachment_variable = attachments.attached_doc

    resp = login(get_app(pub), username='admin', password='admin').get(attachment_variable.url).follow()
    assert attachment_variable.content == resp.body
    assert attachment_variable.b64_content == base64.b64encode(resp.body)
    assert attachment_variable.content_type == resp._headers['content-type'].split(';')[0]
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(attachment_variable.filename)


def test_formdata_attachment_download_to_backoffice_file_field(pub):
    create_user(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
    ]
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    attach.backoffice_filefield_id = 'bo1'
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    assert attach.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    # backoffice file field is set
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'bo1' in formdata.data
    bo1 = formdata.data['bo1']
    assert bo1.base_filename == 'test.txt'
    assert bo1.content_type == 'text/plain'
    assert bo1.get_content() == b'foobar'

    # and file is in history, too
    assert formdata.evolution[-1].parts[0].__class__.__name__ == 'AttachmentEvolutionPart'
    attachment = formdata.evolution[-1].parts[0]
    assert attachment.content_type == 'text/plain'
    assert attachment.orig_filename == 'test.txt'


def test_formdata_attachment_download_to_backoffice_file_field_only(pub):
    create_user(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
    ]
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    attach.backoffice_filefield_id = 'bo1'
    attach.attach_to_history = False  # store only in backoffice field
    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    assert attach.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp.forms[0]['attachment_attach$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('button_attach')

    # backoffice file field is set
    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'bo1' in formdata.data
    bo1 = formdata.data['bo1']
    assert bo1.base_filename == 'test.txt'
    assert bo1.content_type == 'text/plain'
    assert bo1.get_content() == b'foobar'

    # but nothing in history
    for evo in formdata.evolution:
        assert not evo.parts


def test_formdata_attachment_file_options(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    attach = AddAttachmentWorkflowStatusItem()
    attach.id = '_attach'
    attach.by = ['_submitter']
    attach.document_type = {'label': 'Fichiers vidéo', 'mimetypes': ['video/*'], 'id': '_video'}
    attach.max_file_size = '3Mo'

    st1.items.append(attach)
    attach.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    file_input = resp.forms[0]['attachment_attach$file']
    assert file_input.attrs['accept'] == 'video/*'
    assert file_input.attrs['data-max-file-size'] == '3000000'


def test_formdata_generated_document_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.text == 'HELLO WORLD'

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    resp = resp.click('test.rtf')
    assert resp.location.endswith('/test.rtf')
    resp = resp.follow()
    assert resp.content_type == 'application/rtf'
    assert resp.text == 'HELLO WORLD'

    # change export model to now be a RTF file, do the action again on the same form and
    # check that both the old .odt file and the new .rtf file are there and valid.
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    assert resp.click('test.rtf', index=0).follow().text == 'HELLO WORLD'
    assert resp.click('test.rtf', index=1).follow().text == 'HELLO NEW WORLD'

    # use substitution variables on rtf: only ezt format is accepted
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO {{DJANGO}} WORLD [form_name]')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()

    assert resp.click('test.rtf', index=2).follow().text == 'HELLO {{DJANGO}} WORLD {\\uc1{test}}'


@pytest.fixture(params=['template.odt', 'template-django.odt'])
def odt_template(request):
    return request.param


def test_formdata_generated_document_odt_download(pub, odt_template):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    template_filename = os.path.join(os.path.dirname(__file__), '..', odt_template)
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/' + odt_template, content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        assert_equal_zip(io.BytesIO(resp.body), f)

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    resp = resp.follow()  # $form/$id/create_doc
    with mock.patch('wcs.wf.export_to_model.get_formdata_template_context') as get_context_1:
        with mock.patch('wcs.workflows.get_formdata_template_context') as get_context_never:
            get_context_1.return_value = {}
            get_context_never.return_value = {}
            resp = resp.follow()  # $form/$id/create_doc/
            # substitution variables are computed only one :
            assert get_context_1.call_count == 1
            assert get_context_never.call_count == 0

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    resp = resp.click(odt_template)
    assert resp.location.endswith('/' + odt_template)
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        assert_equal_zip(io.BytesIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        body = resp.click(odt_template, index=0).follow().body
        assert_equal_zip(io.BytesIO(body), f)
    assert resp.click('test.rtf', index=0).follow().body == b'HELLO NEW WORLD'


def test_formdata_generated_document_odt_download_with_substitution_variable(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        assert_equal_zip(io.BytesIO(resp.body), f)

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    response1 = resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        assert_equal_zip(io.BytesIO(resp.body), f)

    # change file content, same name
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO NEW WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        body = resp.click('template.odt', index=0).follow().body
        assert_equal_zip(io.BytesIO(body), f)
    response2 = resp.click('test.rtf', index=0).follow()
    assert response2.body == b'HELLO NEW WORLD'
    # Test attachment substitution variables
    variables = formdef.data_class().select()[0].get_substitution_variables()
    assert 'attachments' in variables
    attachments = variables['attachments']
    assert attachments is not None
    file1 = attachments.created_doc
    assert file1.content == response2.body
    assert file1.b64_content == base64.b64encode(response2.body)
    assert file1.content_type == response2._headers['content-type']
    content_disposition = response2._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response2.request.environ['PATH_INFO'].endswith(file1.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file1.url).follow()
    assert file1.content == resp.body
    assert file1.b64_content == base64.b64encode(resp.body)
    assert file1.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file1.filename)

    file2 = attachments.created_doc[1]
    assert file2.content == response1.body
    assert file2.b64_content == base64.b64encode(response1.body)
    assert file2.content_type == response1._headers['content-type']
    content_disposition = response1._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert response1.request.environ['PATH_INFO'].endswith(file2.filename)

    resp = login(get_app(pub), username='admin', password='admin').get(file2.url).follow()
    assert file2.content == resp.body
    assert file2.b64_content == base64.b64encode(resp.body)
    assert file2.content_type == resp._headers['content-type']
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'attachment'
    assert resp.request.environ['PATH_INFO'].endswith(file2.filename)


@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_export_to')

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    resp = resp.form.submit('button_export_to')
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    content_disposition = resp._headers['content-disposition']
    assert len(content_disposition.split(';')) == 2
    assert content_disposition.split(';')[0] == 'inline'
    assert resp.body.startswith(b'%PDF-')


@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_formdata_generated_document_odt_to_pdf_download_push_to_portfolio(
    pub, fargo_url, fargo_secret, caplog
):
    user = create_user(pub)
    user.name = 'Foo Baré'
    user.store()

    pub.cfg['debug'] = {'logger': True}
    pub.write_cfg()
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    export_to.varname = 'created_doc'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    export_to.convert_to_pdf = True
    export_to.push_to_portfolio = True
    st1.items.append(export_to)
    export_to.parent = st1
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    form_location = resp.location
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        if locale.getpreferredencoding() == 'UTF-8':
            assert "file 'template.pdf' pushed to portfolio of 'Foo Baré'" == caplog.records[-1].message
        else:  # Python < 3.7
            assert "file 'template.pdf' pushed to portfolio of 'Foo Bar\xe9'" == caplog.records[-1].message

    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 400, 'null', None  # fail
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert caplog.records[-1].message.startswith(
            "file 'template.pdf' failed to be pushed to portfolio of 'Foo"
        )

    # failed to push to portfolio, but document is here
    resp = resp.follow()  # $form/$id/create_doc
    resp = resp.follow()  # $form/$id/create_doc/
    assert resp.content_type == 'application/pdf'
    assert b'PDF' in resp.body

    export_to.attach_to_history = True
    wf.store()

    resp = login(get_app(pub), username='foo', password='foo').get(form_location)
    with mock.patch('wcs.portfolio.http_post_request') as http_post_request:
        http_post_request.return_value = None, 200, 'null', None
        resp = resp.form.submit('button_export_to')
        assert http_post_request.call_count == 1
        assert http_post_request.call_args[0][0].startswith('http://fargo.example.net/api/documents/push/')
        payload = json.loads(http_post_request.call_args[0][1])
        assert payload['file_name'] == 'template.pdf'
        assert payload['user_email'] == 'foo@localhost'
        assert payload['origin'] == 'example.net'
        assert base64.decodebytes(force_bytes(payload['file_b64_content'])).startswith(b'%PDF')
        assert caplog.records[-1].message.startswith("file 'template.pdf' pushed to portfolio of 'Foo")
    assert resp.location == form_location + '#action-zone'
    resp = resp.follow()  # back to form page

    resp = resp.click('template.pdf')
    assert resp.location.endswith('/template.pdf')
    resp = resp.follow()
    assert resp.content_type == 'application/pdf'
    assert resp.body.startswith(b'%PDF-')


def test_formdata_generated_document_non_interactive(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    st1.items.append(export_to)
    export_to.parent = st1

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    wf.add_status('Status2', 'st2')

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.click('template.odt')
    assert resp.location.endswith('/template.odt')
    resp = resp.follow()
    assert resp.content_type == 'application/octet-stream'
    with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
        assert_equal_zip(io.BytesIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'


def test_formdata_generated_document_to_backoffice_field(pub):
    create_user_and_admin(pub)
    wf = Workflow(name='status')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
        fields.StringField(id='bo2', label='bo field 2', type='string'),
    ]

    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.convert_to_pdf = False
    export_to.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    template = open(template_filename, 'rb').read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.id = '_export_to'
    export_to.attach_to_history = True
    export_to.backoffice_filefield_id = 'bo1'
    st1.items.append(export_to)
    export_to.parent = st1

    assert export_to.get_backoffice_filefield_options() == [('bo1', 'bo field 1', 'bo1')]

    jump = JumpWorkflowStatusItem()
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1
    wf.add_status('Status2', 'st2')
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [fields.TextField(id='0', label='comment', type='text', varname='comment')]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp.form['f0'] = 'Hello\n\nWorld.'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    # get the two generated files from backoffice: in backoffice fields
    # (export_to.backoffice_filefield_id), and in history (export_to.attach_to_history)
    for index in (0, 1):
        resp = login(get_app(pub), username='admin', password='admin').get('/backoffice/management/test/1/')
        resp = resp.click('template.odt', index=index)
        assert resp.location.endswith('/template.odt')
        resp = resp.follow()
        assert resp.content_type == 'application/octet-stream'
        with open(os.path.join(os.path.dirname(__file__), '..', 'template-out.odt'), 'rb') as f:
            assert_equal_zip(io.BytesIO(resp.body), f)

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'


def test_formdata_generated_document_in_private_history(pub):
    user = create_user(pub)

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    user.roles = [role.id]
    user.store()

    wf = Workflow(name='status')
    st0 = wf.add_status('Status0', 'st0')
    st1 = wf.add_status('Status1', 'st1')
    export_to = ExportToModel()
    export_to.label = 'create doc'
    upload = QuixoteUpload('/foo/test.rtf', content_type='application/rtf')
    upload.fp = io.BytesIO()
    upload.fp.write(b'HELLO WORLD')
    upload.fp.seek(0)
    export_to.model_file = UploadedFile(pub.app_dir, None, upload)
    export_to.attach_to_history = True
    export_to.id = '_export_to'
    export_to.by = ['_submitter']
    st1.items.append(export_to)
    export_to.parent = st1

    st2 = wf.add_status('Status2', 'st2')

    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_receiver']
    jump1.status = st1.id
    jump1.parent = st0
    st0.items.append(jump1)

    jump2 = ChoiceWorkflowStatusItem()
    jump2.id = '_jump2'
    jump2.label = 'Jump 2'
    jump2.by = ['_receiver']
    jump2.status = st2.id
    jump2.parent = st1
    st1.items.append(jump2)

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_jump1')
    resp = resp.follow()

    resp = resp.form.submit('button_export_to')
    resp = resp.follow()
    assert 'Form exported in a model' in resp.text

    resp = resp.form.submit('button_jump2')
    resp = resp.follow()

    # limit visibility of status with document
    st1.visibility = ['_receiver']
    wf.store()

    formdata = formdef.data_class().select()[0]
    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url())
    assert 'Form exported in a model' not in resp.text

    # check status is visible in backoffice
    resp = login(get_app(pub), username='foo', password='foo').get(formdata.get_url(backoffice=True))
    assert 'visibility-off' in resp.text
    assert 'Form exported in a model' in resp.text


def test_formdata_empty_form_action(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    wf.add_status('Status2', 'st2')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    st1.items.append(display_form)
    display_form.parent = st1

    jump1 = ChoiceWorkflowStatusItem()
    jump1.id = '_jump1'
    jump1.label = 'Jump 1'
    jump1.by = ['_submitter']
    jump1.status = 'st2'
    jump1.parent = st1
    st1.items.append(jump1)

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    resp = resp.form.submit('button_jump1')
    assert formdef.data_class().select()[0].status == 'wf-st2'


def test_formdata_form_file_download(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(fields.FileField(id='1', label='File', type='file', varname='yyy'))
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    assert 'qommon.fileupload.js' in resp.text
    resp.forms[0]['fxxx_1$file'] = Upload('test.txt', b'foobar', 'text/plain')
    resp = resp.forms[0].submit('submit')

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    assert 'xxx_var_yyy_raw' in formdata.workflow_data

    download = resp.test_app.get(urllib.parse.urljoin(resp.location, 'files/form-xxx-yyy/test.txt'))
    assert download.content_type == 'text/plain'
    assert download.body == b'foobar'

    # go back to the status page, this will exercise the substitution variables
    # codepath.
    resp = resp.follow()


def test_formdata_workflow_form_prefill(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(
        fields.StringField(
            id='1', label='blah', type='string', varname='yyy', prefill={'type': 'user', 'value': 'email'}
        )
    )
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert resp.forms[0]['fxxx_1'].value == 'foo@localhost'


def test_formdata_workflow_form_prefill_conditional_field(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(
        fields.StringField(
            id='1',
            label='blah1',
            type='string',
            varname='yyy1',
            prefill={'type': 'user', 'value': 'email'},
            condition={'type': 'django', 'value': '0'},
        )
    )
    display_form.formdef.fields.append(
        fields.StringField(
            id='2',
            label='blah2',
            type='string',
            varname='yyy2',
            prefill={'type': 'user', 'value': 'email'},
            condition={'type': 'django', 'value': '1'},
        )
    )
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert resp.forms[0]['fxxx_2'].value == 'foo@localhost'


def test_formdata_workflow_form_prefill_checkbox(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(
        fields.BoolField(
            id='1', label='blah', type='bool', varname='yyy', prefill={'type': 'formula', 'value': 'True'}
        )
    )
    display_form.formdef.fields.append(
        fields.BoolField(
            id='2', label='blah2', type='bool', varname='zzz', prefill={'type': 'formula', 'value': 'True'}
        )
    )
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp
    resp = resp.form.submit('submit').follow()
    assert 'The form has been recorded' in resp
    assert resp.form['fxxx_1'].checked is True
    assert resp.form['fxxx_2'].checked is True
    resp.form['fxxx_1'].checked = False
    resp = resp.form.submit('submit')

    formdata = formdef.data_class().select()[0]
    assert formdata.workflow_data['xxx_var_yyy_raw'] is False
    assert formdata.workflow_data['xxx_var_zzz_raw'] is True


def test_formdata_workflow_form_prefill_autocomplete(pub):
    create_user(pub)

    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {'type': 'json', 'value': 'http://local-mock/test'}
    data_source.query_parameter = 'q'
    data_source.id_parameter = 'id'
    data_source.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = FormWorkflowStatusItem()
    display_form.id = '_x'
    display_form.by = ['_submitter']
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
        fields.ItemField(
            id='4',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            required=False,
            display_mode='autocomplete',
            prefill={'type': 'string', 'value': '{{ form_var_foo_raw }}'},
        ),
    ]
    st1.items.append(display_form)
    display_form.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            data_source={'type': 'foobar'},
            display_mode='autocomplete',
            required=False,
            varname='foo',
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': '1', 'text': 'hello', 'extra': 'foo'}]}

        def side_effect(url, *args):
            return io.StringIO(json.dumps(data))

        urlopen.side_effect = side_effect

        assert 'data-select2-url=' in resp
        # simulate select2
        resp.form.fields['f0_display'] = Hidden(form=resp.form, tag='input', name='f0_display', pos=10)
        resp.form['f0'].force_value('1')
        resp.form.fields['f0_display'].force_value('foo')
        resp = resp.form.submit('submit')
        assert 'Check values then click submit.' in resp
        resp = resp.form.submit('submit').follow()
        assert 'The form has been recorded' in resp

        # check display value is in form action widget
        assert resp.form['fxxx_4'].attrs['data-value'] == '1'
        assert resp.form['fxxx_4'].attrs['data-initial-display-value'] == 'hello'

        # check it is also displayed in a fresh session
        resp = login(get_app(pub), username='foo', password='foo').get(resp.request.url)
        assert resp.form['fxxx_4'].attrs['data-value'] == '1'
        assert resp.form['fxxx_4'].attrs['data-initial-display-value'] == 'hello'


def test_formdata_workflow_many_forms(pub):
    create_user(pub)
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    # first form
    display_form1 = FormWorkflowStatusItem()
    display_form1.id = '_x'
    display_form1.by = ['_submitter']
    display_form1.varname = 'xxx'
    display_form1.formdef = WorkflowFormFieldsFormDef(item=display_form1)
    display_form1.formdef.fields = [fields.StringField(id='1', label='blah1')]
    st1.items.append(display_form1)
    display_form1.parent = st1

    # second form with live condition
    display_form2 = FormWorkflowStatusItem()
    display_form2.id = '_y'
    display_form2.by = ['_submitter']
    display_form2.varname = 'yyy'
    display_form2.formdef = WorkflowFormFieldsFormDef(item=display_form2)
    display_form2.formdef.fields = [
        fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
        fields.StringField(
            id='2',
            label='Test2',
            varname='str2',
            type='string',
            required=True,
            condition={'type': 'django', 'value': 'yyy_var_str == "xxx"'},
        ),
    ]
    st1.items.append(display_form2)
    display_form2.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = [fields.StringField(id='1', label='blah')]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp.form['f1'] = 'foobar'
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert 'fxxx_1' in resp.form.fields
    assert 'fyyy_1' in resp.form.fields
    assert 'fyyy_2' in resp.form.fields
    assert 'submit' in resp.form.fields  # only one submit button

    assert resp.html.find('div', {'data-field-id': 'yyy_1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': 'yyy_2'}).attrs.get('style') == 'display: none'
    live_url = resp.html.find('form').attrs['data-live-url']
    resp.form['fyyy_1'] = ''
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['xxx_1']['visible']
    assert live_resp.json['result']['yyy_1']['visible']
    assert not live_resp.json['result']['yyy_2']['visible']
    resp.form['fyyy_1'] = 'xxx'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['xxx_1']['visible']
    assert live_resp.json['result']['yyy_1']['visible']
    assert live_resp.json['result']['yyy_2']['visible']


def test_formdata_named_wscall(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    comment = RegisterCommenterWorkflowStatusItem()
    comment.id = '_comment'
    comment.comment = 'Hello [webservice.hello_world.foo] World'
    st1.items.append(comment)
    comment.parent = st1

    display = DisplayMessageWorkflowStatusItem()
    display.message = 'The form has been recorded and: X[webservice.hello_world.foo]Y'
    display.to = []
    st1.items.append(display)
    display.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'

    # check with publisher variable in named webservice call
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': '[example_url]json'}
    wscall.store()

    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded and: XbarY' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello bar World'


def test_formdata_named_wscall_in_conditions(http_requests, pub):
    create_user(pub)
    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json', 'method': 'GET'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.PageField(
            id='1',
            label='2nd page',
            type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'},
        ),
        fields.PageField(
            id='1',
            label='3rd page',
            type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] != "bar"'},
        ),
        fields.PageField(
            id='1',
            label='4th page',
            type='page',
            condition={'type': 'python', 'value': 'webservice.hello_world["foo"] == "bar"'},
        ),
    ]
    formdef.store()
    formdef.data_class().wipe()

    resp = login(get_app(pub), username='foo', password='foo').get('/test/')
    assert '>1st page<' in resp.text
    assert '>2nd page<' in resp.text
    assert '>3rd page<' not in resp.text
    assert '>4th page<' in resp.text
    assert len(http_requests.requests) == 1


def test_formdata_evolution_registercommenter_to(pub):
    user = create_user(pub)

    pub.role_class.wipe()
    role1 = pub.role_class(name='role the user does not have')
    role1.store()
    role2 = pub.role_class(name='role the user does have')
    role2.store()
    user.roles = [role2.id]
    user.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')
    comment = RegisterCommenterWorkflowStatusItem()
    comment.id = '_comment'
    comment.comment = 'Hello World'
    comment.to = None
    st1.items.append(comment)
    comment.parent = st1

    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.workflow_id = wf.id
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    # register comment to all users
    app = login(get_app(pub), username='foo', password='foo')
    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello World'
    assert formdata.evolution[0].parts[0].to is None
    resp = app.get('/test/%s/' % formdata.id)
    resp.status_int = 200
    assert resp.html.find('div', {'id': 'evolution-log'}).find('p').text == 'Hello World'

    # register comment to other users
    formdef.data_class().wipe()
    comment.to = [role1.id]
    wf.store()

    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello World'
    assert formdata.evolution[0].parts[0].to == [role1.id]
    resp = app.get('/test/%s/' % formdata.id)
    resp.status_int = 200
    assert not resp.html.find('div', {'id': 'evolution-log'}).find('p')

    # register comment to this user
    formdef.data_class().wipe()
    comment.to = [role2.id]
    wf.store()

    resp = app.get('/test/')
    resp = resp.forms[0].submit('submit')
    assert 'Check values then click submit.' in resp.text
    resp = resp.forms[0].submit('submit')
    assert resp.status_int == 302
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text

    formdata = formdef.data_class().select()[0]
    assert formdata.evolution[0].parts[0].content == 'Hello World'
    assert formdata.evolution[0].parts[0].to == [role2.id]
    resp = app.get('/test/%s/' % formdata.id)
    resp.status_int = 200
    assert resp.html.find('div', {'id': 'evolution-log'}).find('p').text == 'Hello World'