wcs/tests/test_formdata.py

import collections
import datetime
import io
import os.path
import time
import uuid
from unittest import mock

import pytest
from quixote import get_publisher
from quixote.http_request import Upload

from wcs import fields, sessions
from wcs.blocks import BlockDef
from wcs.carddef import CardDef
from wcs.categories import Category
from wcs.conditions import Condition
from wcs.data_sources import NamedDataSource
from wcs.formdata import Evolution
from wcs.formdef import FormDef
from wcs.qommon import force_str
from wcs.qommon.http_request import HTTPRequest
from wcs.qommon.misc import file_digest
from wcs.qommon.storage import atomic_write
from wcs.qommon.substitution import CompatibilityNamesDict
from wcs.qommon.template import Template
from wcs.qommon.upload_storage import PicklableUpload
from wcs.variables import LazyFormData
from wcs.wf.register_comment import JournalEvolutionPart
from wcs.wf.wscall import JournalWsCallErrorPart
from wcs.workflows import (
    AttachmentEvolutionPart,
    Workflow,
    WorkflowBackofficeFieldsFormDef,
    WorkflowCriticalityLevel,
    WorkflowStatusItem,
    WorkflowVariablesFieldsFormDef,
)

from .utilities import clean_temporary_pub, create_temporary_pub


@pytest.fixture
def pub(request):
    pub = create_temporary_pub(sql_mode=True)

    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
    pub.set_app_dir(req)
    pub._set_request(req)
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()

    return pub


@pytest.fixture
def formdef(pub):
    FormDef.wipe()

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = []
    formdef.store()

    return formdef


def teardown_module(module):
    clean_temporary_pub()


@pytest.fixture
def local_user():
    get_publisher().user_class.wipe()
    user = get_publisher().user_class()
    user.name = 'Jean Darmette'
    user.email = 'jean.darmette@triffouilis.fr'
    user.name_identifiers = ['0123456789']
    user.store()
    return user


def test_basic(pub, formdef):
    Category.wipe()
    cat = Category(name='test category')
    cat.store()
    formdef.category_id = cat.id
    formdef.store()
    formdata = formdef.data_class()()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_status') == 'Unknown'
    assert substvars.get('form_name') == 'foobar'
    assert substvars.get('form_slug') == 'foobar'
    assert substvars.get('category_name') == 'test category'


def test_saved(pub, formdef):
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.store()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_number') == '%s-1' % formdef.id
    assert substvars.get('form_number_raw') == '1'
    assert substvars.get('form_url').endswith('/foobar/1/')
    assert substvars.get('form_url_backoffice').endswith('/backoffice/management/foobar/1/')
    assert substvars.get('form_status_url').endswith('/foobar/1/status')


def test_auto_display_id(pub, formdef):
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.store()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_number') == '%s-%s' % (formdef.id, formdata.id)
    assert substvars.get('form_number_raw') == str(formdata.id)


def test_manual_display_id(pub, formdef):
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.id_display = 'bar'
    formdata.store()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_number') == 'bar'
    assert substvars.get('form_number_raw') == str(formdata.id)


def test_submission_context(pub, formdef, local_user):
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.backoffice_submission = True
    formdata.submission_channel = 'mail'
    formdata.submission_agent_id = str(local_user.id)
    formdata.submission_context = {
        'mail_url': 'http://www.example.com/test.pdf',
    }
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_submission_backoffice') is True
    assert substvars.get('form_submission_channel') == 'mail'
    assert substvars.get('form_submission_channel_label') == 'Mail'
    assert substvars.get('form_submission_context_mail_url') == 'http://www.example.com/test.pdf'
    assert substvars.get('form_submission_agent_email') == local_user.email

    formdata = formdef.data_class()()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_submission_backoffice') is False
    assert substvars.get('form_submission_channel') is None
    assert substvars.get('form_submission_channel_label') == 'Web'
    assert substvars.get('form_submission_agent_email') is None


def test_just_created(pub, formdef):
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_status') == 'Just Submitted'
    assert substvars.get('form_status_is_endpoint') is False
    assert substvars.get('form_receipt_date')
    assert substvars.get('form_receipt_time')
    assert substvars.get('form_receipt_datetime')
    assert substvars.get('form_last_update_datetime')
    assert substvars.get('form_evolution')


def test_field(pub, formdef):
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string')]
    formdef.store()
    formdata = formdef.data_class()()
    substvars = formdata.get_substitution_variables()
    assert not substvars.get('form_f0')

    formdata.data = {'0': 'test'}
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_f0') == 'test'
    assert substvars.get('form_field_string') == 'test'


def test_field_varname(pub, formdef):
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', varname='foo')]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': 'test'}
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_f0') == 'test'
    assert substvars.get('form_var_foo') == 'test'


def test_file_field(pub, formdef):
    formdef.data_class().wipe()
    formdef.fields = [fields.FileField(id='0', label='file', varname='foo')]
    formdef.store()
    formdata = formdef.data_class()()
    upload = Upload('test.txt', 'text/plain', 'ascii')
    upload.receive([b'first line', b'second line'])
    formdata.data = {'0': upload}
    formdata.id = 1
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_var_foo') == 'test.txt'
    assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0')
    assert isinstance(substvars.get('form_var_foo_raw'), Upload)

    formdata.data = {'0': None}
    substvars = formdata.get_substitution_variables()
    assert substvars['form_var_foo'] is None
    assert substvars['form_var_foo_raw'] is None
    assert substvars['form_var_foo_url'] is None

    formdata.data = {}
    substvars = formdata.get_substitution_variables()
    assert substvars['form_var_foo'] is None
    assert substvars['form_var_foo_raw'] is None
    assert substvars['form_var_foo_url'] is None


def test_get_submitter(pub, formdef):
    BlockDef.wipe()
    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.StringField(id='0', label='email', varname='foo', prefill={'type': 'user', 'value': 'email'})
    ]
    block.store()
    formdef.data_class().wipe()
    formdef.fields = [
        fields.StringField(id='0', label='email', varname='foo', prefill={'type': 'user', 'value': 'email'}),
        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='block'),
        fields.StringField(id='2', label='other'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    assert formdef.get_submitter_email(formdata) is None
    formdata.data = {'0': 'foo@localhost'}
    assert formdef.get_submitter_email(formdata) == 'foo@localhost'

    formdata.data = {
        '1': {
            'data': [{'0': 'baz@localhost'}, {'0': ''}],
            'schema': {'0': 'string'},
        }
    }
    assert formdef.get_submitter_email(formdata) == 'baz@localhost'

    formdata.data = {
        '1': {
            'data': [{'0': 'baz@localhost'}, {'0': 'foo@localhost'}],
            'schema': {'0': 'string'},
        }
    }
    assert formdef.get_submitter_email(formdata) == 'baz@localhost'

    formdata.data = {
        '1': {
            'data': [{'0': ''}, {'0': 'foo@localhost'}],
            'schema': {'0': 'string'},
        }
    }
    assert formdef.get_submitter_email(formdata) == 'foo@localhost'

    formdata.data = {'1': {}}
    assert formdef.get_submitter_email(formdata) is None

    formdata.data = {'2': 'other'}
    assert formdef.get_submitter_email(formdata) is None

    formdata.data = {
        '0': 'foo@localhost',
        '1': {
            'data': [{'0': 'baz@localhost'}, {'0': ''}],
            'schema': {'0': 'string'},
        },
    }
    assert formdef.get_submitter_email(formdata) == 'foo@localhost'

    user = pub.user_class()
    user.email = 'bar@localhost'
    user.store()

    formdata.user_id = user.id
    assert formdef.get_submitter_email(formdata) == 'foo@localhost'

    formdata.data = {}
    assert formdef.get_submitter_email(formdata) == 'bar@localhost'


def test_get_last_update_time(pub, formdef):
    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    assert formdata.last_update_time is None

    formdata.just_created()
    assert formdata.last_update_time == formdata.evolution[-1].time

    time.sleep(1)
    evo = Evolution()
    evo.time = time.localtime()
    evo.status = formdata.status
    evo.comment = 'hello world'
    formdata.evolution.append(evo)
    assert formdata.last_update_time != formdata.receipt_time
    assert formdata.last_update_time == formdata.evolution[-1].time

    # check with missing 'evolution' values
    formdata.evolution = None
    assert formdata.last_update_time == formdata.receipt_time


def test_password_field(pub, formdef):
    formdef.data_class().wipe()
    formdef.fields = [fields.PasswordField(id='0', label='pwd')]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': {'cleartext': 'foo'}}
    formdata.store()

    formdata2 = formdata.get(formdata.id)
    assert formdata2.data == {'0': {'cleartext': 'foo'}}


def test_date_field(pub, formdef):
    formdef.data_class().wipe()
    formdef.fields = [fields.DateField(id='0', label='date')]
    formdef.store()
    formdata = formdef.data_class()()
    value = time.strptime('2015-05-12', '%Y-%m-%d')
    formdata.data = {'0': value}
    formdata.store()

    formdata2 = formdata.get(formdata.id)
    assert formdata2.data == {'0': value}

    assert formdata2.get_substitution_variables()['form_field_date'] == '2015-05-12'
    pub.cfg['language'] = {'language': 'fr'}
    assert formdata2.get_substitution_variables()['form_field_date'] == '12/05/2015'
    pub.cfg['language'] = {'language': 'en'}


def test_clean_drafts(pub):
    formdef = FormDef()
    formdef.name = 'foo'
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    d = formdef.data_class()()
    d.status = 'draft'
    d.receipt_time = time.localtime()
    d.store()
    d_id1 = d.id

    d = formdef.data_class()()
    d.status = 'draft'
    d.receipt_time = time.localtime(0)  # epoch, 1970-01-01
    d.store()

    assert formdef.data_class().count() == 2
    from wcs.formdef import clean_drafts

    clean_drafts(pub)
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].id == d_id1

    d = formdef.data_class()()
    d.status = 'draft'
    d.receipt_time = time.localtime(time.time() - 86400 * 5)
    d.store()
    clean_drafts(pub)
    assert formdef.data_class().count() == 2
    formdef.drafts_lifespan = '3'
    formdef.store()
    clean_drafts(pub)
    assert formdef.data_class().count() == 1


def test_criticality_levels(pub):
    workflow = Workflow(name='criticality')
    workflow.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    workflow.store()

    formdef = FormDef()
    formdef.name = 'foo'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    d = formdef.data_class()()
    assert d.get_criticality_level_object().name == 'green'
    d.increase_criticality_level()
    assert d.get_criticality_level_object().name == 'yellow'
    d.increase_criticality_level()
    assert d.get_criticality_level_object().name == 'red'
    d.increase_criticality_level()
    assert d.get_criticality_level_object().name == 'red'
    d.decrease_criticality_level()
    assert d.get_criticality_level_object().name == 'yellow'
    d.decrease_criticality_level()
    assert d.get_criticality_level_object().name == 'green'
    d.decrease_criticality_level()
    assert d.get_criticality_level_object().name == 'green'
    d.set_criticality_level(1)
    assert d.get_criticality_level_object().name == 'yellow'
    d.set_criticality_level(2)
    assert d.get_criticality_level_object().name == 'red'
    d.set_criticality_level(4)
    assert d.get_criticality_level_object().name == 'red'

    workflow.criticality_levels = [WorkflowCriticalityLevel(name='green')]
    workflow.store()
    formdef = FormDef.get(id=formdef.id)  # reload formdef
    d = formdef.data_class()()
    assert d.get_criticality_level_object().name == 'green'
    d.increase_criticality_level()
    assert d.get_criticality_level_object().name == 'green'

    workflow.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
    ]
    workflow.store()
    formdef = FormDef.get(id=formdef.id)  # reload formdef
    d = formdef.data_class()()
    d.criticality_level = 104
    # set too high, this simulates a workflow being changed to have less
    # levels than before.
    assert d.get_criticality_level_object().name == 'yellow'
    d.increase_criticality_level()
    assert d.get_criticality_level_object().name == 'yellow'
    d.decrease_criticality_level()
    assert d.get_criticality_level_object().name == 'green'

    d.criticality_level = 104
    d.decrease_criticality_level()
    assert d.get_criticality_level_object().name == 'green'

    assert d.get_static_substitution_variables().get('form_criticality_label') == 'green'
    assert d.get_substitution_variables().get('form_criticality_label') == 'green'


def test_field_item_substvars(pub):
    ds = {
        'type': 'formula',
        'value': repr([('1', 'un'), ('2', 'deux')]),
    }

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds, varname='xxx')]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': '1', '0_display': 'un'}

    variables = formdata.get_substitution_variables()
    assert variables.get('form_var_xxx') == 'un'
    assert variables.get('form_var_xxx_raw') == '1'


def test_get_json_export_dict_evolution(pub, local_user):
    Workflow.wipe()
    workflow = Workflow(name='test')
    st_new = workflow.add_status('New')
    st_finished = workflow.add_status('Finished')
    workflow.store()

    formdef = FormDef()
    formdef.workflow_id = workflow.id
    formdef.name = 'foo'
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    d = formdef.data_class()()
    d.status = 'wf-%s' % st_new.id
    d.user_id = local_user.id
    d.receipt_time = time.localtime()
    evo = Evolution()
    evo.time = time.localtime()
    evo.status = 'wf-%s' % st_new.id
    evo.who = '_submitter'
    d.evolution = [evo]
    d.store()
    evo.add_part(JournalEvolutionPart(d, "ok", None))
    evo.add_part(JournalWsCallErrorPart("summary", "label", "data"))
    evo = Evolution()
    evo.time = time.localtime()
    evo.status = 'wf-%s' % st_finished.id
    evo.who = '_submitter'
    d.evolution.append(evo)
    d.store()

    export = d.get_json_export_dict()
    assert 'evolution' in export
    assert len(export['evolution']) == 2
    assert export['evolution'][0]['status'] == st_new.id
    assert 'time' in export['evolution'][0]
    assert export['evolution'][0]['who']['id'] == local_user.id
    assert export['evolution'][0]['who']['email'] == local_user.email
    assert export['evolution'][0]['who']['NameID'] == local_user.name_identifiers
    assert 'parts' in export['evolution'][0]
    assert len(export['evolution'][0]['parts']) == 2
    assert export['evolution'][0]['parts'][0]['type'] == 'workflow-comment'
    assert export['evolution'][0]['parts'][0]['content'] == 'ok'
    assert export['evolution'][0]['parts'][1]['type'] == 'wscall-error'
    assert export['evolution'][0]['parts'][1]['summary'] == 'summary'
    assert export['evolution'][0]['parts'][1]['label'] == 'label'
    assert export['evolution'][0]['parts'][1]['data'] == 'data'
    assert export['evolution'][1]['status'] == st_finished.id
    assert 'time' in export['evolution'][1]
    assert export['evolution'][1]['who']['id'] == local_user.id
    assert export['evolution'][1]['who']['email'] == local_user.email
    assert export['evolution'][1]['who']['NameID'] == local_user.name_identifiers
    assert 'parts' not in export['evolution'][1]

    export = d.get_json_export_dict(anonymise=True)
    assert 'evolution' in export
    assert len(export['evolution']) == 2
    assert export['evolution'][0]['status'] == st_new.id
    assert 'time' in export['evolution'][0]
    assert 'who' not in export['evolution'][0]
    assert 'parts' in export['evolution'][0]
    assert len(export['evolution'][0]['parts']) == 2
    assert len(export['evolution'][0]['parts'][0]) == 2
    assert export['evolution'][0]['parts'][0]['type'] == 'workflow-comment'
    assert len(export['evolution'][0]['parts'][1]) == 1
    assert export['evolution'][0]['parts'][1]['type'] == 'wscall-error'
    assert export['evolution'][1]['status'] == st_finished.id
    assert 'time' in export['evolution'][1]
    assert 'who' not in export['evolution'][0]
    assert 'parts' not in export['evolution'][1]


def test_field_bool_substvars(pub):
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [fields.BoolField(id='0', label='checkbox', varname='xxx')]
    formdef.store()

    formdata = formdef.data_class()()

    formdata.data = {'0': False}
    variables = formdata.get_substitution_variables()
    assert variables.get('form_var_xxx') == 'False'
    assert variables.get('form_var_xxx_raw') is False

    formdata.data = {'0': True}
    variables = formdata.get_substitution_variables()
    assert variables.get('form_var_xxx') == 'True'
    assert variables.get('form_var_xxx_raw') is True


def test_backoffice_field_varname(pub, formdef):
    wf = Workflow(name='bo fields')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='1st backoffice field', type='string', varname='backoffice_blah'),
    ]
    wf.add_status('Status1')
    wf.store()

    formdef.workflow_id = wf.id
    formdef.data_class().wipe()
    formdef.fields = [fields.StringField(id='0', label='string', varname='foo')]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'bo1': 'test'}
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_var_backoffice_blah') == 'test'


def test_workflow_data_file_url(pub, formdef):
    upload = PicklableUpload('test.txt', 'text/plain', 'ascii')
    upload.receive([b'first line', b'second line'])

    formdata = formdef.data_class()()
    formdata.store()
    # create workflow_data as ordered dict to be sure _url comes last, to
    # trigger #17233.
    formdata.workflow_data = collections.OrderedDict(
        foo_var_file='test.txt',
        foo_var_file_raw=upload,
        foo_var_file_url=None,
    )
    substvars = formdata.get_substitution_variables()
    assert substvars['foo_var_file_url']


def test_workflow_data_invalid_keys(pub, formdef):
    formdata = formdef.data_class()()
    formdata.store()
    formdata.workflow_data = {
        'valid_key': {'invalid key': 'foo', 'valid_key': 'bar'},
        'invalid key': 'baz',
    }
    substvars = formdata.get_substitution_variables()
    assert 'form_workflow_data_valid_key' in substvars
    assert 'form_workflow_data_invalid key' not in substvars
    assert 'form_workflow_data_valid_key_valid_key' in substvars
    assert 'form_workflow_data_valid_key_invalid key' not in substvars
    assert substvars['form_workflow_data_valid_key_valid_key'] == 'bar'
    with pytest.raises(KeyError):
        # noqa pylint: disable=pointless-statement
        substvars['form_workflow_data_invalid key']


def test_evolution_get_status(pub):
    Workflow.wipe()
    workflow = Workflow(name='test')
    st_new = workflow.add_status('New')
    st_finished = workflow.add_status('Finished')
    workflow.store()

    formdef = FormDef()
    formdef.workflow_id = workflow.id
    formdef.name = 'foo'
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    d = formdef.data_class()()
    d.evolution = []

    evo = Evolution()
    evo.time = time.localtime()
    evo.status = 'wf-%s' % st_new.id
    d.evolution.append(evo)

    evo = Evolution()
    evo.time = time.localtime()
    d.evolution.append(evo)

    evo = Evolution()
    evo.time = time.localtime()
    d.evolution.append(evo)

    evo = Evolution()
    evo.time = time.localtime()
    evo.status = 'wf-%s' % st_finished.id
    d.evolution.append(evo)

    evo = Evolution()
    evo.time = time.localtime()
    d.evolution.append(evo)

    d.store()
    d = formdef.data_class().get(d.id)

    assert [x.get_status().id for x in d.evolution] == ['1', '1', '1', '2', '2']


@pytest.fixture
def variable_test_data(pub):
    pub.user_class.wipe()
    user = pub.user_class()
    user.email = 'bar@localhost'
    user.name_identifiers = ['....']
    user.store()

    role = pub.role_class(name='foobar')
    role.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobarlazy'
    formdef.fields = [
        fields.StringField(id='0', label='string', varname='foo_foo'),
        fields.BoolField(id='1', label='checkbox', varname='boolfield', type='bool'),
        fields.BoolField(id='2', label='checkbox', varname='boolfield2', type='bool'),
        fields.BoolField(id='2b', label='checkbox', varname='boolfield3', type='bool'),
        fields.DateField(id='3', label='date', varname='datefield'),
        fields.ItemsField(id='4', label='items', items=['aa', 'ab', 'ac'], varname='itemsfield'),
        fields.FileField(id='5', label='file', varname='filefield'),
        fields.StringField(id='6', label='string2', varname='foo_foo_baz_baz'),
        fields.MapField(id='7', label='map', varname='map'),
        fields.DateField(id='8', label='date2', varname='datefield2'),
        fields.StringField(id='9', label='string2', varname='datestring'),
        fields.StringField(id='10', label='number1', varname='term1'),
        fields.StringField(id='11', label='number2', varname='term2'),
        fields.StringField(id='12', label='float1', varname='value'),
        fields.PasswordField(id='13', label='pwd', varname='pwd'),
        fields.EmailField(id='14', label='email', varname='email'),
    ]
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.geolocations = {'base': 'Base'}
    formdef.store()
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.user_id = user.id
    formdata.data = {
        '0': 'bar',
        '1': False,
        '2': True,
        '3': time.strptime('2018-07-31', '%Y-%m-%d'),
        '4': ['aa', 'ac'],
        '4_display': 'aa, ac',
        '5': PicklableUpload('test.txt', 'text/plain'),
        '6': 'other',
        '7': '2;4',  # map
        '8': time.strptime('2018-08-31', '%Y-%m-%d'),
        '9': '2018-07-31',
        '10': '3',
        '11': '4',
        '12': '3.14',
        '13': {
            'cleartext': 'a',
            'md5': '0cc175b9c0f1b6a831c399e269772661',
            'sha1': '86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
        },
        '14': 'test@localhost',
    }
    formdata.data['5'].receive([b'hello world'])
    formdata.geolocations = {'base': {'lat': 1, 'lon': 2}}
    formdata.store()
    pub.substitutions.feed(pub)
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)
    return LazyFormData(formdata)


def test_lazy_formdata(pub, variable_test_data):
    formdata = FormDef.select()[0].data_class().select()[0]
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.receipt_date == time.strftime('%Y-%m-%d', formdata.receipt_time)
    assert lazy_formdata.receipt_time == time.strftime('%H:%M', formdata.receipt_time)
    assert lazy_formdata.last_update_datetime.timetuple()[:6] == formdata.last_update_time[:6]
    assert lazy_formdata.internal_id == formdata.id
    assert lazy_formdata.name == 'foobarlazy'
    assert lazy_formdata.display_name == 'foobarlazy #%s' % formdata.get_display_id()
    assert lazy_formdata.page_no == 0
    assert lazy_formdata.url.endswith('/foobarlazy/%s/' % formdata.id)
    assert lazy_formdata.url_backoffice.endswith('/backoffice/management/foobarlazy/%s/' % formdata.id)
    assert lazy_formdata.backoffice_url == lazy_formdata.url_backoffice
    assert lazy_formdata.api_url == formdata.get_api_url()
    assert lazy_formdata.attachments
    assert lazy_formdata.geoloc['base'] == {'lat': 1, 'lon': 2}
    assert lazy_formdata.geoloc['base_lon'] == 2
    static_vars = formdata.get_static_substitution_variables()
    for attribute in (
        'name',
        'receipt_date',
        'receipt_time',
        'previous_status',
        'uri',
        'status_changed',
        'comment',
        'evolution',
        'details',
        'criticality_level',
        'digest',
    ):
        assert getattr(lazy_formdata, attribute) == static_vars['form_' + attribute]

    assert lazy_formdata.user.email == 'bar@localhost'
    assert lazy_formdata.var.foo_foo == 'bar'
    assert lazy_formdata.var.boolfield == 'False'
    assert bool(lazy_formdata.var.boolfield) is False
    assert lazy_formdata.var.boolfield.raw is False
    assert lazy_formdata.var.boolfield2 == 'True'
    assert lazy_formdata.var.boolfield2.raw is True
    assert bool(lazy_formdata.var.boolfield2) is True
    assert lazy_formdata.var.boolfield3.raw is False
    assert lazy_formdata.var.datefield.raw == time.strptime('2018-07-31', '%Y-%m-%d')
    assert lazy_formdata.var.datefield.tm_year == 2018
    assert lazy_formdata.var.datefield.tm_mon == 7
    assert lazy_formdata.var.datefield.tm_mday == 31
    for attr in ('tm_year', 'tm_mon', 'tm_mday', 'tm_hour', 'tm_min', 'tm_sec', 'tm_wday', 'tm_yday'):
        getattr(lazy_formdata.var.datefield, attr)

    # flexible date comparison
    assert lazy_formdata.var.datefield == lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield == lazy_formdata.var.datefield2
    assert not lazy_formdata.var.datefield2 == lazy_formdata.var.datefield
    assert lazy_formdata.var.datefield != lazy_formdata.var.datefield2
    assert lazy_formdata.var.datefield2 != lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield != lazy_formdata.var.datefield
    assert lazy_formdata.var.datefield < lazy_formdata.var.datefield2
    assert lazy_formdata.var.datefield <= lazy_formdata.var.datefield2
    assert not lazy_formdata.var.datefield > lazy_formdata.var.datefield2
    assert not lazy_formdata.var.datefield >= lazy_formdata.var.datefield2
    assert lazy_formdata.var.datefield2 > lazy_formdata.var.datefield
    assert lazy_formdata.var.datefield2 >= lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield2 < lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield2 <= lazy_formdata.var.datefield

    assert lazy_formdata.var.datefield == lazy_formdata.var.datestring
    assert lazy_formdata.var.datestring == lazy_formdata.var.datefield
    assert lazy_formdata.var.datefield >= lazy_formdata.var.datestring
    assert lazy_formdata.var.datestring >= lazy_formdata.var.datefield
    assert lazy_formdata.var.datefield <= lazy_formdata.var.datestring
    assert lazy_formdata.var.datestring <= lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield != lazy_formdata.var.datestring
    assert not lazy_formdata.var.datestring != lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield < lazy_formdata.var.datestring
    assert not lazy_formdata.var.datestring < lazy_formdata.var.datefield
    assert not lazy_formdata.var.datefield > lazy_formdata.var.datestring
    assert not lazy_formdata.var.datestring > lazy_formdata.var.datefield

    for date in (
        '2018-07-31',
        '2018-07-31 00:00',
        '2018-07-31 00:00:00',
        '31/07/2018',
        '31/07/2018 00h00',
        '31/07/2018 00:00:00',
        datetime.date(2018, 7, 31),
        datetime.datetime(2018, 7, 31, 0, 0),
        time.strptime('2018-07-31', '%Y-%m-%d'),
    ):
        assert lazy_formdata.var.datefield == date
        assert lazy_formdata.var.datefield >= date
        assert lazy_formdata.var.datefield <= date
        assert date == lazy_formdata.var.datefield
        assert date <= lazy_formdata.var.datefield
        assert date >= lazy_formdata.var.datefield
        assert not lazy_formdata.var.datefield != date
        assert not date != lazy_formdata.var.datefield
        assert not lazy_formdata.var.datefield > date
        assert not lazy_formdata.var.datefield < date
        assert not date < lazy_formdata.var.datefield
        assert not date > lazy_formdata.var.datefield

    for date in (
        '2018-08-31',
        '2018-07-31 01:00',
        '2018-07-31 01:00:00',
        '31/08/2018',
        '31/07/2018 01h00',
        '31/07/2018 01:00:00',
        datetime.date(2018, 8, 31),
        datetime.datetime(2018, 8, 31, 0, 0),
        time.strptime('2018-08-31', '%Y-%m-%d'),
    ):
        assert lazy_formdata.var.datefield != date
        assert date != lazy_formdata.var.datefield
        assert not lazy_formdata.var.datefield == date
        assert not date == lazy_formdata.var.datefield
        assert lazy_formdata.var.datefield < date
        assert not lazy_formdata.var.datefield >= date
        assert lazy_formdata.var.datefield <= date
        assert not lazy_formdata.var.datefield > date
        assert date > lazy_formdata.var.datefield
        assert not date <= lazy_formdata.var.datefield
        assert date >= lazy_formdata.var.datefield
        assert not date < lazy_formdata.var.datefield

    assert lazy_formdata.var.itemsfield == 'aa, ac'
    assert 'aa' in lazy_formdata.var.itemsfield  # taken as a list
    assert 'aa,' not in lazy_formdata.var.itemsfield  # not as a string
    assert lazy_formdata.var.filefield == 'test.txt'
    assert lazy_formdata.var.filefield.raw.base_filename == 'test.txt'
    assert lazy_formdata.var.filefield.raw.content_type == 'text/plain'

    formdata = FormDef.select()[0].data_class()
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.tracking_code is None
    formdata.data = {'future_tracking_code': 'CDCBGWQX'}
    assert lazy_formdata.tracking_code == 'CDCBGWQX'

    formdata = FormDef.select()[0].data_class().select()[0]
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.tracking_code is None

    tracking_code = pub.tracking_code_class()
    tracking_code.formdata = formdata
    tracking_code.store()
    formdata = FormDef.select()[0].data_class().get(formdata.id)
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.tracking_code == tracking_code.id


def test_lazy_formdata_duplicated_varname(pub, variable_test_data):
    formdef = FormDef.select()[0]
    formdata = FormDef.select()[0].data_class().select()[0]
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.var.foo_foo == 'bar'

    formdef.fields.append(fields.StringField(id='100', label='string', varname='foo_foo'))
    formdef.store()
    formdata = FormDef.select()[0].data_class().select()[0]
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.var.foo_foo == 'bar'

    # add a value to 2nd field with foo_foo as varname
    formdata.data['100'] = 'baz'
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.var.foo_foo == 'bar'  # 1st value

    # remove value from 1st field with foo_foo as varname
    formdata.data['0'] = None
    lazy_formdata = LazyFormData(formdata)
    assert lazy_formdata.var.foo_foo == 'baz'  # 2nd value


def test_lazy_formdata_workflow_data(pub):
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.workflow_data = {
        '_markers_stack': ['1', '2', '3'],
        'foo_bar': 'plop',
        'other': {'test': 'foobar'},
    }
    pub.substitutions.feed(pub)
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)

    context = pub.substitutions.get_context_variables(mode='lazy')
    assert 'form_workflow_data_foo_bar' in context.get_flat_keys()
    assert context['form_workflow_data_foo_bar'] == 'plop'
    assert 'form_workflow_data_other_test' in context.get_flat_keys()
    assert context['form_workflow_data_other_test'] == 'foobar'

    assert 'form_workflow_data__markers_stack' not in context.get_flat_keys()
    with pytest.raises(KeyError):
        # noqa pylint: disable=pointless-statement
        context['form_workflow_data__markers_stack']


def test_lazy_formdata_live_item(pub):
    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        fields.StringField(id='0', label='string', varname='name'),
        fields.StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()

    ds = {'type': 'carddef:%s' % carddef.url_name}
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemField(
            id='0', label='string', type='item', varname='foo', data_source=ds, display_disabled_items=True
        )
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {
        '0': str(carddata.id),
    }
    formdata.data['0_display'] = formdef.fields[0].store_display_value(formdata.data, '0')
    formdata.data['0_structured'] = formdef.fields[0].store_structured_value(formdata.data, '0')
    pub.substitutions.feed(pub)
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)

    context = pub.substitutions.get_context_variables(mode='lazy')
    assert context['form_var_foo_live_name'] == 'items'
    assert context['form_var_foo_live_number'] == carddata.get_display_id()
    assert context['form_var_foo_live_var_name'] == 'baz'
    assert context['form_var_foo_live_var_attr'] == 'attr2'
    assert 'form_var_foo_live_var_attr' in context.get_flat_keys()

    # check it also works with custom views
    ds = {'type': 'carddef:%s:xxx' % carddef.url_name}
    formdef.fields = [
        fields.ItemField(
            id='0', label='string', type='item', varname='foo', data_source=ds, display_disabled_items=True
        )
    ]
    formdef.store()
    pub.substitutions.reset()
    pub.substitutions.feed(pub)
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)
    pub.get_request().live_card_cache = {}
    assert 'form_var_foo_live_var_attr' in context.get_flat_keys()
    assert context['form_var_foo_live_var_name'] == 'baz'

    # mock missing carddef to get call count
    pub.get_request().live_card_cache = {}
    context = pub.substitutions.get_context_variables(mode='lazy')
    with mock.patch('wcs.carddef.CardDef.get_by_urlname') as get_by_urlname:
        get_by_urlname.side_effect = KeyError

        with pytest.raises(KeyError):
            # noqa pylint: disable=pointless-statement
            context['form_var_foo_live_name']
        assert get_by_urlname.call_count == 1
        with pytest.raises(KeyError):  # repeated access, will go through cache
            # noqa pylint: disable=pointless-statement
            context['form_var_foo_live_name']
        assert get_by_urlname.call_count == 1
        assert 'form_var_foo_live_var_attr' not in context.get_flat_keys()


def test_lazy_formdata_live_user_item(pub, local_user):
    NamedDataSource.wipe()
    datasource = NamedDataSource(name='foo')
    datasource.data_source = {'type': 'wcs:users'}
    datasource.store()

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemField(
            id='0',
            label='string',
            type='item',
            varname='foo',
            data_source={'type': 'foo'},
        )
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {
        '0': str(local_user.id),
    }
    formdata.data['0_display'] = formdef.fields[0].store_display_value(formdata.data, '0')
    formdata.data['0_structured'] = formdef.fields[0].store_structured_value(formdata.data, '0')
    pub.substitutions.feed(pub)
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)

    context = pub.substitutions.get_context_variables(mode='lazy')
    assert context['form_var_foo_live_name'] == local_user.name
    assert context['form_var_foo_live_email'] == local_user.email
    assert 'form_var_foo_live_email' in context.get_flat_keys()


def test_lazy_formdata_queryset(pub, variable_test_data):
    lazy_formdata = variable_test_data
    data_class = lazy_formdata._formdef.data_class()
    for _ in range(6):
        formdata = data_class()
        formdata.just_created()
        formdata.store()
    for _ in range(4):
        formdata = data_class()
        formdata.just_created()
        formdata.jump_status('finished')
        formdata.store()

    formdata = data_class()
    formdata.status = 'draft'
    formdata.store()

    assert lazy_formdata.objects.count == 11
    assert lazy_formdata.objects.drafts().count == 1
    assert lazy_formdata.objects.pending().count == 7
    assert lazy_formdata.objects.done().count == 4
    assert lazy_formdata.objects.drafts().count == 1
    # check __len__
    assert len(lazy_formdata.objects.drafts()) == 1

    assert lazy_formdata.objects.current_user().count == 0
    pub.get_request()._user = ()  # reset cache
    pub.get_request().session = sessions.BasicSession(id=1)
    pub.get_request().session.set_user(pub.user_class.select()[0].id)
    assert lazy_formdata.objects.current_user().count == 1

    qs = lazy_formdata.objects.drafts()
    # check __iter__
    for draft in qs:
        assert draft.internal_id == formdata.id
    # check __getitem__
    assert qs[0].internal_id == formdata.id
    # check against for cached resultset
    assert qs[0].internal_id == formdata.id
    # check __iter__ with cached resultset
    for draft in qs:
        assert draft.internal_id == formdata.id

    # check __iter__ creates a cached resultset
    qs = lazy_formdata.objects.drafts()
    for formdata1, formdata2 in zip(list(qs), list(qs)):
        assert formdata1 is formdata2

    # check ordering
    qs = lazy_formdata.objects.pending().order_by('id')
    assert [x.number for x in qs] == ['1-1', '1-2', '1-3', '1-4', '1-5', '1-6', '1-7']

    # Check accessing an non-numeric attribute doesn't try to cache things
    # (see code for explanation)
    manager = lazy_formdata.objects
    with pytest.raises(TypeError):
        # noqa pylint: disable=pointless-statement
        manager['drafts']
    assert manager._cached_resultset is None


def test_lazy_formdata_queryset_distance(pub, variable_test_data):
    # Form
    lazy_formdata = variable_test_data
    formdef = lazy_formdata._formdef
    formdef.geolocations = {'base': 'Base'}
    formdef.store()
    data_class = lazy_formdata._formdef.data_class()
    # Card
    CardDef.wipe()
    carddef = CardDef()
    carddef.name = 'items'
    carddef.fields = []
    carddef.geolocations = {'base': 'Base'}
    carddef.store()
    carddata_class = carddef.data_class()
    carddata_class.wipe()

    # create initial carddata like lazy_formdata, with same geolocations
    carddata = carddata_class()
    carddata.geolocations = {'base': {'lat': 1, 'lon': 2}}
    carddata.just_created()
    carddata.store()
    lazy_carddata = LazyFormData(carddata)

    # create objects
    for i in range(6):
        for dclass in [data_class, carddata_class]:
            data = dclass()
            data.geolocations = {'base': {'lat': i, 'lon': i}}
            data.just_created()
            data.store()
    for i in range(4):
        for dclass in [data_class, carddata_class]:
            data = dclass()
            data.geolocations = {'base': {'lat': i + 0.5, 'lon': i + 0.5}}
            data.just_created()
            data.jump_status('finished')
            data.store()

    # drafts
    formdata = data_class()
    formdata.status = 'draft'
    formdata.geolocations = {'base': {'lat': 1, 'lon': 2}}
    formdata.store()
    carddata = carddata_class()
    carddata.status = 'draft'
    carddata.geolocations = {'base': {'lat': 1, 'lon': 2}}
    carddata.store()

    # compute distance against map field of lazy formdata
    nearby = lazy_formdata.objects.distance_filter(200000)
    assert len(nearby) == 6
    assert {x.number for x in nearby} == {'1-1', '1-3', '1-4', '1-8', '1-9', '1-10'}
    nearby = lazy_carddata.objects.set_geo_center(lazy_formdata).distance_filter(200000)
    assert len(nearby) == 6
    assert {x.number for x in nearby} == {'1-1', '1-3', '1-4', '1-8', '1-9', '1-10'}

    # compute distance against geolocation
    lazy_formdata._formdata.geolocations = {'base': {'lat': 2, 'lon': 2.5}}
    nearby = lazy_formdata.objects.distance_filter(200000)
    assert len(nearby) == 6
    assert {x.number for x in nearby} == {'1-1', '1-4', '1-5', '1-9', '1-10', '1-11'}
    assert bool(nearby) is True
    nearby = lazy_carddata.objects.set_geo_center(lazy_formdata).distance_filter(200000)
    assert len(nearby) == 6
    assert {x.number for x in nearby} == {'1-1', '1-4', '1-5', '1-9', '1-10', '1-11'}
    assert bool(nearby) is True

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|distance_filter:200000|count}}')
    assert tmpl.render(context) == '6'
    tmpl = Template('{{cards|objects:"items"|set_geo_center:form|distance_filter:200000|count}}')
    assert tmpl.render(context) == '6'

    lazy_formdata._formdata.geolocations = {'base': {'lat': 7, 'lon': 7.5}}
    nearby = lazy_formdata.objects.distance_filter(200000)
    assert bool(nearby) is False
    assert len(nearby) == 0
    nearby = lazy_carddata.objects.set_geo_center(lazy_formdata).distance_filter(200000)
    assert bool(nearby) is False
    assert len(nearby) == 0

    tmpl = Template('{{form_objects|distance_filter:200000|count}}')
    assert tmpl.render(context) == '0'
    tmpl = Template('{{cards|objects:"items"|set_geo_center:form|distance_filter:200000|count}}')
    assert tmpl.render(context) == '0'


def test_lazy_formdata_queryset_filter(pub, variable_test_data):
    local_user_id = variable_test_data.user.id

    wf = Workflow.get_default_workflow()
    wf.id = None
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='1st backoffice field', type='string', varname='backoffice_blah'),
    ]
    wf.store()

    lazy_formdata = variable_test_data
    formdef = lazy_formdata._formdef
    formdef.workflow = wf
    formdef.store()
    data_class = lazy_formdata._formdef.data_class()
    for i in range(6):
        formdata = data_class()
        formdata.data = {'0': 'bar', '1': True, 'bo1': 'plop1', '10': '3'}
        if i == 5:
            formdata.data['3'] = datetime.date(2018, 8, 31).timetuple()
        formdata.just_created()
        formdata.store()
    for _ in range(4):
        formdata = data_class()
        formdata.data = {
            '0': 'foo',
            '1': False,
            '3': datetime.date(2018, 7, 31).timetuple(),
            'bo1': 'plop2',
            '10': '4',
        }
        formdata.just_created()
        formdata.jump_status('finished')
        formdata.store()

    finished_formdata = formdata

    formdata = data_class()
    formdata.data = {'0': 'bar', 'bo1': 'plop1'}
    formdata.status = 'draft'
    formdata.store()

    formdata = data_class()
    formdata.just_created()
    formdata.data = {'0': 'bar'}
    formdata.anonymise()

    # filter function
    queryset = lazy_formdata.objects.filter_by('foo_foo').apply_filter_value('bar')
    assert queryset.count == 7
    queryset = lazy_formdata.objects.filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 4
    queryset = lazy_formdata.objects.filter_by('foo_foo').apply_filter_value('X')
    assert queryset.count == 0
    pub.loggederror_class.wipe()
    queryset = lazy_formdata.objects.filter_by('unknown').apply_filter_value('X')
    assert queryset.count == 0
    assert pub.loggederror_class.count() == 1
    logged_error = pub.loggederror_class.select()[0]
    assert logged_error.summary == 'Invalid filter "unknown"'

    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value(
        datetime.date(2018, 7, 31).timetuple()
    )
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value(datetime.date(2018, 7, 31))
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value(datetime.datetime(2018, 7, 31))
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value('2018-07-31')
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value(None)
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('datefield').apply_filter_value('not a date')
    assert queryset.count == 0
    assert pub.loggederror_class.count() == 2
    logged_error = pub.loggederror_class.select(order_by='id')[1]
    assert logged_error.summary == 'Invalid value "not a date" for filter "datefield"'

    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(
        datetime.date(2018, 7, 31).timetuple()
    )
    assert queryset.count == 6  # 1 + 5 null
    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(datetime.date(2018, 7, 31))
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(
        datetime.datetime(2018, 7, 31)
    )
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value('2018-07-31')
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value(None)
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('datefield').apply_exclude_value('still not a date')
    assert queryset.count == 0
    assert pub.loggederror_class.count() == 3
    logged_error = pub.loggederror_class.select(order_by='id')[2]
    assert logged_error.summary == 'Invalid value "still not a date" for filter "datefield"'

    queryset = lazy_formdata.objects.filter_by('boolfield').apply_filter_value(True)
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('boolfield').apply_filter_value(1)
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('boolfield').apply_filter_value('yes')
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('boolfield').apply_filter_value(False)
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('boolfield').apply_filter_value(0)
    assert queryset.count == 5
    queryset = lazy_formdata.objects.filter_by('boolfield').apply_exclude_value(0)
    assert queryset.count == 6

    queryset = lazy_formdata.objects.filter_by('term1').apply_filter_value('3')
    assert queryset.count == 7
    queryset = lazy_formdata.objects.filter_by('term1').apply_filter_value(3)
    assert queryset.count == 7
    queryset = lazy_formdata.objects.filter_by('term1').apply_filter_value('foobar')
    assert queryset.count == 0
    queryset = lazy_formdata.objects.filter_by('term1').apply_exclude_value('3')
    assert queryset.count == 4
    queryset = lazy_formdata.objects.filter_by('term1').apply_exclude_value('foobar')
    assert queryset.count == 11

    queryset = lazy_formdata.objects.filter_by('email').apply_filter_value('bar')
    assert queryset.count == 0
    queryset = lazy_formdata.objects.filter_by('email').apply_filter_value('test@localhost')
    assert queryset.count == 1

    # filter function on backoffice field
    queryset = lazy_formdata.objects.filter_by('backoffice_blah').apply_filter_value('plop1')
    assert queryset.count == 6
    queryset = lazy_formdata.objects.filter_by('backoffice_blah').apply_filter_value('plop2')
    assert queryset.count == 4
    queryset = lazy_formdata.objects.filter_by('backoffice_blah').apply_filter_value('X')
    assert queryset.count == 0

    # filter using attribute name
    queryset = lazy_formdata.objects.filter_by_foo_foo().apply_filter_value('bar')
    assert queryset.count == 7

    # filter + exclude current formdata
    queryset = lazy_formdata.objects.exclude_self().filter_by('foo_foo').apply_filter_value('bar')
    assert queryset.count == 6

    queryset = lazy_formdata.objects.exclude_self().filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 4

    # filter + limit to same user
    queryset = lazy_formdata.objects.exclude_self().same_user().filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 0

    for lazy in lazy_formdata.objects.filter_by('foo_foo').apply_filter_value('foo')[:2]:
        lazy._formdata.user_id = variable_test_data._formdata.user_id
        lazy._formdata.store()

    queryset = lazy_formdata.objects.exclude_self().same_user().filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 2

    # + exclude self (lazy being set from the for loop)
    queryset = lazy.objects.exclude_self().same_user().filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 1

    # and with anonymous user
    lazy._formdata.user_id = None
    lazy._formdata.store()
    queryset = lazy.objects.same_user().filter_by('foo_foo').apply_filter_value('foo')
    assert queryset.count == 4

    # filter with anonymised
    queryset = lazy_formdata.objects.with_anonymised().filter_by('foo_foo').apply_filter_value('bar')
    assert queryset.count == 8

    # template tags
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"bar"|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"bar"|length}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|count}}')
    assert tmpl.render(context) == '4'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|length}}')
    assert tmpl.render(context) == '4'

    pub.substitutions.feed(formdata)
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:form_var_foo_foo|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|filter_by:"datefield"|filter_value:form_var_datefield|count}}')
    assert tmpl.render(context) == '5'
    tmpl = Template('{{form_objects|filter_by:"boolfield"|filter_value:form_var_boolfield|count}}')
    assert tmpl.render(context) == '5'
    tmpl = Template('{{form_objects|filter_by:"term1"|filter_value:form_var_term1|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:form_var_foo_foo|count}}')
    assert tmpl.render(context) == '4'
    tmpl = Template('{{form_objects|filter_by:"datefield"|exclude_value:form_var_datefield|count}}')
    assert tmpl.render(context) == '6'
    tmpl = Template('{{form_objects|filter_by:"boolfield"|exclude_value:form_var_boolfield|count}}')
    assert tmpl.render(context) == '6'
    tmpl = Template('{{form_objects|filter_by:"term1"|exclude_value:form_var_term1|count}}')
    assert tmpl.render(context) == '4'

    tmpl = Template('{{form.objects|exclude_self|filter_by:"foo_foo"|filter_value:form_var_foo_foo|count}}')
    assert tmpl.render(context) == '6'

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"bar"|count}}')
    assert tmpl.render(context) == '7'

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|same_user|filter_by:"foo_foo"|filter_value:"foo"|count}}')
    assert tmpl.render(context) == '4'

    # attach a user to context form to get form_user variable
    formdata.user_id = local_user_id
    formdata.store()

    # test |filter_by_user
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by_user:form_user|filter_by:"foo_foo"|filter_value:"foo"|count}}')
    assert tmpl.render(context) == '1'
    assert (
        LazyFormData(formdata)
        .objects.filter_by_user(lazy_formdata.user)
        .filter_by('foo_foo')
        .apply_filter_value('foo')
        .count
        == 1
    )
    tmpl = Template(
        '{{form_objects|filter_by_user:form_user_email|filter_by:"foo_foo"|filter_value:"foo"|count}}'
    )
    assert tmpl.render(context) == '1'
    tmpl = Template(
        '{{form_objects|filter_by_user:form_user_nameid|filter_by:"foo_foo"|filter_value:"foo"|count}}'
    )
    assert tmpl.render(context) == '1'
    tmpl = Template('{{form_objects|filter_by_user:"foo@bar"|filter_by:"foo_foo"|filter_value:"foo"|count}}')
    assert tmpl.render(context) == '0'

    # test |current_user
    pub.get_request()._user = ()  # reset cache
    pub.get_request().session = sessions.BasicSession(id=1)
    pub.get_request().session.set_user(local_user_id)
    tmpl = Template('{{form_objects|current_user|filter_by:"foo_foo"|filter_value:"foo"|count}}')
    assert tmpl.render(context) == '1'

    # test |filter_by_status
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by_status:"Just Submitted"|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template(
        '{{form_objects|filter_by_status:"Just Submitted"|filter_by:"foo_foo"|filter_value:"foo"|count}}'
    )
    assert tmpl.render(context) == '0'
    assert LazyFormData(formdata).objects.filter_by_status('Just Submitted').count == 7
    assert (
        LazyFormData(formdata)
        .objects.filter_by_status('Just Submitted')
        .filter_by('foo_foo')
        .apply_filter_value('foo')
        .count
        == 0
    )
    assert LazyFormData(formdata).objects.filter_by_status('Finished').count == 4
    assert LazyFormData(formdata).objects.filter_by_status('Unknown').count == 0

    # test |pending
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|pending|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|pending|filter_by:"foo_foo"|filter_value:"foo"|count}}')
    assert tmpl.render(context) == '0'
    assert LazyFormData(formdata).objects.pending().count == 7
    assert LazyFormData(formdata).objects.pending().filter_by('foo_foo').apply_filter_value('foo').count == 0

    # test |done
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|done|count}}')
    assert tmpl.render(context) == '4'

    # test |filter_by_internal_id
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by_internal_id:"%s"|count}}' % finished_formdata.id)
    assert tmpl.render(context) == '1'
    tmpl = Template('{{form_objects|filter_by_internal_id:"%s"|count}}' % '0')
    assert tmpl.render(context) == '0'
    tmpl = Template('{{form_objects|filter_by_internal_id:"%s"|count}}' % 'invalid value')
    assert tmpl.render(context) == '0'
    assert pub.loggederror_class.count() == 4
    logged_error = pub.loggederror_class.select(order_by='id')[3]
    assert logged_error.summary == 'Invalid value "invalid value" for "filter_by_internal_id"'

    # test |filter_by_number
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by_number:"%s"|count}}' % finished_formdata.get_display_id())
    assert tmpl.render(context) == '1'
    tmpl = Template('{{form_objects|filter_by_number:"%s"|count}}' % 'invalid value')
    assert tmpl.render(context) == '0'

    # test |is_empty
    tmpl = Template('{{form_objects|pending|is_empty}}')
    assert tmpl.render(context) == 'False'
    tmpl = Template('{{form_objects|pending|filter_by:"foo_foo"|filter_value:"foo"|is_empty}}')
    assert tmpl.render(context) == 'True'

    # test |getlist
    tmpl = Template('{% for v in form_objects|order_by:"id"|getlist:"foo_foo" %}{{ v }},{% endfor %}')
    assert tmpl.render(context) == 'bar,bar,bar,bar,bar,bar,bar,foo,foo,foo,foo,'
    tmpl = Template('{% for v in form_objects|order_by:"id"|getlist:"datefield" %}{{ v|date }},{% endfor %}')
    assert tmpl.render(context) == '2018-07-31,,,,,,2018-08-31,2018-07-31,2018-07-31,2018-07-31,2018-07-31,'
    tmpl = Template('{% if "foo" in form_objects|getlist:"foo_foo" %}OK{% else %}KO{% endif%}')
    assert tmpl.render(context) == 'OK'
    tmpl = Template('{% if "fooooooooooooooo" in form_objects|getlist:"foo_foo" %}OK{% else %}KO{% endif%}')
    assert tmpl.render(context) == 'KO'
    tmpl = Template('{{ form_objects|order_by:"id"|getlist:"foo_foo"|count }}')
    assert tmpl.render(context) == '11'
    assert LazyFormData(formdata).objects.order_by('id').getlist('foo_foo') == [
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'foo',
        'foo',
        'foo',
        'foo',
    ]
    assert LazyFormData(formdata).objects.order_by('id').getlist('form_var_foo_foo') == [
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'bar',
        'foo',
        'foo',
        'foo',
        'foo',
    ]
    assert set(LazyFormData(formdata).objects.getlist('unknown')) == {None}
    tmpl = Template('{{form_objects|pending|getlist:"foo_foo"|is_empty}}')
    assert tmpl.render(context) == 'False'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"foo"|getlist:"foo_foo"|is_empty}}')
    assert tmpl.render(context) == 'False'
    tmpl = Template(
        '{{form_objects|pending|filter_by:"foo_foo"|filter_value:"foo"|getlist:"foo_foo"|is_empty}}'
    )
    assert tmpl.render(context) == 'True'

    # test with cache populated
    for value in [
        datetime.date(2018, 7, 31).timetuple(),
        datetime.date(2018, 7, 31),
        datetime.datetime(2018, 7, 31),
        '2018-07-31',
    ]:
        assert value in LazyFormData(formdata).objects.getlist('datefield')
        tmpl = Template(
            '''{% with form_objects|getlist:"datefield" as objects %}
        {% for v in objects %}{% endfor %}{% if value in objects %}OK{% else %}KO{% endif%}
        {% endwith %}'''
        )
        context['value'] = value
        assert tmpl.render(context) == '\n        OK\n        '
    assert 'not a date' not in LazyFormData(formdata).objects.getlist('datefield')

    # test |getlistdict
    tmpl = Template('{{ form_objects|order_by:"id"|getlistdict:"foo_foo" }}', autoescape=False)
    assert tmpl.render(context) == str(
        [
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'bar'},
            {'foo_foo': 'foo'},
            {'foo_foo': 'foo'},
            {'foo_foo': 'foo'},
            {'foo_foo': 'foo'},
        ]
    )
    tmpl = Template(
        '{{ form_objects|order_by:"id"|getlistdict:"foo_foo:test, boolfield, unknown" }}', autoescape=False
    )
    assert tmpl.render(context) == str(
        [
            {'test': 'bar', 'boolfield': False, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'bar', 'boolfield': True, 'unknown': None},
            {'test': 'foo', 'boolfield': False, 'unknown': None},
            {'test': 'foo', 'boolfield': False, 'unknown': None},
            {'test': 'foo', 'boolfield': False, 'unknown': None},
            {'test': 'foo', 'boolfield': False, 'unknown': None},
        ]
    )
    tmpl = Template(
        '{{ form_objects|order_by:"id"|getlistdict:"datefield"|first|get:"datefield"|date }}',
        autoescape=False,
    )
    assert tmpl.render(context) == '2018-07-31'


def test_lazy_formdata_queryset_filter_non_unique_varname(pub, variable_test_data):
    lazy_formdata = variable_test_data
    formdef = lazy_formdata._formdef
    # modify fields to have foo_foo as varname for both fields[0] and fields[7]
    assert formdef.fields[7].label == 'string2'
    formdef.fields[7].varname = 'foo_foo'
    formdef.store()

    data_class = lazy_formdata._formdef.data_class()
    for i in range(6):
        formdata = data_class()
        formdata.data = {'0': 'bar', '6': 'baz'}
        if i == 5:
            formdata.data['3'] = datetime.date(2018, 8, 31).timetuple()
        formdata.just_created()
        formdata.store()
    formdatas = []
    for _ in range(4):
        formdata = data_class()
        formdata.data = {
            '0': 'foo',
        }
        formdata.just_created()
        formdata.jump_status('finished')
        formdata.store()
        formdatas.append(formdata)

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"bar"|count}}')
    assert tmpl.render(context) == '7'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"other"|count}}')
    assert tmpl.render(context) == '1'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|filter_value:"baz"|count}}')
    assert tmpl.render(context) == '6'
    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|count}}')
    assert tmpl.render(context) == '4'  # 11 - 7

    for formdata in formdatas[:-1]:
        formdata.data['6'] = 'bar'
        formdata.store()

    tmpl = Template('{{form_objects|filter_by:"foo_foo"|exclude_value:"bar"|count}}')
    assert tmpl.render(context) == '1'


def test_lazy_formdata_queryset_get_from_first(pub, variable_test_data):
    context = pub.substitutions.get_context_variables(mode='lazy')
    del context['form']  # remove actual form from context
    # direct attribute access
    tmpl = Template('{{forms|objects:"foobarlazy"|first|get:"foo_foo"}}')
    assert tmpl.render(context) == 'bar'
    # indirect, with full names
    tmpl = Template('{{forms|objects:"foobarlazy"|first|get:"form_var_foo_foo"}}')
    assert tmpl.render(context) == 'bar'


def test_lazy_formdata_queryset_order_by(pub, variable_test_data):
    lazy_formdata = variable_test_data
    data_class = lazy_formdata._formdef.data_class()
    for i in range(6):
        formdata = data_class()
        formdata.data = {
            '0': 'foo%s' % i,
            '1': True,
            'bo1': 'plop1',
            '10': '3',
            '3': datetime.date(2019, 7, 2 + i).timetuple(),
        }
        formdata.just_created()
        formdata.store()
    for i in range(4):
        formdata = data_class()
        formdata.data = {
            '0': 'bar%s' % i,
            '1': False,
            '3': datetime.date(2020, 7, 30 - i).timetuple(),
            'bo1': 'plop2',
            '10': '4',
        }
        formdata.just_created()
        formdata.jump_status('finished')
        formdata.store()

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{% for v in form_objects|order_by:"foo_foo"|getlist:"foo_foo" %}{{ v }},{% endfor %}')
    assert tmpl.render(context) == 'bar,bar0,bar1,bar2,bar3,foo0,foo1,foo2,foo3,foo4,foo5,'

    tmpl = Template('{% for v in form_objects|order_by:"datefield"|getlist:"foo_foo" %}{{ v }},{% endfor %}')
    assert tmpl.render(context) == 'bar,foo0,foo1,foo2,foo3,foo4,foo5,bar3,bar2,bar1,bar0,'


def test_lazy_global_forms(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobarlazy'
    formdef.fields = [
        fields.StringField(id='1', label='Test', type='string', varname='foo'),
        fields.StringField(id='2', label='key', type='string', varname='key'),
        fields.ItemsField(id='3', label='Items', type='items', varname='items'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    for i in range(6):
        formdata = formdef.data_class()()
        formdata.data = {'1': 'bar', '2': str(i), '3': [str(i % 2), str(i % 3)]}
        formdata.just_created()
        formdata.store()
    for i in range(4):
        formdata = formdef.data_class()()
        formdata.data = {'1': 'foo', '2': str(i + 6), '3': [str(i)]}
        formdata.just_created()
        formdata.jump_status('finished')
        formdata.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'bar'}
    formdata.status = 'draft'
    formdata.store()

    # template tags
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{forms.foobarlazy.slug}}')
    assert tmpl.render(context) == 'foobarlazy'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"foo"|filter_value:"bar"|count}}')
    assert tmpl.render(context) == '6'

    # filter value by multiple valued field
    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|filter_value:"3"|count}}')
    assert tmpl.render(context) == '1'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|filter_value:"2"|count}}')
    assert tmpl.render(context) == '3'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|filter_value:2|count}}')  # int
    assert tmpl.render(context) == '3'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|filter_value:"1|2"|count}}')
    assert tmpl.render(context) == '7'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|filter_value:"6"|count}}')
    assert tmpl.render(context) == '0'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|exclude_value:"1|2"|count}}')
    assert tmpl.render(context) == '3'

    tmpl = Template('{{forms|objects:"foobarlazy"|filter_by:"items"|exclude_value:"6"|count}}')
    assert tmpl.render(context) == '10'

    pub.custom_view_class.wipe()

    custom_view1 = pub.custom_view_class()
    custom_view1.title = 'datasource form view'
    custom_view1.formdef = formdef
    custom_view1.columns = {'list': [{'id': 'id'}, {'id': 'time'}, {'id': 'status'}]}
    custom_view1.filters = {'filter-1': 'on', 'filter-1-value': 'bar'}
    custom_view1.visibility = 'datasource'
    custom_view1.order_by = '-f2'
    custom_view1.store()

    custom_view2 = pub.custom_view_class()
    custom_view2.title = 'shared form view'
    custom_view2.formdef = formdef
    custom_view2.columns = {'list': [{'id': 'id'}, {'id': 'time'}, {'id': 'status'}]}
    custom_view2.filters = {'filter-1': 'on', 'filter-1-value': 'foo'}
    custom_view2.visibility = 'any'
    custom_view2.order_by = 'f2'
    custom_view2.store()

    custom_view3 = pub.custom_view_class()
    custom_view3.title = 'private form view'
    custom_view3.formdef = formdef
    custom_view3.columns = {'list': [{'id': 'id'}]}
    custom_view3.filters = {}
    custom_view3.visibility = 'owner'
    custom_view3.usier_id = 42
    custom_view3.order_by = 'id'
    custom_view3.store()

    tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"datasource-form-view"|count}}')
    assert tmpl.render(context) == '6'
    tmpl = Template(
        '{% for data in forms|objects:"foobarlazy"|with_custom_view:"datasource-form-view" %}{{ data.internal_id }},{% endfor %}'
    )
    assert tmpl.render(context) == '6,5,4,3,2,1,'

    tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"shared-form-view"|count}}')
    assert tmpl.render(context) == '4'
    tmpl = Template(
        '{% for data in forms|objects:"foobarlazy"|with_custom_view:"shared-form-view" %}{{ data.internal_id }},{% endfor %}'
    )
    assert tmpl.render(context) == '7,8,9,10,'

    tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"private-form-view"|count}}')
    assert tmpl.render(context) == '0'
    tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown"|count}}')
    assert tmpl.render(context) == '0'

    custom_view4 = pub.custom_view_class()
    custom_view4.title = 'unknown filter'
    custom_view4.formdef = formdef
    custom_view4.columns = {'list': [{'id': 'id'}]}
    custom_view4.filters = {'filter-42': 'on', 'filter-42-value': 'foo', 'filter-foobar': 'baz'}
    custom_view4.visibility = 'any'
    custom_view4.store()
    tmpl = Template('{{forms|objects:"foobarlazy"|with_custom_view:"unknown-filter"|count}}')
    assert tmpl.render(context) == '0'
    assert pub.loggederror_class.count() == 2
    logged_error = pub.loggederror_class.select(order_by='id')[0]
    assert logged_error.summary == 'Invalid filter "42"'
    assert logged_error.formdef_id == str(formdef.id)
    logged_error = pub.loggederror_class.select(order_by='id')[1]
    assert logged_error.summary == 'Invalid filter "foobar"'
    assert logged_error.formdef_id == str(formdef.id)


def test_lazy_variables(pub, variable_test_data):
    formdata = FormDef.select()[0].data_class().select()[0]
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        assert context['form_number'] == formdata.get_display_id()
        assert context['form_display_name'] == formdata.get_display_name()
        assert context['form_var_foo_foo'] == 'bar'
        with pytest.raises(KeyError):
            # noqa pylint: disable=pointless-statement
            context['form_var_xxx']
        assert 'bar' in context['form_var_foo_foo']
        assert context['form_var_foo_foo'] + 'ab' == 'barab'
        for item in enumerate(context['form_var_foo_foo']):
            assert item in [(0, 'b'), (1, 'a'), (2, 'r')]
        assert context['form_var_foo_foo_baz_baz'] == 'other'
        assert context['form_var_pwd_cleartext'] == 'a'


def test_lazy_variables_missing(pub, variable_test_data):
    formdef = FormDef.select()[0]
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {
        '0': 'bar',
    }
    pub.substitutions.reset()
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        assert context['form_var_foo_foo_baz_baz'] is None
        assert context['form_var_foo_foo'] == 'bar'
        with pytest.raises(KeyError):
            assert context['form_var_foo_foo_xxx'] == 'bar'


def test_lazy_variables_length(pub, variable_test_data):
    formdef = FormDef.select()[0]
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {
        '0': 'bar',
        '6': '3',
    }
    pub.substitutions.reset()
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)
    assert WorkflowStatusItem.compute('{{ form_var_foo_foo|length }}') == '3'
    assert WorkflowStatusItem.compute('{% if form_var_foo_foo|length_is:3 %}ok{% endif %}') == 'ok'
    assert (
        WorkflowStatusItem.compute(
            '{% if form_var_foo_foo|length_is:form_var_foo_foo_baz_baz %}ok{% endif %}'
        )
        == 'ok'
    )


def test_lazy_map_variable(pub, variable_test_data):
    formdef = FormDef.select()[0]
    formdata = formdef.data_class().select()[0]
    for mode in (None, 'lazy'):
        pub.substitutions.reset()
        pub.substitutions.feed(formdef)
        with pub.substitutions.temporary_feed(formdata, force_mode=mode):
            assert WorkflowStatusItem.compute('=form_var_map', raises=True) == '2;4'
            assert WorkflowStatusItem.compute('{{ form_var_map }}', raises=True) == '2;4'
            assert WorkflowStatusItem.compute('=form_var_map.split(";")[0]', raises=True) == '2'
            assert WorkflowStatusItem.compute('{{ form_var_map|split:";"|first }}', raises=True) == '2'
            assert WorkflowStatusItem.compute('=form_var_map_lat', raises=True) == 2
            assert WorkflowStatusItem.compute('{{ form_var_map_lat }}', raises=True) == '2.0'
            assert WorkflowStatusItem.compute('=form_var_map_lon', raises=True) == 4
            assert WorkflowStatusItem.compute('{{ form_var_map_lon }}', raises=True) == '4.0'

            assert (
                WorkflowStatusItem.compute(
                    '{{ form_var_map|distance:form_var_map|floatformat }}', raises=True
                )
                == '0'
            )
            assert (
                WorkflowStatusItem.compute('{{ form_var_map|distance:"2.1;4.1"|floatformat }}', raises=True)
                == '15685.4'
            )
            assert (
                WorkflowStatusItem.compute('{{ "2.1;4.1"|distance:form_var_map|floatformat }}', raises=True)
                == '15685.4'
            )
            assert WorkflowStatusItem.compute('{{ form|distance:"1;2"|floatformat }}', raises=True) == '0'
            assert (
                WorkflowStatusItem.compute('{{ form|distance:"1.1;2.1"|floatformat }}', raises=True)
                == '15689.1'
            )
            assert (
                WorkflowStatusItem.compute('{{ "1.1;2.1"|distance:form|floatformat }}', raises=True)
                == '15689.1'
            )
            assert (
                WorkflowStatusItem.compute('{{ form|distance:form_var_map|floatformat }}', raises=True)
                == '248515.5'
            )
            assert (
                WorkflowStatusItem.compute('{{ form_var_map|distance:form|floatformat }}', raises=True)
                == '248515.5'
            )

    formdata.data['7'] = None
    formdata.store()
    pub.substitutions.reset()
    pub.substitutions.feed(formdef)
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        pub.substitutions.reset()
        pub.substitutions.feed(formdef)
        with pub.substitutions.temporary_feed(formdata, force_mode=mode):
            assert WorkflowStatusItem.compute('=form_var_map', raises=True) is None
            assert (
                WorkflowStatusItem.compute('{{ form_var_map|distance:"1;2"|floatformat }}', raises=True) == ''
            )
            assert (
                WorkflowStatusItem.compute('{{ "1;2"|distance:form_var_map|floatformat }}', raises=True) == ''
            )


def test_lazy_file_workflow_option(pub):
    Workflow.wipe()
    FormDef.wipe()

    wf = Workflow(name='variables')
    wf.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=wf)
    wf.variables_formdef.fields = [
        fields.StringField(id='1', label='Test', type='string', varname='foo'),
        fields.FileField(id='2', label='File Test', type='file', varname='bar'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'foo'
    formdef.store()
    formdef.workflow_id = wf.id
    formdef.workflow_options = {'foo': 'bar', 'bar': PicklableUpload('test.txt', 'text/plain')}
    formdef.workflow_options['bar'].receive([b'hello world'])
    formdef.store()

    formdata = formdef.data_class()()
    substvars = CompatibilityNamesDict()
    substvars.update(formdata.get_substitution_variables())
    assert 'form_option_foo' in substvars.get_flat_keys()
    assert 'form_option_bar' in substvars.get_flat_keys()
    assert 'form_option_bar_url' not in substvars.get_flat_keys()


def test_lazy_strip_method(pub, variable_test_data):
    formdef = FormDef.select()[0]
    formdata = formdef.data_class().select()[0]
    formdata.data['0'] = ' bar '
    for mode in (None, 'lazy'):
        pub.substitutions.reset()
        pub.substitutions.feed(formdef)
        with pub.substitutions.temporary_feed(formdata, force_mode=mode):
            assert WorkflowStatusItem.compute('{{ form_var_foo_foo }}', raises=True) == ' bar '
            assert WorkflowStatusItem.compute('{{ form_var_foo_foo.strip }}', raises=True) == 'bar'


def test_lazy_conditions(pub, variable_test_data):
    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo == "bar"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo|startswith:"ba"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo|startswith:"fo"'})
    assert condition.evaluate() is False

    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo|slice:":2" == "ba"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo|slice:":2" == "fo"'})
    assert condition.evaluate() is False

    condition = Condition({'type': 'django', 'value': 'form.var.foo_foo == "bar"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_field_string == "bar"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name == "foobar"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_user_email == "bar@localhost"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_filefield_raw_content_type == "text/plain"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_user_admin_access'})
    assert condition.evaluate() is False

    condition = Condition({'type': 'django', 'value': 'form_user_backoffice_access'})
    assert condition.evaluate() is False

    condition = Condition({'type': 'python', 'value': 'vars().get("form_var_foo_foo_xxx", "") == ""'})
    assert condition.evaluate() is True

    user = pub.user_class.select()[0]
    user.is_admin = True
    user.store()

    condition = Condition({'type': 'django', 'value': 'form_user_admin_access'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_user_backoffice_access'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_datefield == "2018-07-31"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield != "2018-07-31"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_datefield == "31/07/2018"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield != "31/07/2018"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_datefield >= "31/07/2018"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield <= "31/07/2018"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield > "31/07/2018"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_datefield < "31/07/2018"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': '"2018-07-31" == form_var_datefield'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': '"31/07/2018" == form_var_datefield'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield != form_var_datefield2'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield == form_var_datefield2'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_datefield < form_var_datefield2'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield > form_var_datefield2'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_datefield <= form_var_datefield2'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datefield >= form_var_datefield2'})
    assert condition.evaluate() is False
    # compare with string var representing a date
    condition = Condition({'type': 'django', 'value': 'form_var_datefield == form_var_datestring'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datestring == "2018-07-31"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_var_datestring == "31/07/2018"'})
    assert condition.evaluate() is False  # datestring is not a date !
    # non existing form_var
    condition = Condition({'type': 'django', 'value': 'form_var_datefield == form_var_barbarbar'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'not form_var_datefield == form_var_barbarbar'})
    assert condition.evaluate() is True


def test_lazy_conditions_in(pub, variable_test_data):
    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo in "foo, bar, baz"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo not in "foo, bar, baz"'})
    assert condition.evaluate() is False

    condition = Condition({'type': 'django', 'value': '"b" in form_var_foo_foo'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': '"b" not in form_var_foo_foo'})
    assert condition.evaluate() is False


def test_has_role_templatefilter(pub, variable_test_data):
    condition = Condition({'type': 'django', 'value': 'form_user|has_role:"foobar"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_user|has_role:form_role_receiver_name'})
    assert condition.evaluate() is False

    role = pub.role_class.select()[0]
    user = pub.user_class.select()[0]
    user.roles = [role.id, '42']  # role.id 42 does not exist
    user.store()

    condition = Condition({'type': 'django', 'value': 'form_user|has_role:"foobar"'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_user|has_role:form_role_receiver_name'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_user|has_role:"barfoo"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_user|has_role:form_var_foo_foo'})
    assert condition.evaluate() is False

    # non-user object
    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo|has_role:"foobar"'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'xxx|has_role:"foobar"'})
    assert condition.evaluate() is False


def test_roles_templatefilter(pub, variable_test_data):
    condition = Condition({'type': 'django', 'value': '"foobar" in form_user|roles'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name in form_user|roles'})
    assert condition.evaluate() is False

    role = pub.role_class.select()[0]
    user = pub.user_class.select()[0]
    user.roles = [role.id, '42']  # role.id 42 does not exist
    user.store()

    condition = Condition({'type': 'django', 'value': '"foobar" in form_user|roles'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name in form_user|roles'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': '"barfoo" in form_user|roles'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': 'form_var_foo_foo in form_user|roles'})
    assert condition.evaluate() is False

    # non-user object
    condition = Condition({'type': 'django', 'value': '"foobar" in form_var_foo_foo|roles'})
    assert condition.evaluate() is False
    condition = Condition({'type': 'django', 'value': '"foobar" in xxx|roles'})
    assert condition.evaluate() is False


def test_function_users(pub):
    user1 = pub.user_class(name='userA')
    user1.store()
    user2 = pub.user_class(name='userB')
    user2.store()

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.workflow_roles = {'_receiver': '_user:%s' % user1.id}
    formdata.store()
    pub.substitutions.feed(formdata)
    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name == "userA"'})
    assert condition.evaluate() is True
    static_vars = formdata.get_static_substitution_variables()
    assert static_vars['form_role_receiver_name'] == 'userA'

    formdata.workflow_roles = {'_receiver': ['_user:%s' % user1.id, '_user:%s' % user2.id]}
    formdata.store()
    pub.substitutions.feed(formdata)
    condition = Condition({'type': 'django', 'value': 'form_role_receiver_name == "userA, userB"'})
    assert condition.evaluate() is True
    static_vars = formdata.get_static_substitution_variables()
    assert static_vars['form_role_receiver_name'] == 'userA, userB'

    # combined roles and users
    role = pub.role_class('test combined')
    role.store()
    formdata.workflow_roles = {'_receiver': [str(role.id), '_user:%s' % user1.id, '_user:%s' % user2.id]}
    formdata.store()
    pub.substitutions.feed(formdata)
    condition = Condition(
        {'type': 'django', 'value': 'form_role_receiver_name == "test combined, userA, userB"'}
    )
    assert condition.evaluate() is True
    static_vars = formdata.get_static_substitution_variables()
    assert static_vars['form_role_receiver_name'] == 'test combined, userA, userB'

    formdata.workflow_roles = {'_receiver': ['_user:%s' % user1.id, '_user:%s' % user2.id, str(role.id)]}
    formdata.store()
    pub.substitutions.feed(formdata)
    condition = Condition(
        {'type': 'django', 'value': 'form_role_receiver_name == "userA, userB, test combined"'}
    )
    assert condition.evaluate() is True
    static_vars = formdata.get_static_substitution_variables()
    assert static_vars['form_role_receiver_name'] == 'userA, userB, test combined'


def test_lazy_now_and_today(pub, variable_test_data):
    for condition_value in (
        'now > "1970-01-01"',
        '"1970-01-01" < now',
        'now < "2100-01-01"',
        '"2100-01-01" > now',
        'not now < "1970-01-01"',
        'not "1970-01-01" > now',
        'not now > "2100-01-01"',
        'not "2100-01-01" < now',
        # form_var_datefield is in 2018, we hope now is after 2019
        'form_var_datefield < now',
        'not form_var_datefield > now',
        'form_var_datefield <= now',
        'not form_var_datefield >= now',
        'form_var_datefield != now',
        'not form_var_datefield == now',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('now', 'today')})
        assert condition.evaluate() is True


def test_lazy_date_templatetags(pub, variable_test_data):
    for condition_value in (
        '"2017-10-10"|date == "2017-10-10"',
        '"2017-10-10"|date == "2017-10-10 00:00"',
        '"2017-10-10 00:00"|date == "2017-10-10 00:00"',
        '"2017-10-10"|date == "10/10/2017"',
        '"2017-10-10"|date == "10/10/2017 00:00"',
        '"2017-10-10"|date == "10/10/2017 00h00"',
        'not "2017-10-10"|date == "11/10/2017"',
        '"2017-10-10"|date != "11/10/2017"',
        'not "2017-10-10"|date != "10/10/2017"',
        '"2017-10-10"|date > "09/10/2017"',
        'not "2017-10-10"|date > "10/10/2017"',
        '"2017-10-10"|date < "11/10/2017"',
        'not "2017-10-10"|date < "10/10/2017"',
        '"2017-10-10"|date >= "09/10/2017"',
        '"2017-10-10"|date <= "10/10/2017"',
        '"2017-10-10"|date >= "10/10/2017"',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('date', 'datetime')})
        assert condition.evaluate() is True
        # |date on the right member
        condition = Condition({'type': 'django', 'value': condition_value.replace('|date', '') + '|date'})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('|date', '') + '|datetime'})
        assert condition.evaluate() is True

    for condition_value in (
        '"2017-10-11 12:13"|datetime == "2017-10-11 12:13"',
        '"2017-10-11 12:13:14"|datetime == "2017-10-11 12:13:14"',
        '"2017-10-11 12:13"|datetime != "2017-10-11 12:13:14"',
        'not "2017-10-11 12:13"|datetime == "2017-10-11 12:13:14"',
        '"2017-10-11 12:13"|datetime < "2017-10-11 12:13:14"',
        '"2017-10-11 12:13"|datetime <= "2017-10-11 12:13:14"',
        '"2017-10-11 12:13:14"|datetime <= "2017-10-11 12:13:14"',
        '"2017-10-11 12:13:14"|datetime > "2017-10-11 12:13"',
        '"2017-10-11 12:13:14"|datetime >= "2017-10-11 12:13"',
        '"2017-10-11 12:13:14"|datetime >= "2017-10-11 12:13:14"',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        # |datetime on the right member
        condition = Condition(
            {'type': 'django', 'value': condition_value.replace('|datetime', '') + '|datetime'}
        )
        assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': '"2017-10-11 00:00"|datetime == "2017-10-11"|date'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': '"2017-10-11 12:13"|date == "2017-10-11"'})
    assert condition.evaluate() is True
    condition = Condition(
        {'type': 'django', 'value': '"2017-10-11 12:13"|date == "2017-10-11 00:00"|datetime'}
    )
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': '"2017-10-11 12:13"|date == "2017-10-11 14:15"|date'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': '"2017-10-11 01:00"|datetime|date == "2017-10-11"'})
    assert condition.evaluate() is True

    condition = Condition({'type': 'django', 'value': 'now|date == today'})
    assert condition.evaluate() is True
    condition = Condition({'type': 'django', 'value': 'today == now|date'})
    assert condition.evaluate() is True


def test_lazy_date_with_maths(pub, variable_test_data):
    # form_var_datefield  : 2018-07-31
    # form_var_datefield2 : 2018-08-31
    # form_var_datestring : "2018-07-31"
    for condition_value in (
        'form_var_datefield|add_days:0 <= "2019-01-01"',
        'form_var_datefield|add_days:0 >= "1980-01-01"',
        'form_var_datefield|add_days:0 == "2018-07-31"',
        'form_var_datefield|add_days:0 == "31/07/2018"',
        'form_var_datefield|add_days:5 == "2018-08-05"',
        'form_var_datefield|add_days:5 <= "2018-08-05"',
        'form_var_datefield|add_days:5 >= "2018-08-05"',
        'form_var_datefield|add_days:-5 == "2018-07-26"',
        'form_var_datefield|add_days:36500 > "2100-01-01"',  # 2118
        'form_var_datefield|add_days:-36500 < "1950-01-01"',  # 1918
        'form_var_datefield|add_days:31 == form_var_datefield2',
        'form_var_datefield|add_days:5|add_days:-5 == form_var_datestring',
        'form_var_datestring|add_days:31 == form_var_datefield2',
        'form_var_datestring|add_days:32 > form_var_datefield2',
        'form_var_datestring|add_days:30 < form_var_datefield2',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True

    # form_var_datefield is in 2018, we hope today/now is after 2019
    for condition_value in (
        'form_var_datefield <= today',
        '"1970-01-01"|add_days:0 < today',
        '"2100-12-31"|add_days:0 > today',
        'form_var_datefield|add_days:0 <= today',
        'form_var_datefield|add_days:10 < today',
        'form_var_datefield|add_days:36500 > today',
        'form_var_datefield|add_days:36500 >= today',
        'form_var_datefield <= today|add_days:0',
        'form_var_datefield < today|add_days:-10',
        'form_var_datefield > today|add_days:-36500',
        'form_var_datefield >= today|add_days:-36500',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('today', 'now')})
        assert condition.evaluate() is True

    for condition_value in (
        '"1970-01-01"|add_hours:0 < today',
        '"2100-12-31"|add_hours:0 > today',
        'form_var_datefield|add_hours:0 <= today',
        'form_var_datefield|add_hours:10 < today',
        'form_var_datefield|add_hours:876000 > today',  # + 100 years
        'form_var_datefield|add_hours:876000 >= today',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('today', 'now')})
        assert condition.evaluate() is True

    for condition_value in (
        '"1970-01-01"|add_minutes:0 < today',
        '"2100-12-31"|add_minutes:0 > today',
        'form_var_datefield|add_minutes:0 <= today',
        'form_var_datefield|add_minutes:10 < today',
        'form_var_datefield|add_minutes:21024000 > today',  # + 100 years
        'form_var_datefield|add_minutes:21024000 >= today',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('today', 'now')})
        assert condition.evaluate() is True

    for condition_value in (
        'today|add_days:0 == today',
        'today|add_hours:0 == today',
        'today|add_minutes:0 == today',
        'now|add_hours:0 == now',
        'now|add_minutes:0 == now',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True

    for condition_value in (
        'today|add_days:2 >= today',
        'today|add_days:2 > today',
        'today|add_days:2 != today',
        'not today|add_days:2 == today',
        'not today|add_days:2 <= today',
        'not today|add_days:2 < today',
        'today|add_days:-2 <= today',
        'today|add_days:-2 < today',
        'today|add_days:-2 != today',
        'not today|add_days:-2 >= today',
        'not today|add_days:-2 > today',
        'not today|add_days:-2 == today',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True
        condition = Condition({'type': 'django', 'value': condition_value.replace('today', 'now')})
        assert condition.evaluate() is True


def test_lazy_templates(pub, variable_test_data):
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{form_var_foo_foo}}')
    assert tmpl.render(context) == 'bar'

    tmpl = Template('[form_var_foo_foo]')
    assert tmpl.render(context) == 'bar'

    tmpl = Template('[form_name]')
    assert tmpl.render(context) == 'foobarlazy'

    tmpl = Template('[form_user_email]')
    assert tmpl.render(context) == 'bar@localhost'

    tmpl = Template('{{form_user_name_identifier_0}}')
    assert tmpl.render(context) == pub.user_class.select()[0].name_identifiers[0]

    tmpl = Template('{% if form_user_email == "bar@localhost" %}HELLO{% endif %}')
    assert tmpl.render(context) == 'HELLO'


def test_lazy_ezt_templates(pub, variable_test_data):
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('[form_var_foo_foo]')
    assert tmpl.render(context) == 'bar'

    tmpl = Template('[is form_var_foo_foo "bar"]HELLO[else]BYE[end]')
    assert tmpl.render(context) == 'HELLO'

    tmpl = Template('[form_user_name_identifier_0]')
    assert tmpl.render(context) == pub.user_class.select()[0].name_identifiers[0]


def test_lazy_formdata_fields(pub):
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='string', varname='string'),
        fields.ItemField(id='1', label='item', varname='item', items=['Foo', 'Bar']),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': 'Foo', '1': 'Foo', '1_display': 'Foo'}
    formdata.store()

    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')

    tmpl = Template('{% if form_var_string == "Foo" %}HELLO{% endif %}')
    assert tmpl.render(context) == 'HELLO'
    tmpl = Template('{% if form_var_item == "Foo" %}HELLO{% endif %}')
    assert tmpl.render(context) == 'HELLO'

    tmpl = Template('{% if form_var_string != "Foo" %}HELLO{% endif %}')
    assert tmpl.render(context) == ''
    tmpl = Template('{% if form_var_item != "Foo" %}HELLO{% endif %}')
    assert tmpl.render(context) == ''


def test_date_conditions_python(pub, variable_test_data):
    for pycondition in (
        'utils.age_in_days(form_var_datefield, form_var_datefield2) == 31',
        'utils.age_in_days(form_var_datefield, form_var_datefield) == 0',
        'utils.age_in_days(form_var_datefield, form_var_datestring) == 0',
        'utils.age_in_days(utils.add_days(today, -5)) == 5',
        'date(form_var_datefield) < today',
        'utils.add_days(form_var_datefield, 36500) > today',
        'date(form_var_datefield) + days(36500) > today',
        'utils.add_days(form_var_datefield, 31) == date(form_var_datefield2)',
        'date(form_var_datefield) + days(31) == date(form_var_datefield2)',
        'utils.add_days(form_var_datefield, 32) > date(form_var_datefield2)',
        'date(form_var_datefield) + days(32) > date(form_var_datefield2)',
        'utils.add_days(form_var_datefield, 30) < date(form_var_datefield2)',
        'date(form_var_datefield) + days(30) < date(form_var_datefield2)',
        'isinstance(now, datetime.datetime)',
        'isinstance(today, datetime.date)',
        'utils.age_in_days(today) == 0',
        'utils.age_in_days(now) == 0',
        'utils.age_in_years(today) == 0',
        'utils.age_in_years(now) == 0',
        'utils.age_in_years_and_months(today) == (0, 0)',
        'utils.age_in_years_and_months(now) == (0, 0)',
        'utils.age_in_years_and_months(form_var_datefield, form_var_datefield2) == (0, 1)',
    ):
        condition = Condition({'type': 'python', 'value': pycondition})
        assert condition.evaluate() is True


def test_date_conditions_django(pub, variable_test_data):
    for condition_value in (  # hope date is > 2018
        # age_in_days
        '"1970-01-01"|age_in_days > 0',
        '"01/01/1970"|age_in_days > 0',
        '"2500-01-01"|age_in_days < 0',
        '"01/01/2500"|age_in_days < 0',
        'form_var_datefield|age_in_days > 50',
        'form_var_datefield|age_in_days:form_var_datestring == 0',
        'form_var_datefield|age_in_days:form_var_datefield2 == 31',
        'form_var_datefield2|age_in_days:form_var_datefield == -31',
        'form_var_datefield|age_in_days:form_var_datefield == 0',
        'form_var_datestring|age_in_days:form_var_datefield == 0',
        'form_var_datestring|age_in_days:form_var_datestring == 0',
        'today|add_days:-5|age_in_days == 5',
        'today|add_days:5|age_in_days == -5',
        'today|age_in_days == 0',
        # with datetimes
        '"1970-01-01 02:03"|age_in_days > 0',
        '"01/01/1970 02h03"|age_in_days > 0',
        '"2500-01-01 02:03"|age_in_days < 0',
        '"01/01/2500 02h03"|age_in_days < 0',
        'now|age_in_days == 0',
        'now|add_hours:-24|age_in_days == 1',
        'now|add_hours:24|age_in_days == -1',
        '"2010-11-12 13:14"|age_in_days:"2010-11-12 13:14" == 0',
        '"2010-11-12 13:14"|age_in_days:"2010-11-12 12:14" == 0',
        '"2010-11-12 13:14"|age_in_days:"2010-11-12 14:14" == 0',
        '"2010-11-12 13:14"|age_in_days:"2010-11-13 13:13" == 1',
        '"2010-11-12 13:14"|age_in_days:"2010-11-13 13:15" == 1',
        # age_in_hours
        'now|add_hours:-5|age_in_hours == 5',
        'now|add_hours:25|age_in_hours == -24',
        'now|age_in_hours == 0',
        '"2010-11-12 13:14"|age_in_hours:"2010-11-12 13:14" == 0',
        '"2010-11-12 13:14"|age_in_hours:"2010-11-12 12:14" == -1',
        '"2010-11-12 13:14"|age_in_hours:"2010-11-12 14:14" == 1',
        '"2010-11-12 13:14"|age_in_hours:"2010-11-13 13:13" == 23',
        '"2010-11-12 13:14"|age_in_hours:"2010-11-13 13:15" == 24',
        '"1970-01-01 02:03"|age_in_hours > 0',
        '"01/01/1970 02h03"|age_in_hours > 0',
        '"2500-01-01 02:03"|age_in_hours < 0',
        '"01/01/2500 02h03"|age_in_hours < 0',
        # with dates
        '"1970-01-01"|age_in_hours > 0',
        '"01/01/1970"|age_in_hours > 0',
        '"2500-01-01"|age_in_hours < 0',
        '"01/01/2500"|age_in_hours < 0',
        'form_var_datefield|age_in_hours > 1200',
        'form_var_datefield|age_in_hours:form_var_datestring == 0',
        'form_var_datefield|age_in_hours:form_var_datefield2 == 744',  # 31*24
        'form_var_datefield2|age_in_hours:form_var_datefield == -744',
        'form_var_datefield|age_in_hours:form_var_datefield == 0',
        'form_var_datestring|age_in_hours:form_var_datefield == 0',
        'form_var_datestring|age_in_hours:form_var_datestring == 0',
        'today|add_days:-1|age_in_hours >= 24',
        'today|add_days:1|age_in_hours <= -0',
        'today|add_days:1|age_in_hours >= -24',
        'today|age_in_hours >= 0',
        # age_in_years
        '"1970-01-01"|age_in_years > 0',
        '"01/01/1970"|age_in_years > 0',
        '"2500-01-01"|age_in_years < 0',
        '"01/01/2500"|age_in_years < 0',
        'form_var_datefield|age_in_years:"2019-07-31" == 1',
        'form_var_datefield|age_in_years:"2019-09-20" == 1',
        'form_var_datefield|age_in_years:"2020-07-30" == 1',
        'form_var_datefield|age_in_years:"2020-07-31" == 2',
        'form_var_datestring|age_in_years:"2019-07-31" == 1',
        'today|age_in_years == 0',
        'today|add_days:-500|age_in_years == 1',
        'today|add_days:-300|age_in_years == 0',
        'today|add_days:300|age_in_years == -1',
        'now|age_in_years == 0',
        'now|add_days:-500|age_in_years == 1',
        'now|add_days:-300|age_in_years == 0',
        'now|add_days:300|age_in_years == -1',
        '"1970-01-01 02:03"|age_in_years > 0',
        '"2500-01-01 02:03"|age_in_years < 0',
        # age_in_months
        'form_var_datefield|age_in_months:form_var_datefield2 == 1',
        'form_var_datefield2|age_in_months:form_var_datefield == -1',
        'form_var_datefield|age_in_months:"2019-07-31" == 12',
        'form_var_datefield|age_in_months:"2019-08-20" == 12',
        'form_var_datefield|age_in_months:"2019-09-20" == 13',
        'form_var_datestring|age_in_months:"2019-09-20" == 13',
        '"1970-01-01"|age_in_months > 0',
        '"01/01/1970"|age_in_months > 0',
        '"2500-01-01"|age_in_months < 0',
        '"01/01/2500"|age_in_months < 0',
        '"1970-01-01 02:03"|age_in_months > 0',
        '"2500-01-01 02:03"|age_in_months < 0',
        # fail produce empty string
        'foobar|age_in_days == ""',
        '"foobar"|age_in_days == ""',
        '"1970-01-01"|age_in_days:"foobar" == ""',
        'foobar|age_in_hours == ""',
        '"foobar"|age_in_hours == ""',
        '"1970-01-01"|age_in_hours:"foobar" == ""',
        'foobar|age_in_years == ""',
        '"foobar"|age_in_years == ""',
        '"1970-01-01"|age_in_years:"foobar" == ""',
        'foobar|age_in_months == ""',
        '"foobar"|age_in_months == ""',
        '"1970-01-01"|age_in_months:"foobar" == ""',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True


def test_form_digest_date(pub):
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [fields.DateField(id='0', label='date', varname='date')]
    formdef.digest_templates = {'default': 'plop {{ form_var_date }} plop'}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015-05-12 plop'

    pub.cfg['language'] = {'language': 'fr'}
    pub.write_cfg()
    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 12/05/2015 plop'

    formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'

    formdef.digest_templates = {'default': 'plop {{ form_var_date_raw|date:"Y" }} plop'}
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'

    formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': None}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop  plop'

    # check there's no crash when an invalid variable is given
    formdef.digest_templates = {'default': 'plop {{ blah|date:"Y" }} plop'}
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop  plop'


def test_form_digest_error(pub):
    FormDef.wipe()
    pub.custom_view_class.wipe()

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [fields.DateField(id='0', label='date', varname='date')]
    formdef.digest_templates = {'default': 'plop {{ form_var_date|list }} plop'}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['default'] == 'ERROR'

    assert pub.loggederror_class.count() == 1
    logged_error = pub.loggederror_class.select()[0]
    assert logged_error.summary == 'Could not render digest (default)'
    assert logged_error.formdata_id == str(formdata.id)

    formdef.digest_templates = {
        'default': 'plop plop',
        'custom-view:foobar': 'plop {{ form_var_date|list }} plop',
    }
    formdef.store()

    formdata = formdef.data_class()()
    formdata.store()
    assert formdef.data_class().get(formdata.id).digests['custom-view:foobar'] == 'ERROR'

    assert pub.loggederror_class.count() == 2
    logged_error = pub.loggederror_class.select(order_by='id')[1]
    assert logged_error.summary == 'Could not render digest (custom view "foobar")'
    assert logged_error.formdata_id == str(formdata.id)


def test_lazy_formdata_decimal_filter(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='value', varname='value'),
        fields.StringField(id='1', label='arg', varname='arg'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': '3.14', '1': '3'}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)

        tmpl = Template('{{ form_var_value|decimal }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_value|decimal:1 }}')
        assert tmpl.render(context) == '3.1'

        tmpl = Template('{{ form_var_value|decimal:form_var_arg }}')
        assert tmpl.render(context) == '3.140'

        tmpl = Template('{{ 4.12|decimal:form_var_arg }}')
        assert tmpl.render(context) == '4.120'


def test_lazy_formdata_timesince_filter(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.DateField(id='0', label='value', varname='value'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        context['refdate'] = datetime.date(2015, 5, 22)

        tmpl = Template('{{ form_var_value|date|timesince:refdate }}')
        assert force_str(tmpl.render(context)) == '1 week, 3 days'

    # in lazy mode it's not even necessary to add the |date filter.
    context = pub.substitutions.get_context_variables(mode='lazy')
    context['refdate'] = datetime.date(2015, 5, 22)
    tmpl = Template('{{ form_var_value|timesince:refdate }}')
    assert force_str(tmpl.render(context)) == '1 week, 3 days'


def test_decimal_conditions_django(pub, variable_test_data):
    for condition_value in (
        'form_var_foo_foo|decimal == 0',
        'form_var_boolfield|decimal == 0',
        'form_var_boolfield2|decimal == 0',
        'form_var_datefield|decimal == 0',
        'form_var_datefield|decimal == 0',
        'form_var_filefield|decimal == 0',
        'form_var_foo_foo_baz_baz|decimal == 0',
        'form_var_map|decimal == 0',
        'form_var_datefield2|decimal == 0',
        'form_var_datestring|decimal == 0',
        'form_var_term1|decimal == 3',
        'form_var_term2|decimal == 4',
    ):
        condition = Condition({'type': 'django', 'value': condition_value})
        assert condition.evaluate() is True


def test_lazy_formdata_mathematics_filters(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='term1', varname='term1'),
        fields.StringField(id='1', label='term2', varname='term2'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': '3', '1': '4'}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)

        tmpl = Template('{{ form_var_term1|decimal }}')
        assert tmpl.render(context) == '3'

        tmpl = Template('{{ form_var_term1|add:form_var_term2 }}')
        assert tmpl.render(context) == '7'

        tmpl = Template('{{ form_var_term1|subtract:form_var_term2 }}')
        assert tmpl.render(context) == '-1'

        tmpl = Template('{{ form_var_term1|multiply:form_var_term2 }}')
        assert tmpl.render(context) == '12'

        tmpl = Template('{{ form_var_term1|divide:form_var_term2 }}')
        assert tmpl.render(context) == '0.75'


def test_lazy_formdata_add_filters(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='term0', varname='term0'),
        fields.StringField(id='1', label='term1', varname='term1'),
        fields.StringField(id='2', label='term2', varname='term2'),
        fields.StringField(id='3', label='term3', varname='term3'),
        fields.StringField(id='4', label='term4', varname='term4'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': 'foo', '1': '3,14', '2': '', '3': None}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)

        # add
        tmpl = Template('{{ form_var_term1|add:form_var_term1 }}')
        assert tmpl.render(context) == '6.28'

        tmpl = Template('{{ form_var_term1|add:form_var_term2 }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_term1|add:form_var_term3 }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_term1|add:form_var_term4 }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_term2|add:form_var_term1 }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_term3|add:form_var_term1 }}')
        assert tmpl.render(context) == '3.14'

        tmpl = Template('{{ form_var_term4|add:form_var_term1 }}')
        assert tmpl.render(context) == '3.14'

        # fallback to Django native add filter
        tmpl = Template('{{ form_var_term0|add:form_var_term0 }}')
        assert tmpl.render(context) == 'foofoo'

        tmpl = Template('{{ form_var_term0|add:form_var_term1 }}')
        assert tmpl.render(context) == 'foo3,14'

        tmpl = Template('{{ form_var_term0|add:form_var_term2 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term0|add:form_var_term3 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term0|add:form_var_term4 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term1|add:form_var_term0 }}')
        assert tmpl.render(context) == '3,14foo'

        tmpl = Template('{{ form_var_term2|add:form_var_term0 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term2|add:form_var_term2 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term2|add:form_var_term3 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term2|add:form_var_term4 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term3|add:form_var_term0 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term3|add:form_var_term2 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term3|add:form_var_term3 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term3|add:form_var_term4 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term4|add:form_var_term0 }}')
        assert tmpl.render(context) == 'foo'

        tmpl = Template('{{ form_var_term4|add:form_var_term2 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term4|add:form_var_term3 }}')
        assert tmpl.render(context) == ''

        tmpl = Template('{{ form_var_term4|add:form_var_term4 }}')
        assert tmpl.render(context) == ''


def test_mathematic_conditions_django(pub, variable_test_data):
    for true_condition_value in (
        # reminder
        'form_var_term1 == 3',
        'form_var_term2 == 4',
        # add
        'form_var_term1|add:form_var_term2 == 7',
        'form_var_term1|add:form_var_term2 == 7.0',
        'form_var_term1|add:form_var_term2 == "7"|decimal',
        'form_var_term1|add:form_var_term2 > 6',
        # subtract
        'form_var_term1|subtract:form_var_term2 == -1',
        'form_var_term1|subtract:form_var_term2 == -1.0',
        'form_var_term1|subtract:form_var_term2 == "-1"|decimal',
        'form_var_term1|subtract:form_var_term2 < 0',
        # multiply
        'form_var_term1|multiply:form_var_term2 == 12',
        'form_var_term1|multiply:form_var_term2 == 12.0',
        'form_var_term1|multiply:form_var_term2 == "12"|decimal',
        'form_var_term1|multiply:form_var_term2 > 10',
        # divide
        'form_var_term1|divide:form_var_term2 == 0.75',
        'form_var_term1|divide:form_var_term2 == 0.750',
        'form_var_term1|divide:form_var_term2 == "0.75"|decimal',
        'form_var_term1|divide:form_var_term2 > 0.5',
    ):
        condition = Condition({'type': 'django', 'value': true_condition_value})
        assert condition.evaluate() is True

    for false_condition_value in (
        'form_var_term1|add:form_var_term2 > 8',
        'form_var_term1|subtract:form_var_term2 > 0',
        'form_var_term1|multiply:form_var_term2 > 20',
        'form_var_term1|divide:form_var_term2 > 1',
    ):
        condition = Condition({'type': 'django', 'value': false_condition_value})
        assert condition.evaluate() is False


def test_lazy_formdata_ceil_filter(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='value', varname='value'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': '3.14'}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        tmpl = Template('{{ form_var_value|ceil }}')
        assert tmpl.render(context) == '4'
        tmpl = Template('{{ form_var_value|floor }}')
        assert tmpl.render(context) == '3'
        tmpl = Template('{{ form_var_value|abs }}')
        assert tmpl.render(context) == '3.14'


def test_lazy_formdata_count_as_len_filter(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='0', label='value', varname='value'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'0': 'coin'}
    formdata.store()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        tmpl = Template('{{ form_var_value|count }}')
        assert tmpl.render(context) == '4'

    tmpl = Template('{{ form_var_value|count }}')
    assert tmpl.render({}) == '0'
    assert tmpl.render({'form_var_value': None}) == '0'


def test_rounding_and_abs_conditions_django(pub, variable_test_data):
    for true_condition_value in (
        # reminder
        'form_var_value == 3.14',
        # ceil
        'form_var_value|ceil == 4',
        'form_var_value|ceil == 4.0',
        'form_var_value|ceil > 3',
        # floor
        'form_var_value|floor == 3',
        'form_var_value|floor == 3.0',
        'form_var_value|floor >= 3',
        # abs
        'form_var_value|abs == 3.14|decimal',
        'form_var_value|abs == 3.140|decimal',
        'form_var_value|abs >= 3',
    ):
        condition = Condition({'type': 'django', 'value': true_condition_value})
        assert condition.evaluate() is True

    for false_condition_value in (
        'form_var_value|ceil < 4',
        'form_var_value|ceil >= 5',
        'form_var_value|floor < 3',
    ):
        condition = Condition({'type': 'django', 'value': false_condition_value})
        assert condition.evaluate() is False


def test_lazy_url_suffix(pub, variable_test_data):
    ds = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un', 'more': 'foo', 'url': 'xxx'},
                {'id': '2', 'text': 'deux', 'more': 'bar', 'url': 'yyy'},
            ]
        ),
    }

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.ItemField(id='1', label='item', data_source=ds, varname='plop'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '1'}
    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.store()

    pub.substitutions.reset()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        tmpl = Template('{{ form_var_plop_more }}')
        assert tmpl.render(context) == 'foo'
        tmpl = Template('{{ form_var_plop_url }}')
        assert tmpl.render(context) == 'xxx'


def test_lazy_structured_items(pub, variable_test_data):
    ds = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un', 'more': 'foo', 'url': 'xxx', 'invalid:key': 'xxx'},
                {'id': '2', 'text': 'deux', 'more': 'bar', 'url': 'yyy', 'invalid:key': 'yyy'},
            ]
        ),
    }

    ds2 = {
        'type': 'formula',
        'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}]),
    }

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.ItemsField(id='1', label='items', data_source=ds, varname='plop'),
        fields.ItemsField(id='2', label='items', data_source=ds2, varname='plop2'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': ['1', '2'], '2': ['1', '2']}
    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.data['2_display'] = formdef.fields[1].store_display_value(formdata.data, '2')
    formdata.data['2_structured'] = formdef.fields[1].store_structured_value(formdata.data, '2')
    formdata.store()

    pub.substitutions.reset()
    pub.substitutions.feed(formdata)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        tmpl = Template('{{ form_var_plop_0_more }}')
        assert tmpl.render(context) == 'foo'
        tmpl = Template('{{ form_var_plop_0_url }}')
        assert tmpl.render(context) == 'xxx'
        tmpl = Template('{{ form_var_plop_1_more }}')
        assert tmpl.render(context) == 'bar'
        tmpl = Template('{{ form_var_plop_1_url }}')
        assert tmpl.render(context) == 'yyy'
        tmpl = Template('{% for x in form_var_plop_structured_raw %}{{x.more}}{% endfor %}')
        assert tmpl.render(context) == 'foobar'
        tmpl = Template('{% for x in form_var_plop_structured %}{{x.more}}{% endfor %}')
        assert tmpl.render(context) == 'foobar'

    flat_keys = pub.substitutions.get_context_variables(mode='lazy').get_flat_keys()
    assert 'form_var_plop_0_url' in flat_keys
    assert 'form_var_plop_1_more' in flat_keys
    assert 'form_var_plop_structured' in flat_keys
    assert 'form_var_plop_raw' in flat_keys
    assert 'form_var_plop_0_invalid:key' not in flat_keys

    assert 'form_var_plop2' in flat_keys
    assert 'form_var_plop2_raw' in flat_keys
    assert 'form_var_plop2_structured' in flat_keys


def test_formdata_user_field(pub, variable_test_data):
    local_user = variable_test_data._formdata.user

    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.StringField(id='3', label='test', varname='test', type='string'))
    user_formdef.store()
    local_user.form_data = {'3': 'nono'}
    local_user.set_attributes_from_formdata(local_user.form_data)
    local_user.store()

    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        tmpl = Template('{{ form_user_var_test }}')
        assert tmpl.render(context) == 'nono'

    condition = Condition({'type': 'django', 'value': 'form_user_var_test'})
    assert condition.evaluate() is True

    local_user.form_data = {'3': ''}
    local_user.set_attributes_from_formdata(local_user.form_data)
    local_user.store()
    condition = Condition({'type': 'django', 'value': 'form_user_var_test'})
    assert condition.evaluate() is False


def test_string_filters(pub, variable_test_data):
    tmpl = Template('{% with form_var_foo_foo|split:"a" as x %}{{x.0}}{% endwith %}', raises=True)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        assert tmpl.render(context) == 'b'

    tmpl = Template('{% if form_var_foo_foo|startswith:"b" %}test{% endif %}', raises=True)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        assert tmpl.render(context) == 'test'

    tmpl = Template('{% if form_var_foo_foo|startswith:form_var_term1 %}test{% endif %}', raises=True)
    for mode in (None, 'lazy'):
        context = pub.substitutions.get_context_variables(mode=mode)
        assert tmpl.render(context) == ''


def test_user_label(pub):
    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.StringField(id='3', label='first_name', type='string'))
    user_formdef.fields.append(fields.StringField(id='4', label='last_name', type='string'))
    user_formdef.store()
    pub.cfg['users']['field_name'] = ['3', '4']
    pub.write_cfg()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.StringField(id='1', label='first_name', prefill={'type': 'user', 'value': '3'}),
        fields.StringField(id='2', label='last_name', prefill={'type': 'user', 'value': '4'}),
    ]
    formdef.store()

    user = pub.user_class()
    user.email = 'bar@localhost'
    user.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.user_id = user.id
    formdata.store()

    assert str(formdef.data_class().get(formdata.id).user_id) == str(user.id)
    assert formdef.data_class().get(formdata.id).user_label is None
    assert formdef.data_class().get(formdata.id).get_user_label() == 'bar@localhost'

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.store()

    assert formdef.data_class().get(formdata.id).user_id is None
    assert formdef.data_class().get(formdata.id).user_label == ''
    assert formdef.data_class().get(formdata.id).get_user_label() == ''

    formdata = formdef.data_class()()
    formdata.data = {'1': 'blah'}
    formdata.store()

    assert formdef.data_class().get(formdata.id).user_id is None
    assert formdef.data_class().get(formdata.id).user_label == 'blah'
    assert formdef.data_class().get(formdata.id).get_user_label() == 'blah'

    formdata = formdef.data_class()()
    formdata.data = {'1': 'blah', '2': 'xxx'}
    formdata.store()

    assert formdef.data_class().get(formdata.id).user_id is None
    assert formdef.data_class().get(formdata.id).user_label == 'blah xxx'
    assert formdef.data_class().get(formdata.id).get_user_label() == 'blah xxx'


def test_user_label_from_block(pub):
    from wcs.admin.settings import UserFieldsFormDef

    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.StringField(id='3', label='first_name', type='string'))
    user_formdef.fields.append(fields.StringField(id='4', label='last_name', type='string'))
    user_formdef.store()
    pub.cfg['users']['field_name'] = ['3', '4']
    pub.write_cfg()

    BlockDef.wipe()
    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.StringField(id='1', label='first_name', prefill={'type': 'user', 'value': '3'}),
        fields.StringField(id='2', label='last_name', prefill={'type': 'user', 'value': '4'}),
    ]
    block.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='block'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.store()
    assert not formdef.data_class().get(formdata.id).user_label

    formdata = formdef.data_class()()
    formdata.data = {
        '1': {
            'data': [{'1': 'first', '2': 'last'}],
            'schema': {'1': 'string', '2': 'string'},
        }
    }
    formdata.store()
    assert formdef.data_class().get(formdata.id).user_label == 'first last'


def test_form_parent(pub):
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [fields.StringField(id='0', label='foo', varname='foo')]
    formdef.store()

    parent = formdef.data_class()()
    parent.data = {'0': 'hello'}
    parent.store()

    child = formdef.data_class()()
    child.data = {'0': 'world'}
    child.submission_context = {
        'orig_object_type': 'formdef',
        'orig_formdef_id': formdef.id,
        'orig_formdata_id': parent.id,
    }
    variables = parent.get_substitution_variables()
    assert variables.get('form_var_foo') == 'hello'
    assert variables.get('form_parent') is None

    assert str(variables['form'].var.foo) == 'hello'
    assert variables['form'].parent is None

    variables = child.get_substitution_variables()
    assert variables.get('form_var_foo') == 'world'
    assert variables.get('form_parent_form_var_foo') == 'hello'
    assert variables.get('form_parent') is not None

    assert str(variables['form'].var.foo) == 'world'
    assert str(variables['form'].parent['form'].var.foo) == 'hello'
    assert variables['form'].parent is not None


def test_block_variables(pub):
    BlockDef.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.digest_template = 'X{{foobar_var_foo}}Y'
    block.fields = [
        fields.StringField(id='123', required=True, label='Test', type='string', varname='foo'),
        fields.StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    formdef = FormDef()
    formdef.name = 'testblock'
    formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='block'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    # value from test_block_digest in tests/test_form_pages.py
    formdata.data = {
        '1': {
            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
            'schema': {'123': 'string', '234': 'string'},
        },
        '1_display': 'XfooY, Xfoo2Y',
    }
    formdata.store()

    variables = formdata.get_substitution_variables()
    assert 'form_var_block' in variables.get_flat_keys()
    assert 'form_var_block_0' in variables.get_flat_keys()
    assert 'form_var_block_0_foo' in variables.get_flat_keys()
    assert 'form_var_block_0_bar' in variables.get_flat_keys()
    assert 'form_var_block_1' in variables.get_flat_keys()
    assert 'form_var_block_1_foo' in variables.get_flat_keys()
    assert 'form_var_block_1_bar' in variables.get_flat_keys()

    assert variables.get('form_var_block_0_foo') == 'foo'
    assert variables.get('form_var_block_1_foo') == 'foo2'
    assert variables.get('form_var_block_var_foo') == 'foo'  # alias to 1st element

    pub.substitutions.reset()
    pub.substitutions.feed(formdata)

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{ form_var_block }}')
    assert tmpl.render(context) == 'XfooY, Xfoo2Y'

    tmpl = Template('{% for sub in form_var_block %}{{ sub.foo }} {% endfor %}')
    assert tmpl.render(context) == 'foo foo2 '

    tmpl = Template('{{ form_var_block|length }}')
    assert tmpl.render(context) == '2'

    tmpl = Template('{{ form_var_block|count }}')
    assert tmpl.render(context) == '2'

    tmpl = Template('{% for foo in form_var_block|getlist:"foo" %}/{{ foo }}{% endfor %}')
    assert tmpl.render(context) == '/foo/foo2'

    formdata.data = {
        '1': {
            'data': [{'123': '2', '234': '2'}, {'123': '7', '234': 'bla'}],
            'schema': {'123': 'string', '234': 'string'},
        },
    }
    formdata.store()
    tmpl = Template('{{ form_var_block|getlist:"foo"|sum }} {{ form_var_block|getlist:"bar"|sum }}')
    assert tmpl.render(context) == '9 2'

    tmpl = Template('{{ form_var_block|getlistdict:"foo:test, bar, unknown" }}', autoescape=False)
    assert (
        tmpl.render(context)
        == "[{'test': '2', 'bar': '2', 'unknown': None}, {'test': '7', 'bar': 'bla', 'unknown': None}]"
    )

    # check another count of elements
    formdata.data = {
        '1': {
            'data': [
                {'123': 'foo', '234': 'bar'},
                {'123': 'foo2', '234': 'bar2'},
                {'123': 'foo3', '234': 'bar3'},
            ],
            'schema': {'123': 'string', '234': 'string'},
        },
        '1_display': 'XfooY, Xfoo2Y, Xfoo3Z',
    }
    formdata.store()
    tmpl = Template('{{ form_var_block|length }}')
    assert tmpl.render(context) == '3'
    tmpl = Template('{{ form_var_block|count }}')
    assert tmpl.render(context) == '3'

    # check invalid varname are ignored (should not happen)
    block.fields[0].varname = 'foo-bar'
    block.store()
    formdef.refresh_from_storage()
    formdata = formdef.data_class().get(formdata.id)
    variables = formdata.get_substitution_variables()
    assert 'form_var_block_0_foo-bar' not in variables.get_flat_keys()


def test_block_with_empty_digest_variable(pub):
    BlockDef.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.digest_template = '{{foobar_var_foo}}'  # will render as empty string
    block.fields = [
        fields.StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    formdef = FormDef()
    formdef.name = 'testblock'
    formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='block'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdef.fields[0].set_value(formdata.data, {'data': [{'234': 'bar2'}], 'schema': {'234': 'string'}})
    assert formdata.data.get('1')
    assert formdata.data.get('1_display') is None
    formdata.just_created()
    formdata.store()

    variables = formdata.get_substitution_variables()
    assert 'form_var_block' in variables.get_flat_keys()  # advertised
    assert variables.get('form_var_block') == '---'  # placeholder for empty value


def test_block_set_value_to_empty_string(pub):
    BlockDef.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.fields = [
        fields.StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    formdef = FormDef()
    formdef.name = 'testblock'
    formdef.fields = [
        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='block'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdef.fields[0].set_value(formdata.data, '')
    formdata.store()
    assert formdata.data.get('1') is None
    assert formdata.data.get('1_display') is None


def test_items_field_getlist(pub):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un'},
                {'id': '2', 'text': 'deux'},
                {'id': '3', 'text': 'trois'},
                {'id': '4', 'text': 'quatre'},
            ]
        ),
    }
    data_source.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemsField(id='1', label='items', items=['aa', 'ab', 'ac'], varname='itemsfield'),
        fields.ItemsField(id='2', label='items2', varname='items2field', data_source={'type': 'foobar'}),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdef.fields[0].set_value(formdata.data, ['aa', 'ab'])
    formdef.fields[1].set_value(formdata.data, ['1', '3'])
    formdata.store()

    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{% for x in form_var_itemsfield|getlist:"text" %}{{x}},{% endfor %}')
    assert tmpl.render(context) == 'aa,ab,'

    tmpl = Template('{% for x in form_var_items2field|getlist:"text" %}{{x}},{% endfor %}')
    assert tmpl.render(context) == 'un,trois,'

    tmpl = Template('{{ form_var_items2field|getlistdict:"text, unknown" }}', autoescape=False)
    assert tmpl.render(context) == "[{'text': 'un', 'unknown': None}, {'text': 'trois', 'unknown': None}]"


def test_getlist_of_lazyformdata_field(pub):
    CardDef.wipe()
    carddef1 = CardDef()
    carddef1.name = 'Card 1'
    carddef1.fields = [
        fields.StringField(id='0', label='string', varname='name'),
    ]
    carddef1.store()
    ds = {'type': 'carddef:%s' % carddef1.url_name}
    carddef2 = CardDef()
    carddef2.name = 'Card 2'
    carddef2.fields = [fields.ItemField(id='0', label='card 1', type='item', varname='card1', data_source=ds)]
    carddef2.store()
    carddef1.data_class().wipe()
    carddef2.data_class().wipe()

    for i in range(3):
        carddata1 = carddef1.data_class()()
        carddata1.data = {
            '0': 'card1-%s' % i,
        }
        carddata1.just_created()
        carddata1.store()
        carddata2 = carddef2.data_class()()
        carddata2.data = {
            '0': str(carddata1.id),
        }
        carddata2.data['0_display'] = carddef2.fields[0].store_display_value(carddata2.data, '0')
        carddata2.data['0_structured'] = carddef2.fields[0].store_structured_value(carddata2.data, '0')
        carddata2.just_created()
        carddata2.store()

    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{{ cards|objects:"card-2"|order_by:"id"|getlist:"form_var_card1_name"|join:"," }}')
    assert tmpl.render(context) == 'card1-0,card1-1,card1-2'
    tmpl = Template('{{ cards|objects:"card-2"|order_by:"id"|getlist:"form_var_card1_id"|join:"," }}')
    assert tmpl.render(context) == '1,2,3'


def test_items_field_contains(pub):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un'},
                {'id': '2', 'text': 'deux'},
                {'id': '3', 'text': 'trois'},
                {'id': '4', 'text': 'quatre'},
            ]
        ),
    }
    data_source.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemsField(id='1', label='items', items=['aa', 'ab', 'ac'], varname='itemsfield'),
        fields.ItemsField(id='2', label='items2', varname='items2field', data_source={'type': 'foobar'}),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdef.fields[0].set_value(formdata.data, ['aa', 'ab'])
    formdef.fields[1].set_value(formdata.data, ['1', '3'])
    formdata.store()

    pub.substitutions.feed(formdata)
    context = pub.substitutions.get_context_variables(mode='lazy')
    tmpl = Template('{% if "aa" in form_var_itemsfield %}ok{% endif %}')
    assert tmpl.render(context) == 'ok'
    tmpl = Template('{% if "xa" in form_var_itemsfield %}ok{% endif %}')
    assert tmpl.render(context) == ''

    tmpl = Template('{% if "1" in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == 'ok'
    tmpl = Template('{% if 1 in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == 'ok'
    tmpl = Template('{% if "un" in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == 'ok'
    tmpl = Template('{% if "8" in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == ''
    tmpl = Template('{% if 8 in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == ''
    tmpl = Template('{% if "huit" in form_var_items2field %}ok{% endif %}')
    assert tmpl.render(context) == ''


def test_items_field_split(pub):
    NamedDataSource.wipe()
    data_source = NamedDataSource(name='foobar')
    data_source.data_source = {
        'type': 'formula',
        'value': repr(
            [
                {'id': '1', 'text': 'un'},
                {'id': '2', 'text': 'deux'},
                {'id': '3', 'text': 'trois'},
                {'id': '4', 'text': 'quatre'},
            ]
        ),
    }
    data_source.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        fields.ItemsField(id='1', label='items', items=['aa', 'ab', 'ac'], varname='itemsfield'),
        fields.ItemsField(id='2', label='items2', varname='items2field', data_source={'type': 'foobar'}),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdef.fields[0].set_value(formdata.data, ['aa', 'ab'])
    formdef.fields[1].set_value(formdata.data, ['1', '3'])
    formdata.store()

    for mode in (None, 'lazy'):
        pub.substitutions.reset()
        pub.substitutions.feed(formdef)
        with pub.substitutions.temporary_feed(formdata, force_mode=mode):
            assert WorkflowStatusItem.compute('=form_var_itemsfield', raises=True) == 'aa, ab'
            assert WorkflowStatusItem.compute('=form_var_itemsfield.split(",")[0]', raises=True) == 'aa'


def test_attachment_part_path_migration(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.status = 'wf-new'
    formdata.evolution[-1].status = 'wf-new'
    formdata.evolution[-1].parts = [
        AttachmentEvolutionPart(
            'hello.txt', fp=io.BytesIO(b'test'), content_type='text/plain', varname='testfile'
        )
    ]
    formdata.store()
    assert formdata.evolution[-1].parts[0].filename.startswith('attachments/')
    assert os.path.exists(formdata.evolution[-1].parts[0].get_file_path())

    # add full path as it was done before
    formdata.evolution[-1].parts[0].filename = os.path.join(
        pub.app_dir, formdata.evolution[-1].parts[0].filename
    )
    formdata.store()

    # check it was converted to relative path
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.evolution[-1].parts[0].filename.startswith('attachments/')

    # More realistic situation :
    # * an absolute path serialized to storage
    # * fetch the formadata
    # * store it again and check that the path is converted to a a relative path

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.status = 'wf-new'
    formdata.evolution[-1].status = 'wf-new'
    attachment_evolution_part = AttachmentEvolutionPart(
        'hello.txt', fp=io.BytesIO(b'test'), content_type='text/plain', varname='testfile'
    )
    formdata.evolution[-1].parts = [attachment_evolution_part]

    # monkeypatch so that the filename is stored as an absolute path
    def get_state():
        odict = attachment_evolution_part.__dict__.copy()
        del odict['__getstate__']
        if not odict.get('fp'):
            if 'filename' not in odict:
                # we need a filename as an identifier: create one from nothing
                # instead of file_digest(self.fp) (see below)
                odict['filename'] = 'uuid-%s' % uuid.uuid4()
                attachment_evolution_part.filename = odict['filename']
            return odict

        del odict['fp']

        # there is no filename, or it was a temporary one: create it
        if 'filename' not in odict or odict['filename'].startswith('uuid-'):
            filename = file_digest(attachment_evolution_part.fp)
            # create subdirectory with digest prefix as name
            dirname = os.path.join('attachments', filename[:4])
            abs_dirname = os.path.join(get_publisher().app_dir, dirname)
            os.makedirs(abs_dirname, exist_ok=True)
            odict['filename'] = os.path.join(abs_dirname, filename)
            attachment_evolution_part.filename = odict['filename']
            attachment_evolution_part.fp.seek(0)
            atomic_write(attachment_evolution_part.get_file_path(), attachment_evolution_part.fp)

        return odict

    attachment_evolution_part.__getstate__ = get_state

    formdata.store()
    # check that the path is absolute
    assert formdata.evolution[-1].parts[0].filename.startswith(pub.APP_DIR)
    assert os.path.exists(formdata.evolution[-1].parts[0].get_file_path())

    # get a fresh instance not monkeypatched
    formdata = formdef.data_class().get(formdata.id)
    formdata.store()

    # check it was converted to relative path
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.evolution[-1].parts[0].filename.startswith('attachments/')


def test_merged_roles_dict_compat(pub, local_user):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'test'
    formdef.fields = []
    formdef.workflow_roles = {'_receiver': 2}
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.workflow_roles = {'_receiver': 2}
    formdata.store()