wcs/tests/workflow/test_all.py

import base64
import datetime
import io
import json
import os
import shutil
import time
import urllib.parse
import zipfile
from unittest import mock

import pytest

try:
    from PIL import Image
except ImportError:
    Image = None

from django.utils.encoding import force_bytes, force_text
from pyzbar.pyzbar import ZBarSymbol
from pyzbar.pyzbar import decode as zbar_decode_qrcode
from quixote import cleanup, get_publisher, get_response
from quixote.http_request import Upload as QuixoteUpload

from wcs import sessions, sql
from wcs.blocks import BlockDef
from wcs.carddef import CardDef
from wcs.fields import (
    BlockField,
    BoolField,
    CommentField,
    DateField,
    EmailField,
    FileField,
    ItemField,
    ItemsField,
    MapField,
    PageField,
    StringField,
    SubtitleField,
    TableField,
    TextField,
    TitleField,
)
from wcs.formdata import Evolution
from wcs.formdef import FormDef
from wcs.qommon.errors import ConnectionError
from wcs.qommon.form import Form, UploadedFile, UploadValidationError
from wcs.qommon.http_request import HTTPRequest
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.qommon.upload_storage import PicklableUpload
from wcs.wf.aggregation_email import AggregationEmail, send_aggregation_emails
from wcs.wf.anonymise import AnonymiseWorkflowStatusItem
from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
from wcs.wf.create_formdata import Mapping
from wcs.wf.criticality import MODE_DEC, MODE_INC, MODE_SET, ModifyCriticalityWorkflowStatusItem
from wcs.wf.dispatch import DispatchWorkflowStatusItem
from wcs.wf.display_message import DisplayMessageWorkflowStatusItem
from wcs.wf.export_to_model import ExportToModel, transform_to_pdf
from wcs.wf.external_workflow import ManyExternalCallsPart
from wcs.wf.form import WorkflowFormFieldsFormDef
from wcs.wf.geolocate import GeolocateWorkflowStatusItem
from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
from wcs.wf.profile import UpdateUserProfileStatusItem
from wcs.wf.redirect_to_url import RedirectToUrlWorkflowStatusItem
from wcs.wf.register_comment import JournalEvolutionPart, RegisterCommenterWorkflowStatusItem
from wcs.wf.remove import RemoveWorkflowStatusItem
from wcs.wf.remove_tracking_code import RemoveTrackingCodeWorkflowStatusItem
from wcs.wf.roles import AddRoleWorkflowStatusItem, RemoveRoleWorkflowStatusItem
from wcs.wf.sms import SendSMSWorkflowStatusItem
from wcs.wf.wscall import WebserviceCallStatusItem
from wcs.workflows import (
    AbortActionException,
    AttachmentEvolutionPart,
    Workflow,
    WorkflowBackofficeFieldsFormDef,
    WorkflowCriticalityLevel,
    WorkflowStatusItem,
    WorkflowVariablesFieldsFormDef,
    perform_items,
)

from ..test_sql import column_exists_in_table
from ..utilities import MockSubstitutionVariables, clean_temporary_pub, create_temporary_pub, get_app, login


def setup_module(module):
    cleanup()


def teardown_module(module):
    clean_temporary_pub()


def pub_fixture(**kwargs):
    pub = create_temporary_pub(**kwargs)
    pub.cfg['language'] = {'language': 'en'}
    pub.cfg['identification'] = {'methods': ['password']}
    pub.write_cfg()
    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
    req.response.filter = {}
    req._user = None
    pub._set_request(req)
    req.session = sessions.BasicSession(id=1)
    pub.set_config(req)
    return pub


@pytest.fixture
def pub(request):
    return pub_fixture()


@pytest.fixture
def two_pubs(request):
    return pub_fixture(sql_mode=True)


@pytest.fixture
def sql_pub(request):
    return pub_fixture(sql_mode=True)


@pytest.fixture
def admin_user():
    get_publisher().user_class.wipe()
    user = get_publisher().user_class()
    user.name = 'John Doe Admin'
    user.email = 'john.doe@example.com'
    user.name_identifiers = ['0123456789']
    user.is_admin = True
    user.store()

    account = PasswordAccount(id='admin')
    account.set_password('admin')
    account.user_id = user.id
    account.store()

    return user


def test_get_json_export_dict(pub):
    workflow = Workflow(name='wf')
    st1 = workflow.add_status('Status1', 'st1')
    st2 = workflow.add_status('Status2', 'st2')
    st2.forced_endpoint = True

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = 0.1
    jump.status = 'st2'

    workflow.roles['_other'] = 'Other Function'
    root = workflow.get_json_export_dict()
    assert set(root.keys()) >= {'statuses', 'name', 'functions'}

    assert root['name'] == 'wf'
    assert len(root['statuses']) == 2
    assert {st['id'] for st in root['statuses']} == {'st1', 'st2'}
    assert all(set(status.keys()) >= {'id', 'name', 'forced_endpoint'} for status in root['statuses'])
    assert root['statuses'][0]['id'] == 'st1'
    assert root['statuses'][0]['name'] == 'Status1'
    assert root['statuses'][0]['forced_endpoint'] is False
    assert root['statuses'][0]['endpoint'] is False
    assert root['statuses'][1]['id'] == 'st2'
    assert root['statuses'][1]['name'] == 'Status2'
    assert root['statuses'][1]['forced_endpoint'] is True
    assert root['statuses'][1]['endpoint'] is True


def test_action_repr(pub):
    workflow = Workflow(name='wftest')
    st1 = workflow.add_status('Status1', 'st1')
    jump = JumpWorkflowStatusItem()
    assert repr(jump)  # no crash when not attached to status
    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = 0.1
    jump.status = 'st2'

    action = workflow.add_global_action('Timeout')
    criticality = action.add_action('modify_criticality')
    workflow.store()
    with open(workflow.get_object_filename(), 'rb') as fd:
        # make sure parent relations are not stored in pickles
        assert b'parent' not in fd.read()

    for wf in (workflow, Workflow.get(workflow.id)):
        action = wf.possible_status[0].items[0]
        assert 'Status1' in repr(action)
        assert 'wftest' in repr(action)
        action = wf.global_actions[0].items[0]
        assert 'Timeout' in repr(criticality)
        assert 'wftest' in repr(criticality)


def test_variable_compute(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'hello'}
    formdata.store()
    pub.substitutions.feed(formdata)

    item = JumpWorkflowStatusItem()

    # straight string
    assert item.compute('blah') == 'blah'

    # django template
    assert item.compute('{{ form_var_foo }}') == 'hello'
    assert item.compute('{{ form_var_foo }}', render=False) == '{{ form_var_foo }}'
    assert item.compute('{% if form_var_foo %}its here{% endif %}') == 'its here'
    assert item.compute('{% if form_var_foo %}') == '{% if form_var_foo %}'
    with pytest.raises(Exception):
        item.compute('{% if form_var_foo %}', raises=True)

    # ezt string
    assert item.compute('[form_var_foo]') == 'hello'
    # ezt string, but not ezt asked
    assert item.compute('[form_var_foo]', render=False) == '[form_var_foo]'
    # ezt string, with an error
    assert item.compute('[end]', raises=False) == '[end]'
    with pytest.raises(Exception):
        item.compute('[end]', raises=True)

    # python expression
    assert item.compute('=form_var_foo') == 'hello'
    # python expression, with an error
    assert item.compute('=1/0', raises=False) == '=1/0'
    with pytest.raises(Exception):
        item.compute('=1/0', raises=True)

    # with context
    assert item.compute('{{ form_var_foo }} {{ bar }}', context={'bar': 'world'}) == 'hello world'
    assert item.compute('[form_var_foo] [bar]', context={'bar': 'world'}) == 'hello world'
    assert item.compute('=form_var_foo + " " + bar', context={'bar': 'world'}) == 'hello world'

    # django wins
    assert item.compute('{{ form_var_foo }} [bar]', context={'bar': 'world'}) == 'hello [bar]'

    # django template, no escaping by default
    formdata.data = {'1': '<b>hello</b>'}
    formdata.store()
    assert item.compute('{{ form_var_foo }}') == '<b>hello</b>'  # autoescape off by default
    assert item.compute('{{ form_var_foo|safe }}') == '<b>hello</b>'  # no escaping (implicit |safe)
    assert item.compute('{{ form_var_foo|escape }}') == '&lt;b&gt;hello&lt;/b&gt;'  # escaping


def test_variable_compute_dates(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '2017-07-17'}
    formdata.store()
    pub.substitutions.feed(formdata)

    item = JumpWorkflowStatusItem()

    assert item.compute('=date(form_var_foo)') == datetime.date(2017, 7, 17)
    assert item.compute('=date(form_var_foo) + days(1)') == datetime.date(2017, 7, 18)
    assert item.compute('=date(2017, 7, 18)') == datetime.date(2017, 7, 18)


def test_jump_nothing(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.store()
    formdata = formdef.data_class()()
    item = JumpWorkflowStatusItem()
    assert item.check_condition(formdata) is True


def test_jump_datetime_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.store()
    formdata = formdef.data_class()()
    item = JumpWorkflowStatusItem()
    yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
    item.condition = {
        'type': 'python',
        'value': 'datetime.datetime.now() > datetime.datetime(%s, %s, %s)' % yesterday.timetuple()[:3],
    }
    assert item.check_condition(formdata) is True

    tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
    item.condition = {
        'type': 'python',
        'value': 'datetime.datetime.now() > datetime.datetime(%s, %s, %s)' % tomorrow.timetuple()[:3],
    }
    assert item.check_condition(formdata) is False


def test_jump_date_conditions(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [DateField(id='2', label='Date', type='date', varname='date')]
    formdef.store()

    # create/store/get, to make sure the date format is acceptable
    formdata = formdef.data_class()()
    formdata.data = {'2': DateField().convert_value_from_str('2015-01-04')}
    formdata.store()
    formdata = formdef.data_class().get(formdata.id)

    pub.substitutions.feed(formdata)

    item = JumpWorkflowStatusItem()
    item.condition = {
        'type': 'python',
        'value': 'utils.make_date(form_var_date) == utils.make_date("2015-01-04")',
    }
    assert item.check_condition(formdata) is True

    item = JumpWorkflowStatusItem()
    item.condition = {'type': 'python', 'value': 'utils.time_delta(form_var_date, "2015-01-04").days == 0'}
    assert item.check_condition(formdata) is True

    item = JumpWorkflowStatusItem()
    item.condition = {'type': 'python', 'value': 'utils.time_delta(utils.today(), "2015-01-04").days > 0'}
    assert item.check_condition(formdata) is True

    item = JumpWorkflowStatusItem()
    item.condition = {
        'type': 'python',
        'value': 'utils.time_delta(datetime.datetime.now(), "2015-01-04").days > 0',
    }
    assert item.check_condition(formdata) is True

    item = JumpWorkflowStatusItem()
    item.condition = {
        'type': 'python',
        'value': 'utils.time_delta(utils.time.localtime(), "2015-01-04").days > 0',
    }
    assert item.check_condition(formdata) is True


def test_jump_count_condition(pub):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.store()
    pub.substitutions.feed(formdef)
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    item = JumpWorkflowStatusItem()
    item.condition = {'type': 'python', 'value': 'form_objects.count < 2'}
    assert item.check_condition(formdata) is True

    for _ in range(10):
        formdata = formdef.data_class()()
        formdata.just_created()
        formdata.store()

    item.condition = {'type': 'python', 'value': 'form_objects.count < 2'}
    assert item.check_condition(formdata) is False


def test_jump_bad_python_condition(two_pubs):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.store()
    two_pubs.substitutions.feed(formdef)
    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    item = JumpWorkflowStatusItem()

    item.condition = {'type': 'python', 'value': 'form_var_foobar == 0'}
    assert item.check_condition(formdata) is False
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Failed to evaluate condition'
    assert logged_error.exception_class == 'NameError'
    assert logged_error.exception_message == "name 'form_var_foobar' is not defined"
    assert logged_error.expression == 'form_var_foobar == 0'
    assert logged_error.expression_type == 'python'

    two_pubs.loggederror_class.wipe()
    item.condition = {'type': 'python', 'value': '~ invalid ~'}
    assert item.check_condition(formdata) is False
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Failed to evaluate condition'
    assert logged_error.exception_class == 'SyntaxError'
    assert logged_error.exception_message == 'unexpected EOF while parsing (<string>, line 1)'
    assert logged_error.expression == '~ invalid ~'
    assert logged_error.expression_type == 'python'


def test_jump_django_conditions(two_pubs):
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'1': 'hello'}
    two_pubs.substitutions.feed(formdata)
    item = JumpWorkflowStatusItem()

    item.condition = {'type': 'django', 'value': '1 < 2'}
    assert item.check_condition(formdata) is True

    item.condition = {'type': 'django', 'value': 'form_var_foo == "hello"'}
    assert item.check_condition(formdata) is True

    item.condition = {'type': 'django', 'value': 'form_var_foo|first|upper == "H"'}
    assert item.check_condition(formdata) is True

    item.condition = {'type': 'django', 'value': 'form_var_foo|first|upper == "X"'}
    assert item.check_condition(formdata) is False

    assert two_pubs.loggederror_class.count() == 0

    item.condition = {'type': 'django', 'value': '~ invalid ~'}
    assert item.check_condition(formdata) is False
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Failed to evaluate condition'
    assert logged_error.exception_class == 'TemplateSyntaxError'
    assert logged_error.exception_message == "Could not parse the remainder: '~' from '~'"
    assert logged_error.expression == '~ invalid ~'
    assert logged_error.expression_type == 'django'


def test_check_auth(pub):
    user = pub.user_class(name='foo')
    user.store()

    role = pub.role_class(name='bar1')
    role.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    formdata = formdef.data_class()()

    status_item = WorkflowStatusItem()
    assert status_item.check_auth(formdata, user) is True

    status_item.by = []
    assert status_item.check_auth(formdata, user) is False

    status_item.by = ['logged-users']
    assert status_item.check_auth(formdata, user) is True

    status_item.by = [role.id]
    assert status_item.check_auth(formdata, user) is False
    status_item.by = [int(role.id)]
    assert status_item.check_auth(formdata, user) is False

    user.roles = [role.id]
    status_item.by = [role.id]
    assert status_item.check_auth(formdata, user) is True
    status_item.by = [int(role.id)]
    assert status_item.check_auth(formdata, user) is True

    status_item.by = ['_submitter']
    assert status_item.check_auth(formdata, user) is False
    formdata.user_id = user.id
    assert status_item.check_auth(formdata, user) is True
    formdata.user_id = None

    status_item.by = ['_receiver']
    assert status_item.check_auth(formdata, user) is False
    formdata.workflow_roles = {'_receiver': user.id}
    assert status_item.check_auth(formdata, user) is True
    formdef.workflow_roles = {'_receiver': user.id}
    formdata.workflow_roles = None
    assert status_item.check_auth(formdata, user) is True


def test_dispatch(pub):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    item = DispatchWorkflowStatusItem()

    formdata = formdef.data_class()()
    item.perform(formdata)
    assert not formdata.workflow_roles

    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = role.id
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}


def test_dispatch_multi(pub):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()
    role2 = pub.role_class(name='xxx2')
    role2.store()
    role3 = pub.role_class(name='xxx3')
    role3.store()

    item = DispatchWorkflowStatusItem()

    formdata = formdef.data_class()()
    item.perform(formdata)
    assert not formdata.workflow_roles

    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = role.id
    item.operation_mode = 'add'
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    item.role_id = role2.id
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}

    item.operation_mode = 'set'
    item.role_id = role3.id
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role3.id]}

    # test adding to function defined at the formdef level
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    formdata.workflow_roles = {}
    formdata.store()

    item.operation_mode = 'add'
    item.role_id = role2.id
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}

    # test adding a second time doesn't change anything
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id, role2.id]}

    # test removing
    item.operation_mode = 'remove'
    item.role_id = role2.id
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    # test removing a second time doesn't change anything
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}


def test_dispatch_auto(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    item = DispatchWorkflowStatusItem()
    item.role_key = '_receiver'
    item.dispatch_type = 'automatic'

    formdata = formdef.data_class()()
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    assert not formdata.workflow_roles

    two_pubs.role_class.wipe()
    role1 = two_pubs.role_class('xxx1')
    role1.store()
    role2 = two_pubs.role_class('xxx2')
    role2.store()

    for variable in ('form_var_foo', '{{form_var_foo}}'):
        formdata.data = {}
        formdata.workflow_roles = {}
        item.variable = variable
        item.rules = [
            {'role_id': role1.id, 'value': 'foo'},
            {'role_id': role2.id, 'value': 'bar'},
            {'role_id': role1.id, 'value': '42'},
        ]

        two_pubs.substitutions.reset()
        two_pubs.substitutions.feed(formdata)
        item.perform(formdata)
        assert not formdata.workflow_roles

        # no match
        formdata.data = {'1': 'XXX'}
        two_pubs.substitutions.reset()
        two_pubs.substitutions.feed(formdata)
        item.perform(formdata)
        assert not formdata.workflow_roles

        # match
        formdata.data = {'1': 'foo'}
        two_pubs.substitutions.reset()
        two_pubs.substitutions.feed(formdata)
        item.perform(formdata)
        assert formdata.workflow_roles == {'_receiver': [role1.id]}

        # other match
        formdata.data = {'1': 'bar'}
        two_pubs.substitutions.reset()
        two_pubs.substitutions.feed(formdata)
        item.perform(formdata)
        assert formdata.workflow_roles == {'_receiver': [role2.id]}

        # integer match
        two_pubs.substitutions.reset()
        # cannot store an integer in formdata.data, we mock substitutions:
        two_pubs.substitutions.feed({'form_var_foo': 42})
        item.perform(formdata)
        assert formdata.workflow_roles == {'_receiver': [role1.id]}

    # unknown role
    formdata.data = {'1': 'foo'}
    formdata.workflow_roles = {}
    item.variable = variable
    item.rules = [
        {'role_id': 'foobar', 'value': 'foo'},
    ]
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    assert not formdata.workflow_roles
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar' % formdef.id
    assert error.formdef_id == formdef.id
    assert error.workflow_id == '_default'
    assert error.summary == 'error in dispatch, missing role (foobar)'
    assert error.occurences_count == 1


def test_dispatch_computed(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    two_pubs.role_class.wipe()
    role = two_pubs.role_class(name='xxx')
    role.slug = 'yyy'
    role.store()

    item = DispatchWorkflowStatusItem()

    formdata = formdef.data_class()()
    item.perform(formdata)
    assert not formdata.workflow_roles

    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '="yyy"'  # slug
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '="xxx"'  # name
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    # with templates
    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '{{ "yyy" }}'  # slug
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '{{ "xxx" }}'  # name
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': [role.id]}

    # unknown role
    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '="foobar"'
    item.perform(formdata)
    assert not formdata.workflow_roles
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar' % formdef.id
    assert error.formdef_id == formdef.id
    assert error.workflow_id == '_default'
    assert error.summary == 'error in dispatch, missing role (="foobar")'
    assert error.occurences_count == 1

    # unknown role, with template
    two_pubs.loggederror_class.wipe()
    formdata = formdef.data_class()()
    item.role_key = '_receiver'
    item.role_id = '{{ "foobar" }}'
    item.perform(formdata)
    assert not formdata.workflow_roles
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert (
        error.tech_id == '%s-_default-error-in-dispatch-missing-role-foobar-from-foobar-template' % formdef.id
    )
    assert error.formdef_id == formdef.id
    assert error.workflow_id == '_default'
    assert error.summary == 'error in dispatch, missing role (foobar, from "{{ "foobar" }}" template)'
    assert error.occurences_count == 1


def test_dispatch_user(pub):
    pub.user_class.wipe()
    user = pub.user_class(name='foo')
    user.email = 'foo@localhost'
    user.name_identifiers = ['0123456789']
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    item = DispatchWorkflowStatusItem()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    pub.substitutions.feed(formdata)

    item.role_key = '_receiver'
    item.role_id = '{{ form_user }}'
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}

    formdata.workflow_roles = {}
    item.role_id = '{{ form_user_nameid }}'
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}

    formdata.workflow_roles = {}
    item.role_id = '{{ form_user_email }}'
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}

    formdata.workflow_roles = {}
    item.role_id = '{{ form_user_name }}'
    item.perform(formdata)
    assert formdata.workflow_roles == {'_receiver': ['_user:%s' % user.id]}

    formdata.workflow_roles = {}
    item.role_id = 'xyz'
    item.perform(formdata)
    assert formdata.workflow_roles == {}


def test_roles(two_pubs):
    pub = two_pubs
    pub.role_class.wipe()
    role = pub.role_class(name='xxx')
    role.store()

    user = pub.user_class()
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id

    item = AddRoleWorkflowStatusItem()

    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    item.role_id = str(role.id)
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [str(role.id)]

    # check django template
    user.roles = None
    user.store()
    item.role_id = '{{ "%s" }}' % role.id
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [str(role.id)]

    # check python expression
    user.roles = None
    user.store()
    item.role_id = '="%s"' % role.id
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [str(role.id)]

    # check python expression returning None
    user.roles = None
    user.store()
    item.role_id = '=None'
    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    # tests for remove role action
    user.roles = None
    user.store()
    item = RemoveRoleWorkflowStatusItem()

    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    item.role_id = str(role.id)
    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    user.roles = [str(role.id)]
    user.store()
    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    user.roles = [str(role.id), '2']
    user.store()
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == ['2']


def test_add_remove_computed_roles(pub):
    user = pub.user_class()
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id

    role = pub.role_class(name='plop')
    role.store()
    role2 = pub.role_class(name='xxx')
    role2.store()

    item = AddRoleWorkflowStatusItem()

    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    item.role_id = role.name
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [role.id]

    user.roles = None
    user.store()
    item = RemoveRoleWorkflowStatusItem()

    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    item.role_id = role.name
    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    user.roles = [role.id]
    user.store()
    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles

    user.roles = [role2.id, role.id]
    user.store()
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [role2.id]


def test_roles_idp(pub):
    pub.cfg['sp'] = {'idp-manage-user-attributes': True}
    pub.cfg['idp'] = {'xxx': {'metadata_url': 'http://idp.example.net/idp/saml2/metadata'}}
    pub.write_cfg()
    user = pub.user_class()
    user.name_identifiers = ['xxx']
    user.store()

    role = pub.role_class(name='bar1')
    role.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id

    item = AddRoleWorkflowStatusItem()

    item.perform(formdata)
    assert not pub.user_class.get(user.id).roles
    with mock.patch('wcs.wf.roles.http_post_request') as http_post_request:
        http_post_request.return_value = (None, 201, '', None)
        get_response().process_after_jobs()
        assert http_post_request.call_count == 0

    item.role_id = role.id
    item.perform(formdata)
    assert pub.user_class.get(user.id).roles == [role.id]
    with mock.patch('wcs.wf.roles.http_post_request') as http_post_request:
        http_post_request.return_value = (None, 201, '', None)
        get_response().process_after_jobs()
        assert http_post_request.call_count == 1
        assert http_post_request.call_args[0][0].startswith(
            'http://idp.example.net/api/roles/bar1/members/xxx/'
        )
        assert 'signature=' in http_post_request.call_args[0][0]

    user.roles = None
    user.store()

    item2 = RemoveRoleWorkflowStatusItem()

    item2.perform(formdata)
    assert not pub.user_class.get(user.id).roles
    with mock.patch('wcs.wf.roles.http_delete_request') as http_delete_request:
        http_delete_request.return_value = (None, 200, '', None)
        get_response().process_after_jobs()
        assert http_delete_request.call_count == 0

    item2.role_id = role.id
    user.roles = [role.id]
    user.store()
    item2.perform(formdata)
    assert not pub.user_class.get(user.id).roles
    with mock.patch('wcs.wf.roles.http_delete_request') as http_delete_request:
        http_delete_request.return_value = (None, 200, '', None)
        get_response().process_after_jobs()
        assert http_delete_request.call_count == 1
        assert http_delete_request.call_args[0][0].startswith(
            'http://idp.example.net/api/roles/bar1/members/xxx/'
        )
        assert 'signature=' in http_delete_request.call_args[0][0]

    # out of http request/response cycle
    pub._set_request(None)
    with mock.patch('wcs.wf.roles.http_post_request') as http_post_request:
        http_post_request.return_value = (None, 201, '', None)
        item.perform(formdata)
        assert pub.user_class.get(user.id).roles == [role.id]

    with mock.patch('wcs.wf.roles.http_delete_request') as http_delete_request:
        http_delete_request.return_value = (None, 200, '', None)
        item2.perform(formdata)
        assert pub.user_class.get(user.id).roles == []


def test_anonymise(two_pubs):
    # build a backoffice field
    Workflow.wipe()
    wf = Workflow(name='wf with backoffice field')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string'),
        ItemField(id='bo2', label='list', type='item', items=['bofoo', 'bobar']),
    ]
    wf.add_status('Status1')
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='field 1', type='string'),
        ItemField(id='2', label='list', type='item', items=['abc', 'def']),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.user_id = '1'
    formdata.data = {
        '1': 'foo',
        '2': 'abc',
        '2_display': 'abc',
        'bo1': 'bar',
        'bo2': 'foo',
        'bo2_display': 'foo',
    }
    formdata.workflow_data = {'e': 'mc2'}
    formdata.submission_context = {'foo': 'bar'}
    formdata.store()

    item = AnonymiseWorkflowStatusItem()
    item.perform(formdata)
    assert formdef.data_class().get(formdata.id).user_id is None
    assert formdef.data_class().get(formdata.id).anonymised
    assert formdef.data_class().get(formdata.id).submission_context is None
    assert formdef.data_class().get(formdata.id).data == {
        '1': None,
        '2': 'abc',
        '2_display': 'abc',
        'bo1': None,
        'bo2': 'foo',
        'bo2_display': 'foo',
    }
    assert formdef.data_class().get(formdata.id).workflow_data is None
    assert formdef.data_class().get(formdata.id).evolution[0].who is None


def test_remove(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.store()

    item = RemoveWorkflowStatusItem()
    assert formdef.data_class().count() == 1
    with pytest.raises(AbortActionException) as e:
        item.perform(formdata)
        assert e.url == 'http://example.net'
    assert formdef.data_class().count() == 0

    formdata = formdef.data_class()()
    formdata.store()

    item = RemoveWorkflowStatusItem()
    req = two_pubs.get_request()
    req.response.filter['in_backoffice'] = True
    assert formdef.data_class().count() == 1
    with pytest.raises(AbortActionException) as e:
        item.perform(formdata)
        assert e.url == '..'
    assert formdef.data_class().count() == 0
    req.response.filter = {}
    assert req.session.message


def test_stop_on_remove(two_pubs, emails):
    workflow = Workflow(name='stop-on-remove')
    st1 = workflow.add_status('Status1', 'st1')

    # sendmail + remove + sendmail
    mail1 = st1.add_action('sendmail')
    mail1.to = ['bar@localhost']
    mail1.subject = 'Foobar'
    mail1.body = 'email body'
    st1.add_action('remove')
    mail2 = st1.add_action('sendmail')
    mail2.to = ['bar@localhost']
    mail2.subject = 'Foobar2'
    mail2.body = 'email body 2'

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz%s' % id(two_pubs)
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdef.data_class().wipe()
    emails.empty()
    assert formdef.data_class().count() == 0
    assert emails.count() == 0

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    url = perform_items(st1.items, formdata)
    get_response().process_after_jobs()

    # formdata is removed, no email were sent
    assert formdef.data_class().count() == 0
    assert emails.count() == 1
    assert url == 'http://example.net'

    # check the url from a redirect action is used
    redirect = RedirectToUrlWorkflowStatusItem()
    redirect.url = 'https://www.example.net/custom-redirect'
    st1.items.insert(0, redirect)
    redirect.parent = st1
    workflow.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    url = perform_items(st1.items, formdata)
    get_response().process_after_jobs()
    assert url == redirect.url


def test_register_comment(pub):
    pub.substitutions.feed(MockSubstitutionVariables())

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    item = RegisterCommenterWorkflowStatusItem()
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts() == []

    item.comment = 'Hello world'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>Hello world</p>'

    item.comment = '<div>Hello world</div>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>Hello world</div>'

    formdata.evolution[-1].parts = []
    formdata.store()
    item.comment = '{{ test }}'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts() == []

    item.comment = '[test]'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>[test]</p>'

    item.comment = '{{ bar }}'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>Foobar</div>'

    item.comment = '[bar]'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>Foobar</p>'

    item.comment = '<p>{{ foo }}</p>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 &lt; 3</p>'

    item.comment = '<p>{{ foo|safe }}</p>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 < 3</p>'

    item.comment = '{{ foo }}'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'

    item.comment = '{{ foo|safe }}'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 < 3</div>'

    item.comment = '[foo]'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<p>1 &lt; 3</p>'

    item.comment = '<div>{{ foo }}</div>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'

    item.comment = '<div>[foo]</div>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>1 &lt; 3</div>'


def test_register_comment_django_escaping(pub, emails):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {'1': '<p>hello</p>'}
    formdata.store()
    pub.substitutions.feed(formdata)

    item = RegisterCommenterWorkflowStatusItem()
    item.comment = '<div>{{form_var_foo}}</div>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div>&lt;p&gt;hello&lt;/p&gt;</div>'

    # |safe
    item = RegisterCommenterWorkflowStatusItem()
    item.comment = '<div>{{form_var_foo|safe}}</div>'
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts()[-1] == '<div><p>hello</p></div>'


def test_register_comment_attachment(two_pubs):
    two_pubs.substitutions.feed(MockSubstitutionVariables())

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    item = RegisterCommenterWorkflowStatusItem()
    item.perform(formdata)
    formdata.evolution[-1]._display_parts = None
    assert formdata.evolution[-1].display_parts() == []

    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))

    formdata.evolution[-1].parts = [
        AttachmentEvolutionPart('hello.txt', fp=io.BytesIO(b'hello world'), varname='testfile')
    ]
    formdata.store()
    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
        assert len(subdir) == 4
        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1

    item.comment = '{{ attachments.testfile.url }}'

    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    url1 = formdata.evolution[-1].parts[-1].content
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.kind == 'deprecated_usage'
    assert error.occurences_count == 1

    two_pubs.substitutions.feed(formdata)
    item.comment = '{{ form_attachments.testfile.url }}'
    item.perform(formdata)
    url2 = formdata.evolution[-1].parts[-1].content
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.kind == 'deprecated_usage'
    assert error.occurences_count == 1

    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
        assert len(subdir) == 4
        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
    assert url1 == url2

    # test with a condition
    item.comment = '{% if form_attachments.testfile %}file is there{% endif %}'
    item.perform(formdata)
    assert formdata.evolution[-1].parts[-1].content == '<div>file is there</div>'
    item.comment = '{% if form_attachments.nope %}file is there{% endif %}'
    item.perform(formdata)
    assert formdata.evolution[-1].parts[-1].content == ''

    # test with an action condition
    item.condition = {'type': 'django', 'value': 'form_attachments.testfile'}
    assert item.check_condition(formdata) is True

    item.condition = {'type': 'django', 'value': 'form_attachments.missing'}
    assert item.check_condition(formdata) is False

    two_pubs.substitutions.feed(formdata)
    item.comment = '[attachments.testfile.url]'
    item.perform(formdata)
    url3 = formdata.evolution[-1].parts[-1].content
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.kind == 'deprecated_usage'
    assert error.occurences_count == 2
    two_pubs.substitutions.feed(formdata)
    item.comment = '[form_attachments.testfile.url]'
    item.perform(formdata)
    url4 = formdata.evolution[-1].parts[-1].content
    assert url3 == url4
    assert two_pubs.loggederror_class.count() == 1
    error = two_pubs.loggederror_class.select()[0]
    assert error.kind == 'deprecated_usage'
    assert error.occurences_count == 2


def test_register_comment_with_attachment_file(pub):
    wf = Workflow(name='comment with attachments')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo1', label='bo field 1', type='file', varname='backoffice_file1'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
        upload.receive([fd.read()])

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        FileField(id='1', label='File', type='file', varname='frontoffice_file'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': upload}
    formdata.just_created()
    formdata.store()

    pub.substitutions.feed(formdata)

    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_frontoffice_file_raw'}]
    setbo.perform(formdata)

    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))

    comment_text = 'File is attached to the form history'

    item = RegisterCommenterWorkflowStatusItem()
    item.attachments = ['form_var_backoffice_file1_raw']
    item.comment = comment_text
    item.perform(formdata)

    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
        assert len(subdir) == 4
        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1

    assert len(formdata.evolution[-1].parts) == 2
    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
    assert formdata.evolution[-1].parts[0].orig_filename == upload.orig_filename

    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
    assert len(formdata.evolution[-1].parts[1].content) > 0
    comment_view = str(formdata.evolution[-1].parts[1].view())
    assert comment_view == '<p>%s</p>' % comment_text

    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))

    formdata.evolution[-1].parts = []
    formdata.store()

    ws_response_varname = 'ws_response_afile'
    wf_data = {
        '%s_filename' % ws_response_varname: 'hello.txt',
        '%s_content_type' % ws_response_varname: 'text/plain',
        '%s_b64_content' % ws_response_varname: base64.encodebytes(b'hello world'),
    }
    formdata.update_workflow_data(wf_data)
    formdata.store()
    assert hasattr(formdata, 'workflow_data')
    assert isinstance(formdata.workflow_data, dict)

    item = RegisterCommenterWorkflowStatusItem()
    item.attachments = ["utils.dict_from_prefix('%s_', locals())" % ws_response_varname]
    item.comment = comment_text
    item.perform(formdata)

    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
        assert len(subdir) == 4
        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1

    assert len(formdata.evolution[-1].parts) == 2
    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
    assert formdata.evolution[-1].parts[0].orig_filename == 'hello.txt'

    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
    assert len(formdata.evolution[-1].parts[1].content) > 0
    comment_view = str(formdata.evolution[-1].parts[1].view())
    assert comment_view == '<p>%s</p>' % comment_text


def test_register_comment_to(pub):
    workflow = Workflow(name='register comment to')
    st1 = workflow.add_status('Status1', 'st1')

    role = pub.role_class(name='foorole')
    role.store()
    role2 = pub.role_class(name='no-one-role')
    role2.store()
    user = pub.user_class(name='baruser')
    user.roles = []
    user.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef._workflow = workflow
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    assert formdata.status == 'wf-st1'
    formdata.store()

    register_commenter = st1.add_action('register-comment')

    def display_parts():
        formdata.evolution[-1]._display_parts = None  # invalidate cache
        return [str(x) for x in formdata.evolution[-1].display_parts()]

    register_commenter.comment = 'all'
    register_commenter.to = None
    register_commenter.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 1
    assert display_parts() == ['<p>all</p>']

    register_commenter.comment = 'to-role'
    register_commenter.to = [role.id]
    register_commenter.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 2
    assert len(display_parts()) == 1
    pub._request._user = user
    assert display_parts() == ['<p>all</p>']
    user.roles = [role.id]
    assert display_parts() == ['<p>all</p>', '<p>to-role</p>']

    user.roles = []
    register_commenter.comment = 'to-submitter'
    register_commenter.to = ['_submitter']
    register_commenter.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 3
    assert display_parts() == ['<p>all</p>']
    formdata.user_id = user.id
    assert display_parts() == ['<p>all</p>', '<p>to-submitter</p>']

    register_commenter.comment = 'to-role-or-submitter'
    register_commenter.to = [role.id, '_submitter']
    register_commenter.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 4
    assert display_parts() == ['<p>all</p>', '<p>to-submitter</p>', '<p>to-role-or-submitter</p>']
    formdata.user_id = None
    assert display_parts() == ['<p>all</p>']
    user.roles = [role.id]
    assert display_parts() == ['<p>all</p>', '<p>to-role</p>', '<p>to-role-or-submitter</p>']
    formdata.user_id = user.id
    assert display_parts() == [
        '<p>all</p>',
        '<p>to-role</p>',
        '<p>to-submitter</p>',
        '<p>to-role-or-submitter</p>',
    ]

    register_commenter.comment = 'd1'
    register_commenter.to = [role2.id]
    register_commenter.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 5
    assert display_parts() == [
        '<p>all</p>',
        '<p>to-role</p>',
        '<p>to-submitter</p>',
        '<p>to-role-or-submitter</p>',
    ]
    register_commenter2 = st1.add_action('register-comment')
    register_commenter2.comment = 'd2'
    register_commenter2.to = [role.id, '_submitter']
    user.roles = [role.id, role2.id]
    register_commenter2.perform(formdata)
    assert len(formdata.evolution[-1].parts) == 6
    assert '<p>d1</p>' in [str(x) for x in display_parts()]
    assert '<p>d2</p>' in [str(x) for x in display_parts()]


def test_register_comment_to_with_attachment(pub):
    workflow = Workflow(name='register comment to with attachment')
    st1 = workflow.add_status('Status1', 'st1')

    role = pub.role_class(name='foorole')
    role.store()
    role2 = pub.role_class(name='no-one-role')
    role2.store()
    user = pub.user_class(name='baruser')
    user.roles = []
    user.store()

    upload1 = PicklableUpload('all.txt', 'text/plain')
    upload1.receive([b'barfoo'])
    upload2 = PicklableUpload('to-role.txt', 'text/plain')
    upload2.receive([b'barfoo'])
    upload3 = PicklableUpload('to-submitter.txt', 'text/plain')
    upload3.receive([b'barfoo'])
    upload4 = PicklableUpload('to-role-or-submitter.txt', 'text/plain')
    upload4.receive([b'barfoo'])

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.fields = [
        FileField(id='1', label='File1', type='file', varname='file1'),
        FileField(id='2', label='File2', type='file', varname='file2'),
        FileField(id='3', label='File3', type='file', varname='file3'),
        FileField(id='4', label='File4', type='file', varname='file4'),
    ]
    formdef._workflow = workflow
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': upload1, '2': upload2, '3': upload3, '4': upload4}
    formdata.just_created()
    assert formdata.status == 'wf-st1'
    pub.substitutions.feed(formdata)

    register_commenter = st1.add_action('register-comment')

    def display_parts():
        formdata.evolution[-1]._display_parts = None  # invalidate cache
        return [str(x) for x in formdata.evolution[-1].display_parts()]

    register_commenter.comment = 'all'
    register_commenter.attachments = ['form_var_file1_raw']
    register_commenter.to = None
    register_commenter.perform(formdata)

    register_commenter.comment = 'to-role'
    register_commenter.attachments = ['form_var_file2_raw']
    register_commenter.to = [role.id]
    register_commenter.perform(formdata)

    register_commenter.comment = 'to-submitter'
    register_commenter.attachments = ['form_var_file3_raw']
    register_commenter.to = ['_submitter']
    register_commenter.perform(formdata)

    register_commenter.comment = 'to-role-or-submitter'
    register_commenter.attachments = ['form_var_file4_raw']
    register_commenter.to = [role.id, '_submitter']
    register_commenter.perform(formdata)

    assert len(formdata.evolution[-1].parts) == 8

    assert user.roles == []
    assert len(display_parts()) == 2
    assert 'all.txt' in display_parts()[0]
    assert display_parts()[1] == '<p>all</p>'

    pub._request._user = user
    user.roles = [role.id]
    assert len(display_parts()) == 6
    assert 'all.txt' in display_parts()[0]
    assert 'to-role.txt' in display_parts()[2]
    assert 'to-role-or-submitter.txt' in display_parts()[4]

    user.roles = []
    formdata.user_id = user.id
    assert len(display_parts()) == 6
    assert 'all.txt' in display_parts()[0]
    assert 'to-submitter.txt' in display_parts()[2]
    assert 'to-role-or-submitter.txt' in display_parts()[4]

    user.roles = [role.id]
    assert len(display_parts()) == 8
    assert 'all.txt' in display_parts()[0]
    assert 'to-role.txt' in display_parts()[2]
    assert 'to-submitter.txt' in display_parts()[4]
    assert 'to-role-or-submitter.txt' in display_parts()[6]


def test_webservice_call(http_requests, two_pubs):
    pub = two_pubs
    pub.substitutions.feed(MockSubstitutionVariables())

    wf = Workflow(name='wf1')
    st1 = wf.add_status('Status1', 'st1')
    wf.add_status('StatusErr', 'sterr')
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'GET'

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.post = True
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'POST'
    payload = json.loads(http_requests.get_last('body'))
    assert payload['url'] == 'http://example.net/baz/%s/' % formdata.id
    assert payload['display_id'] == formdata.get_display_id()

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.post_data = {
        'str': 'abcd',
        'one': '=1',
        'django': '{{ form_number }}',
        'evalme': '=form_number',
        'error': '=1=3',
    }
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'POST'
    payload = json.loads(http_requests.get_last('body'))
    assert payload == {
        'one': 1,
        'str': 'abcd',
        'evalme': formdata.get_display_id(),
        'django': formdata.get_display_id(),
    }

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.post = True
    item.post_data = {
        'str': 'abcd',
        'one': '=1',
        'decimal': '=Decimal(2)',
        'evalme': '=form_number',
        'error': '=1=3',
    }
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'POST'
    payload = json.loads(http_requests.get_last('body'))
    assert payload['extra'] == {'one': 1, 'str': 'abcd', 'decimal': '2', 'evalme': formdata.get_display_id()}
    assert payload['url'] == 'http://example.net/baz/%s/' % formdata.id
    assert payload['display_id'] == formdata.get_display_id()

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_response'] == {'foo': 'bar'}
    assert formdata.workflow_data.get('xxx_time')

    get_publisher().substitutions.reset()
    get_publisher().substitutions.feed(formdata)
    substvars = get_publisher().substitutions.get_context_variables(mode='lazy')
    assert str(substvars['xxx_status']) == '200'
    assert 'xxx_status' in substvars.get_flat_keys()
    assert str(substvars['xxx_response_foo']) == 'bar'
    assert 'xxx_response_foo' in substvars.get_flat_keys()

    pub.substitutions.reset()
    pub.substitutions.feed(MockSubstitutionVariables())

    formdata.workflow_data = None
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = 'xxx'
    item.perform(formdata)
    assert 'signature=' in http_requests.get_last('url')
    assert http_requests.get_last('method') == 'GET'

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = '{{ doesntexist }}'
    item.perform(formdata)
    assert 'signature=' not in http_requests.get_last('url')

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = '{{ empty }}'
    item.perform(formdata)
    assert 'signature=' not in http_requests.get_last('url')

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = '[empty]'
    item.perform(formdata)
    assert 'signature=' not in http_requests.get_last('url')

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = '{{ bar }}'
    item.perform(formdata)
    assert 'signature=' in http_requests.get_last('url')

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.request_signature_key = '[bar]'
    item.perform(formdata)
    assert 'signature=' in http_requests.get_last('url')

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/204'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 204
    assert formdata.workflow_data.get('xxx_time')
    assert 'xxx_response' not in formdata.workflow_data
    assert 'xxx_error_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/404'
    item.varname = 'xxx'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 404
    assert formdata.workflow_data.get('xxx_time')
    assert 'xxx_error_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/404-json'
    item.varname = 'xxx'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 404
    assert formdata.workflow_data.get('xxx_error_response') == {'err': 1}
    assert formdata.workflow_data.get('xxx_time')
    assert 'xxx_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/404'
    item.action_on_4xx = ':pass'
    item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/500'
    with pytest.raises(AbortActionException):
        item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/500'
    item.action_on_5xx = ':pass'
    item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.parent = st1
    assert item.get_jump_label(st1.id) == 'Webservice'
    assert item.get_jump_label('sterr') == 'Error calling webservice'
    item.label = 'Plop'
    assert item.get_jump_label(st1.id) == 'Webservice "Plop"'
    assert item.get_jump_label('sterr') == 'Error calling webservice "Plop"'
    item.url = 'http://remote.example.net/500'
    item.action_on_5xx = 'sterr'  # jump to status
    formdata.status = 'wf-st1'
    formdata.store()
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.status == 'wf-sterr'

    pub.loggederror_class.wipe()
    item.action_on_5xx = 'stdeleted'  # removed status
    formdata.status = 'wf-st1'
    formdata.store()
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.status == 'wf-st1'  # unknown status acts like :stop
    assert pub.loggederror_class.count() == 1
    error = pub.loggederror_class.select()[0]
    assert 'reference-to-invalid-status-stdeleted-in-workflow' in error.tech_id
    assert error.occurences_count == 1

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.varname = 'xxx'
    item.action_on_bad_data = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 200
    assert 'xxx_response' not in formdata.workflow_data
    assert 'xxx_error_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/404'
    item.record_errors = True
    item.action_on_4xx = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.evolution[-1].parts[-1].summary == '404 whatever'

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.varname = 'xxx'
    item.action_on_bad_data = ':stop'
    item.record_errors = True
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert (
        formdata.evolution[-1].parts[-1].summary
        == 'json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)\n'
    )
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_time')
    assert 'xxx_error_response' not in formdata.workflow_data
    formdata.workflow_data = None

    # check storing response as attachment
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.varname = 'xxx'
    item.response_type = 'attachment'
    item.record_errors = True
    item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_content_type') == 'text/xml'
    attachment = formdata.evolution[-1].parts[-1]
    assert isinstance(attachment, AttachmentEvolutionPart)
    assert attachment.base_filename == 'xxx.xml'
    assert attachment.content_type == 'text/xml'
    attachment.fp.seek(0)
    assert attachment.fp.read(5) == b'<?xml'
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/400-json'
    item.record_errors = True
    item.action_on_4xx = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.evolution[-1].parts[-1].is_hidden()  # not displayed in front
    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': '/backoffice/'})
    pub._set_request(req)
    assert not formdata.evolution[-1].parts[-1].is_hidden()
    rendered = formdata.evolution[-1].parts[-1].view()
    assert 'Error during webservice call' in str(rendered)
    assert 'Error Code: 1' in str(rendered)
    assert 'Error Description: :(' in str(rendered)

    item.label = 'do that'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    rendered = formdata.evolution[-1].parts[-1].view()
    assert 'Error during webservice call &quot;do that&quot;' in str(rendered)

    item = WebserviceCallStatusItem()
    item.method = 'GET'
    item.url = 'http://remote.example.net?in_url=1'
    item.qs_data = {
        'str': 'abcd',
        'one': '=1',
        'evalme': '=form_number',
        'django': '{{ form_number }}',
        'ezt': '[form_number]',
        'error': '=1=3',
        'in_url': '2',
    }
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('method') == 'GET'
    qs = urllib.parse.parse_qs(http_requests.get_last('url').split('?')[1])
    assert set(qs.keys()) == {'in_url', 'str', 'one', 'evalme', 'django', 'ezt'}
    assert qs['in_url'] == ['1', '2']
    assert qs['one'] == ['1']
    assert qs['evalme'] == [formdata.get_display_id()]
    assert qs['django'] == [formdata.get_display_id()]
    assert qs['ezt'] == [formdata.get_display_id()]
    assert qs['str'] == ['abcd']

    item = WebserviceCallStatusItem()
    item.method = 'DELETE'
    item.post = False
    item.url = 'http://remote.example.net/json'
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('method') == 'DELETE'

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.method = 'PUT'
    item.post = False
    item.post_data = {'str': 'abcd', 'one': '=1', 'evalme': '=form_number', 'error': '=1=3'}
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'PUT'
    payload = json.loads(http_requests.get_last('body'))
    assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net'
    item.method = 'PATCH'
    item.post = False
    item.post_data = {'str': 'abcd', 'one': '=1', 'evalme': '=form_number', 'error': '=1=3'}
    pub.substitutions.feed(formdata)
    item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'PATCH'
    payload = json.loads(http_requests.get_last('body'))
    assert payload == {'one': 1, 'str': 'abcd', 'evalme': formdata.get_display_id()}


def test_webservice_waitpoint(pub):
    item = WebserviceCallStatusItem()
    assert item.waitpoint
    item.action_on_app_error = ':pass'
    item.action_on_4xx = ':pass'
    item.action_on_5xx = ':pass'
    item.action_on_bad_data = ':pass'
    item.action_on_network_errors = ':pass'
    assert not item.waitpoint
    item.action_on_network_errors = ':stop'
    assert item.waitpoint


def test_webservice_call_error_handling(http_requests, pub):
    pub.substitutions.feed(MockSubstitutionVariables())

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err1'
    item.action_on_app_error = ':stop'
    item.action_on_4xx = ':pass'
    item.action_on_5xx = ':pass'
    item.action_on_network_errors = ':pass'
    with pytest.raises(AbortActionException):
        item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheader1'
    item.action_on_app_error = ':stop'
    item.action_on_4xx = ':pass'
    item.action_on_5xx = ':pass'
    item.action_on_network_errors = ':pass'
    with pytest.raises(AbortActionException):
        item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheaderstr'
    item.action_on_app_error = ':stop'
    item.action_on_4xx = ':pass'
    item.action_on_5xx = ':pass'
    item.action_on_network_errors = ':pass'
    with pytest.raises(AbortActionException):
        item.perform(formdata)

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err0'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 0
    assert formdata.workflow_data['xxx_response'] == {'data': 'foo', 'err': 0}
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err0'
    item.varname = 'xxx'
    item.action_on_app_error = ':stop'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 0
    assert formdata.workflow_data['xxx_response'] == {'data': 'foo', 'err': 0}
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err1'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 1
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errstr'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 'bug'
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err1'
    item.varname = 'xxx'
    item.action_on_app_error = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 1
    assert formdata.workflow_data['xxx_error_response'] == {'data': '', 'err': 1}
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheader0'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 0
    assert formdata.workflow_data['xxx_app_error_header'] == '0'
    assert formdata.workflow_data['xxx_response'] == {'foo': 'bar'}
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheader1'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 1
    assert formdata.workflow_data['xxx_app_error_header'] == '1'
    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheaderstr'
    item.varname = 'xxx'
    item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 'bug'
    assert formdata.workflow_data['xxx_app_error_header'] == 'bug'
    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheader1'
    item.varname = 'xxx'
    item.action_on_app_error = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data['xxx_status'] == 200
    assert formdata.workflow_data['xxx_app_error_code'] == 1
    assert formdata.workflow_data['xxx_app_error_header'] == '1'
    assert formdata.workflow_data['xxx_error_response'] == {'foo': 'bar'}
    assert 'xxx_response' not in formdata.workflow_data
    assert formdata.workflow_data.get('xxx_time')
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml-errheader'
    item.varname = 'xxx'
    item.response_type = 'attachment'
    item.record_errors = True
    item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_app_error_code') == 1
    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
    assert 'xxx_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml-errheader'
    item.varname = 'xxx'
    item.response_type = 'attachment'
    item.record_errors = True
    item.action_on_app_error = ':stop'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_app_error_code') == 1
    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
    assert 'xxx_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml-errheader'
    item.varname = 'xxx'
    item.response_type = 'json'  # wait for json but receive xml
    item.record_errors = True
    item.perform(formdata)
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_app_error_code') == 1
    assert formdata.workflow_data.get('xxx_app_error_header') == '1'
    assert 'xxx_response' not in formdata.workflow_data
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-err1'
    item.action_on_app_error = ':stop'
    item.response_type = 'attachment'  # err value is not an error
    item.perform(formdata)  # so, everything is "ok" here
    formdata.workflow_data = None

    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/json-errheaderstr'
    item.action_on_app_error = ':stop'
    item.response_type = 'attachment'
    with pytest.raises(AbortActionException):
        item.perform(formdata)
    formdata.workflow_data = None

    # xml instead of json is not a app_error
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.varname = 'xxx'
    item.action_on_app_error = ':stop'
    item.action_on_4xx = ':pass'
    item.action_on_5xx = ':pass'
    item.action_on_network_errors = ':pass'
    item.action_on_bad_data = ':pass'
    item.perform(formdata)
    formdata.workflow_data = None

    # connection error
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/connection-error'
    item.record_errors = True
    item.action_on_network_errors = ':pass'
    item.perform(formdata)
    assert not formdata.workflow_data

    # connection error, with varname
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/connection-error'
    item.varname = 'plop'
    item.record_errors = True
    item.action_on_network_errors = ':pass'
    item.perform(formdata)
    assert 'ConnectionError: error\n' in formdata.evolution[-1].parts[-1].summary
    assert formdata.workflow_data['plop_connection_error'] == 'error'


def test_webservice_call_store_in_backoffice_filefield(http_requests, pub):
    wf = Workflow(name='wscall to backoffice file field')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo1', label='bo field 1', type='file', varname='backoffice_file1'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {}
    formdata.store()

    # check storing response in backoffice file field
    item = WebserviceCallStatusItem()
    item.parent = st1
    item.backoffice_filefield_id = 'bo1'
    item.url = 'http://remote.example.net/xml'
    item.response_type = 'attachment'
    item.record_errors = True
    item.perform(formdata)

    assert 'bo1' in formdata.data
    fbo1 = formdata.data['bo1']
    assert fbo1.base_filename == 'file-bo1.xml'
    assert fbo1.content_type == 'text/xml'
    assert fbo1.get_content().startswith(b'<?xml')
    # nothing else is stored
    assert formdata.workflow_data is None
    assert not formdata.evolution[-1].parts

    # store in backoffice file field + varname
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    item.varname = 'xxx'
    item.perform(formdata)
    # backoffice file field
    assert 'bo1' in formdata.data
    fbo1 = formdata.data['bo1']
    assert fbo1.base_filename == 'xxx.xml'
    assert fbo1.content_type == 'text/xml'
    assert fbo1.get_content().startswith(b'<?xml')
    # varname => workflow_data and AttachmentEvolutionPart
    assert formdata.workflow_data.get('xxx_status') == 200
    assert formdata.workflow_data.get('xxx_content_type') == 'text/xml'
    attachment = formdata.evolution[-1].parts[-1]
    assert isinstance(attachment, AttachmentEvolutionPart)
    assert attachment.base_filename == 'xxx.xml'
    assert attachment.content_type == 'text/xml'
    attachment.fp.seek(0)
    assert attachment.fp.read(5) == b'<?xml'

    # no more 'bo1' backoffice field: do nothing
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo2', label='bo field 2', type='file'),  # id != 'bo1'
    ]
    item.perform(formdata)
    assert formdata.data == {}
    # backoffice field is not a field file:
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string'),
    ]
    item.perform(formdata)
    assert formdata.data == {}
    # no field at all:
    wf.backoffice_fields_formdef.fields = []
    item.perform(formdata)
    assert formdata.data == {}


def test_webservice_target_status(pub):
    wf = Workflow(name='boo')
    status1 = wf.add_status('Status1', 'st1')
    status2 = wf.add_status('Status2', 'st2')
    wf.store()

    item = WebserviceCallStatusItem()
    item.parent = status1
    assert item.get_target_status() == [status1.id]

    item.action_on_app_error = status1.id
    item.action_on_4xx = status2.id
    item.action_on_5xx = status2.id
    targets = item.get_target_status()
    assert len(item.get_target_status()) == 4
    assert targets.count(status1) == 2
    assert targets.count(status2) == 2

    item.action_on_bad_data = 'st3'  # doesn't exist
    targets = item.get_target_status()
    assert len(item.get_target_status()) == 4
    assert targets.count(status1) == 2
    assert targets.count(status2) == 2


def test_webservice_with_complex_data(http_requests, pub):
    pub.substitutions.feed(MockSubstitutionVariables())

    wf = Workflow(name='wf1')
    wf.add_status('Status1', 'st1')
    wf.add_status('StatusErr', 'sterr')
    wf.store()

    datasource = {
        'type': 'formula',
        'value': repr(
            [
                {'id': 'a', 'text': 'aa', 'more': 'aaa'},
                {'id': 'b', 'text': 'bb', 'more': 'bbb'},
                {'id': 'c', 'text': 'cc', 'more': 'ccc'},
            ]
        ),
    }

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        ItemField(id='1', label='1st field', type='item', varname='item', data_source=datasource),
        ItemsField(id='2', label='2nd field', type='items', varname='items', data_source=datasource),
        StringField(id='3', label='3rd field', type='str', varname='str'),
        StringField(id='4', label='4th field', type='str', varname='empty_str'),
        StringField(id='5', label='5th field', type='str', varname='none'),
        BoolField(id='6', label='6th field', type='bool', varname='bool'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.data['1'] = 'a'
    formdata.data['1_display'] = 'aa'
    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.data['2'] = ['a', 'b']
    formdata.data['2_display'] = 'aa, bb'
    formdata.data['2_structured'] = formdef.fields[1].store_structured_value(formdata.data, '2')
    formdata.data['3'] = 'tutuche'
    formdata.data['4'] = 'empty_str'
    formdata.data['5'] = None
    formdata.data['6'] = False
    formdata.just_created()
    formdata.store()

    item = WebserviceCallStatusItem()
    item.method = 'POST'
    item.url = 'http://remote.example.net'
    item.post_data = {
        'item': '{{ form_var_item }}',
        'ezt_item': '[form_var_item]',
        'items': '{{ form_var_items }}',
        'ezt_items': '[form_var_items]',
        'item_raw': '{{ form_var_item_raw }}',
        'ezt_item_raw': '[form_var_item_raw]',
        'items_raw': '{{ form_var_items_raw }}',
        'with_items_raw': '{% with x=form_var_items_raw %}{{ x }}{% endwith %}',
        'with_items_upper': '{% with x=form_var_items_raw %}{{ x.1|upper }}{% endwith %}',
        'ezt_items_raw': '[form_var_items_raw]',
        'joined_items_raw': '{{ form_var_items_raw|join:"|" }}',
        'forloop_items_raw': '{% for item in form_var_items_raw %}{{item}}|{% endfor %}',
        'str': '{{ form_var_str }}',
        'str_mod': '{{ form_var_str }}--plop',
        'decimal': '{{ "1000"|decimal }}',
        'empty_string': '{{ form_var_empty }}',
        'none': '{{ form_var_none }}',
        'bool': '{{ form_var_bool_raw }}',
    }
    pub.substitutions.feed(formdata)
    with get_publisher().complex_data():
        item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'POST'
    payload = json.loads(http_requests.get_last('body'))
    assert payload == {
        'item': 'aa',
        'ezt_item': 'aa',
        'items': 'aa, bb',
        'ezt_items': 'aa, bb',
        'item_raw': 'a',
        'ezt_item_raw': 'a',
        'items_raw': ['a', 'b'],
        'with_items_raw': ['a', 'b'],
        'with_items_upper': 'B',
        'ezt_items_raw': repr(['a', 'b']),
        'joined_items_raw': 'a|b',
        'forloop_items_raw': 'a|b|',
        'str': 'tutuche',
        'str_mod': 'tutuche--plop',
        'decimal': '1000',
        'empty_string': '',
        'none': None,
        'bool': False,
    }

    # check an empty boolean field is sent as False
    del formdata.data['6']
    with get_publisher().complex_data():
        item.perform(formdata)
    assert http_requests.get_last('url') == 'http://remote.example.net'
    assert http_requests.get_last('method') == 'POST'
    payload = json.loads(http_requests.get_last('body'))
    assert payload['bool'] is False


def rewind(formdata, seconds):
    # utility function to move formdata back in time
    def rewind_time(timetuple):
        return time.localtime(datetime.datetime.fromtimestamp(time.mktime(timetuple) - seconds).timestamp())

    formdata.receipt_time = rewind_time(formdata.receipt_time)
    formdata.evolution[-1].time = rewind_time(formdata.evolution[-1].time)


def test_timeout(two_pubs):
    workflow = Workflow(name='timeout')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.add_status('Status2', 'st2')

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = 30 * 60  # 30 minutes
    jump.status = 'st2'

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    rewind(formdata, seconds=40 * 60)
    formdata.store()
    formdata_id = formdata.id

    _apply_timeouts(two_pubs)

    assert formdef.data_class().get(formdata_id).status == 'wf-st2'

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    formdata_id = formdata.id
    with mock.patch('wcs.wf.jump.JumpWorkflowStatusItem.check_condition') as must_jump:
        must_jump.return_value = False
        _apply_timeouts(two_pubs)
        assert must_jump.call_count == 0  # not enough time has passed

        # check a lower than minimal delay is not considered
        jump.timeout = 5 * 50  # 5 minutes
        workflow.store()
        rewind(formdata, seconds=10 * 60)
        formdata.store()
        _apply_timeouts(two_pubs)
        assert must_jump.call_count == 0

        # but is executed once delay is reached
        rewind(formdata, seconds=10 * 60)
        formdata.store()
        _apply_timeouts(two_pubs)
        assert must_jump.call_count == 1

        # check a templated timeout is considered as minimal delay for explicit evaluation
        jump.timeout = '{{ "0" }}'
        workflow.store()
        _apply_timeouts(two_pubs)
        assert must_jump.call_count == 2

    # check there's no crash on workflow without jumps
    formdef = FormDef()
    formdef.name = 'xxx'
    formdef.store()
    _apply_timeouts(two_pubs)


def test_timeout_with_humantime_template(two_pubs):
    workflow = Workflow(name='timeout')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.add_status('Status2', 'st2')

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = '{{ 30 }} minutes'
    jump.status = 'st2'

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    formdata_id = formdata.id

    _apply_timeouts(two_pubs)
    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change

    rewind(formdata, seconds=40 * 60)
    formdata.store()
    _apply_timeouts(two_pubs)
    assert formdef.data_class().get(formdata_id).status == 'wf-st2'

    # invalid timeout value
    jump.timeout = '{{ 30 }} plop'
    workflow.store()
    formdef.refresh_from_storage()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    formdata_id = formdata.id

    two_pubs.loggederror_class.wipe()

    rewind(formdata, seconds=40 * 60)
    formdata.store()
    _apply_timeouts(two_pubs)
    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change

    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == "Error in timeout value '30 plop' (computed from '{{ 30 }} plop')"

    # template timeout value returning nothing
    jump.timeout = '{% if 1 %}{% endif %}'
    workflow.store()
    formdef.refresh_from_storage()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    formdata_id = formdata.id

    two_pubs.loggederror_class.wipe()

    rewind(formdata, seconds=40 * 60)
    formdata.store()
    _apply_timeouts(two_pubs)
    assert formdef.data_class().get(formdata_id).status == 'wf-st1'  # no change

    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == "Error in timeout value '' (computed from '{% if 1 %}{% endif %}')"


def test_legacy_timeout(pub):
    workflow = Workflow(name='timeout')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.add_status('Status2', 'st2')

    jump = st1.add_action('timeout', id='_jump')
    jump.timeout = 30 * 60  # 30 minutes
    jump.status = 'st2'

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    rewind(formdata, seconds=40 * 60)
    formdata.store()
    formdata_id = formdata.id

    _apply_timeouts(pub)

    assert formdef.data_class().get(formdata_id).status == 'wf-st2'


def test_timeout_then_remove(two_pubs):
    workflow = Workflow(name='timeout-then-remove')
    st1 = workflow.add_status('Status1', 'st1')
    st2 = workflow.add_status('Status2', 'st2')

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = 30 * 60  # 30 minutes
    jump.status = 'st2'

    st2.add_action('remove')

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz%s' % id(two_pubs)
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    rewind(formdata, seconds=40 * 60)
    formdata.store()
    formdata_id = formdata.id

    assert str(formdata_id) in [str(x) for x in formdef.data_class().keys()]

    _apply_timeouts(two_pubs)

    assert not str(formdata_id) in [str(x) for x in formdef.data_class().keys()]


def test_timeout_with_mark(two_pubs):
    workflow = Workflow(name='timeout')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.add_status('Status2', 'st2')

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.timeout = 30 * 60  # 30 minutes
    jump.status = 'st2'
    jump.set_marker_on_status = True

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    assert formdef.get_workflow().id == workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    rewind(formdata, seconds=40 * 60)
    formdata.store()
    formdata_id = formdata.id

    _apply_timeouts(two_pubs)

    formdata = formdef.data_class().get(formdata_id)
    assert formdata.workflow_data.get('_markers_stack') == [{'status_id': 'st1'}]


def test_jump_missing_previous_mark(two_pubs):
    FormDef.wipe()
    Workflow.wipe()

    workflow = Workflow(name='jump-mark')
    st1 = workflow.add_status('Status1', 'st1')

    jump = st1.add_action('jump', id='_jump')
    jump.by = ['_submitter', '_receiver']
    jump.status = '_previous'
    jump.timeout = 30 * 60  # 30 minutes

    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    rewind(formdata, seconds=40 * 60)
    formdata.store()

    two_pubs.loggederror_class.wipe()
    _apply_timeouts(two_pubs)
    assert two_pubs.loggederror_class.count() == 1


def test_sms(pub, sms_mocking):
    pub.cfg['sms'] = {'sender': 'xxx', 'passerelle_url': 'http://passerelle.invalid/'}
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    item = SendSMSWorkflowStatusItem()
    # no recipients
    item.perform(formdata)
    assert len(sms_mocking.sms) == 0
    # body
    item.to = ['000']
    assert len(sms_mocking.sms) == 0
    # action configured
    item.body = 'XXX'
    item.perform(formdata)
    assert len(sms_mocking.sms) == 1
    assert sms_mocking.sms[0]['destinations'] == ['000']
    assert sms_mocking.sms[0]['text'] == 'XXX'

    # check None as recipient is not passed to the SMS backend
    item.to = [None]
    item.perform(formdata)  # nothing
    assert len(sms_mocking.sms) == 1

    item.to = ['000', None]
    item.perform(formdata)
    assert len(sms_mocking.sms) == 2
    assert sms_mocking.sms[1]['destinations'] == ['000']
    assert sms_mocking.sms[1]['text'] == 'XXX'

    pub.substitutions.feed(MockSubstitutionVariables())
    item.body = 'dj{{ bar }}'
    item.perform(formdata)
    assert len(sms_mocking.sms) == 3
    assert sms_mocking.sms[2]['destinations'] == ['000']
    assert sms_mocking.sms[2]['text'] == 'djFoobar'

    item.body = 'ezt[bar]'
    item.perform(formdata)
    assert len(sms_mocking.sms) == 4
    assert sms_mocking.sms[3]['destinations'] == ['000']
    assert sms_mocking.sms[3]['text'] == 'eztFoobar'

    # invalid recipient
    sms_mocking.empty()
    item.to = ['{% xx']
    item.body = 'xxx'
    item.perform(formdata)
    assert len(sms_mocking.sms) == 0

    # invalid body
    sms_mocking.empty()
    item.to = ['000']
    item.body = '{% xx'
    item.perform(formdata)
    assert len(sms_mocking.sms) == 0

    # disable SMS system
    sms_mocking.empty()
    pub.cfg['sms'] = {'mode': 'none'}
    item.to = ['000']
    item.body = 'XXX'
    item.perform(formdata)  # nothing
    assert len(sms_mocking.sms) == 0


def test_sms_many_recipients(pub, sms_mocking):
    pub.cfg['sms'] = {'sender': 'xxx', 'passerelle_url': 'http://passerelle.invalid/'}

    formdef = FormDef()
    formdef.name = 'foo'
    formdef.fields = [StringField(id='1', label='number', varname='number')]
    formdef.store()

    formdef.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.data = {'1': '000'}
    formdata.just_created()
    formdata.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '001'}
    formdata.just_created()
    formdata.store()

    item = SendSMSWorkflowStatusItem()
    item.body = 'XXX'

    for recipient in [
        '000,001',
        '000,001,',
        '{% for obj in forms|objects:"foo" %}{{ obj|get:"form_var_number" }},{% endfor %}',
        '{{ forms|objects:"foo"|getlist:"form_var_number" }}',
        '{{ forms|objects:"foo"|getlist:"form_var_number"|list }}',
    ]:
        item.to = [recipient]
        sms_mocking.empty()
        item.perform(formdata)
        get_response().process_after_jobs()
        assert len(sms_mocking.sms) == 1
        assert set(sms_mocking.sms[0]['destinations']) == {'000', '001'}
        assert sms_mocking.sms[0]['text'] == 'XXX'


def test_sms_with_passerelle(two_pubs):
    pub = two_pubs
    pub.cfg['sms'] = {
        'mode': 'passerelle',
        'passerelle_url': 'http://passerelle.example.com/send?nostop=1',
        'sender': 'Passerelle',
    }
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='quotes'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data = {'1': 'with "quotes"'}
    formdata.store()
    pub.substitutions.feed(formdata)

    item = SendSMSWorkflowStatusItem()
    item.to = ['1234']
    item.body = 'my "message" {{ form_var_quotes }}'
    with mock.patch('wcs.wscalls.get_secret_and_orig') as mocked_secret_and_orig:
        mocked_secret_and_orig.return_value = ('secret', 'localhost')
        with mock.patch('wcs.qommon.misc._http_request') as mocked_http_post:
            mocked_http_post.return_value = (mock.Mock(headers={}), 200, 'data', 'headers')
            item.perform(formdata)
            url = mocked_http_post.call_args[1]['url']
            payload = mocked_http_post.call_args[1]['body']
            assert 'http://passerelle.example.com' in url
            assert '?nostop=1' in url
            assert 'orig=localhost' in url
            assert 'signature=' in url
            json_payload = json.loads(payload)
            assert 'message' in json_payload
            assert json_payload['message'] == 'my "message" with "quotes"'
            assert json_payload['to'] == ['1234']
            assert json_payload['from'] == 'Passerelle'

    pub.loggederror_class.wipe()
    with mock.patch('wcs.wscalls.get_secret_and_orig') as mocked_secret_and_orig:
        mocked_secret_and_orig.return_value = ('secret', 'localhost')
        with mock.patch('wcs.qommon.misc._http_request') as mocked_http_post:
            mocked_http_post.return_value = (mock.Mock(headers={}), 400, '{"err": 1}', 'headers')
            item.perform(formdata)
            assert pub.loggederror_class.count() == 1
            assert pub.loggederror_class.select()[0].summary == 'Could not send SMS'


def test_display_form(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = st1.add_action('form', id='_x')
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(StringField(id='1', label='Test', type='string'))
    display_form.formdef.fields.append(DateField(id='2', label='Date', type='date', varname='date'))

    form = Form(action='#', use_tokens=False)
    display_form.fill_form(form, formdata, None)
    assert form.widgets[0].title == 'Test'
    assert form.widgets[1].title == 'Date'

    two_pubs.get_request().environ['REQUEST_METHOD'] = 'POST'
    two_pubs.get_request().form = {'fxxx_1': 'Foobar', 'fxxx_2': '2015-05-12', 'submit': 'submit'}
    display_form.submit_form(form, formdata, None, None)

    assert formdata.get_substitution_variables()['xxx_var_date'] == '2015-05-12'

    two_pubs.cfg['language'] = {'language': 'fr'}
    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    form = Form(action='#', use_tokens=False)
    display_form.fill_form(form, formdata, None)
    two_pubs.get_request().environ['REQUEST_METHOD'] = 'POST'
    two_pubs.get_request().form = {'fxxx_1': 'Foobar', 'fxxx_2': '12/05/2015', 'submit': 'submit'}
    display_form.submit_form(form, formdata, None, None)
    assert formdata.get_substitution_variables()['xxx_var_date'] == '12/05/2015'

    assert formdata.get_substitution_variables()['xxx_var_date_raw'] == time.strptime(
        '2015-05-12', '%Y-%m-%d'
    )

    two_pubs.cfg['language'] = {'language': 'en'}


def test_display_form_and_comment(pub):
    role = pub.role_class(name='bar1')
    role.store()

    user = pub.user_class()
    user.roles = [role.id]
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = st1.add_action('form', id='_x')
    display_form.by = [role.id]
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(CommentField(id='1', label='Test', type='comment'))

    commentable = st1.add_action('commentable')
    commentable.by = [role.id]

    wf.store()

    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdata.get_status().name == 'Status1'

    form = formdata.get_workflow_form(user)
    assert 'Test' in str(form.widgets[0].render())
    assert '<textarea' in str(form.widgets[1].render())


def test_display_form_migration(pub):
    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    display_form = st1.add_action('form', id='_x')
    display_form.varname = 'xxx'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [ItemField(id='1', label='Test', type='item')]

    display_form.formdef.fields[0].show_as_radio = True
    wf.store()

    wf = Workflow.get(wf.id)
    assert wf.possible_status[0].items[0].formdef.fields[0].display_mode == 'radio'


def test_choice_button_no_label(pub):
    role = pub.role_class(name='bar1')
    role.store()

    user = pub.user_class()
    user.roles = [role.id]
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    choice = st1.add_action('choice', id='_x')
    choice.by = [role.id]

    choice2 = st1.add_action('choice', id='_x2')
    choice2.label = 'TEST'
    choice2.by = [role.id]

    wf.store()

    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    assert formdata.get_status().name == 'Status1'

    form = formdata.get_workflow_form(user)
    form.render()
    assert str(form.render()).count('<button') == 1
    assert '>TEST</button>' in str(form.render())


def test_choice_button_template_label(pub):
    role = pub.role_class(name='bar1')
    role.store()

    user = pub.user_class()
    user.roles = [role.id]
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    wf = Workflow(name='status')
    st1 = wf.add_status('Status1', 'st1')

    choice = st1.add_action('choice', id='_x')
    choice.label = '{{ "a"|add:"b" }}'
    choice.by = [role.id]

    wf.store()

    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    form = formdata.get_workflow_form(user)
    form.render()
    assert '>ab</button>' in str(form.render())

    # check ezt template are not interpreted
    choice.label = '[if-any test]a[else]b[end]'
    wf.store()

    formdata.refresh_from_storage()
    form = formdata.get_workflow_form(user)
    form.render()
    assert '>[if-any test]a[else]b[end]</button>' in str(form.render())


def test_workflow_role_type_migration(pub):
    workflow = Workflow(name='role migration')
    st1 = workflow.add_status('Status1', 'st1')

    jump = st1.add_action('jump', id='_jump')
    jump.by = [1, 2]

    workflow.store()

    reloaded_workflow = Workflow.get(workflow.id)
    assert reloaded_workflow.possible_status[0].items[0].by == ['1', '2']


def test_workflow_display_message(pub):
    pub.substitutions.feed(MockSubstitutionVariables())

    workflow = Workflow(name='display message')
    st1 = workflow.add_status('Status1', 'st1')

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.store()
    formdata = formdef.data_class()()
    formdata.id = '1'

    display_message = DisplayMessageWorkflowStatusItem()
    display_message.parent = st1

    display_message.message = 'test'
    assert display_message.get_message(formdata) == 'test'

    display_message.message = '{{ number }}'
    assert display_message.get_message(formdata) == str(formdata.id)

    display_message.message = '[number]'
    assert display_message.get_message(formdata) == str(formdata.id)

    display_message.message = '{{ bar }}'
    assert display_message.get_message(formdata) == 'Foobar'

    display_message.message = '[bar]'
    assert display_message.get_message(formdata) == 'Foobar'

    # makes sure the string is correctly escaped for HTML
    display_message.message = '{{ foo }}'
    assert display_message.get_message(formdata) == '1 &lt; 3'
    display_message.message = '[foo]'
    assert display_message.get_message(formdata) == '1 &lt; 3'


def test_workflow_display_message_to(pub):
    workflow = Workflow(name='display message to')
    st1 = workflow.add_status('Status1', 'st1')

    role = pub.role_class(name='foorole')
    role.store()
    role2 = pub.role_class(name='no-one-role')
    role2.store()
    user = pub.user_class(name='baruser')
    user.roles = []
    user.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef._workflow = workflow
    formdef.store()

    formdata = formdef.data_class()()
    formdata.status = 'wf-st1'

    display_message = st1.add_action('displaymsg')

    display_message.message = 'all'
    display_message.to = None
    assert display_message.get_message(formdata) == 'all'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['all']

    display_message.message = 'to-role'
    display_message.to = [role.id]
    assert display_message.get_message(formdata) == ''
    assert formdata.get_workflow_messages(user=pub._request._user) == []

    pub._request._user = user
    display_message.message = 'to-role'
    display_message.to = [role.id]
    assert display_message.get_message(formdata) == ''
    assert formdata.get_workflow_messages(user=pub._request._user) == []
    user.roles = [role.id]
    assert display_message.get_message(formdata) == 'to-role'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['to-role']

    user.roles = []
    display_message.message = 'to-submitter'
    display_message.to = ['_submitter']
    assert display_message.get_message(formdata) == ''
    assert formdata.get_workflow_messages(user=pub._request._user) == []
    formdata.user_id = user.id
    assert display_message.get_message(formdata) == 'to-submitter'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['to-submitter']

    display_message.message = 'to-role-or-submitter'
    display_message.to = [role.id, '_submitter']
    assert display_message.get_message(formdata) == 'to-role-or-submitter'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['to-role-or-submitter']
    formdata.user_id = None
    assert display_message.get_message(formdata) == ''
    assert formdata.get_workflow_messages(user=pub._request._user) == []
    user.roles = [role.id]
    assert display_message.get_message(formdata) == 'to-role-or-submitter'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['to-role-or-submitter']
    formdata.user_id = user.id
    assert display_message.get_message(formdata) == 'to-role-or-submitter'
    assert formdata.get_workflow_messages(user=pub._request._user) == ['to-role-or-submitter']

    display_message.to = [role2.id]
    assert display_message.get_message(formdata) == ''
    assert formdata.get_workflow_messages(user=pub._request._user) == []

    display_message.message = 'd1'
    display_message2 = st1.add_action('displaymsg')
    display_message2.message = 'd2'
    display_message2.to = [role.id, '_submitter']
    assert formdata.get_workflow_messages(user=pub._request._user) == ['d2']
    user.roles = [role.id, role2.id]
    assert 'd1' in formdata.get_workflow_messages(user=pub._request._user)
    assert 'd2' in formdata.get_workflow_messages(user=pub._request._user)


def test_workflow_display_message_line_details(pub):
    workflow = Workflow(name='display message to')
    st1 = workflow.add_status('Status1', 'st1')
    display_message = DisplayMessageWorkflowStatusItem()
    display_message.parent = st1

    assert display_message.get_line_details() == 'top of page'
    display_message.position = 'top'
    assert display_message.get_line_details() == 'top of page'
    display_message.position = 'bottom'
    assert display_message.get_line_details() == 'bottom of page'
    display_message.position = 'actions'
    assert display_message.get_line_details() == 'with actions'

    role = pub.role_class(name='foorole')
    role.store()
    display_message.to = [role.id]
    assert display_message.get_line_details() == 'with actions, for foorole'


def test_choice_line_details(pub):
    workflow = Workflow(name='choice')
    st1 = workflow.add_status('Status1', 'st1')
    choice = st1.add_action('choice')

    assert choice.get_line_details() == 'not completed'

    choice.status = 'wf-%s' % st1.id
    choice.label = 'foobar'
    assert choice.get_line_details() == '"foobar", to Status1'

    role = pub.role_class(name='foorole')
    role.store()
    choice.by = [role.id]
    assert choice.get_line_details() == '"foobar", to Status1, by foorole'

    choice.by = ['_receiver']
    assert choice.get_line_details() == '"foobar", to Status1, by Recipient'

    choice.by = ['logged-users']
    assert choice.get_line_details() == '"foobar", to Status1, by Logged Users'


def test_workflow_roles(pub, emails):
    pub.substitutions.feed(MockSubstitutionVariables())

    user = pub.user_class(name='foo')
    user.email = 'zorg@localhost'
    user.store()

    pub.role_class.wipe()
    role1 = pub.role_class(name='foo')
    role1.emails = ['foo@localhost']
    role1.details = 'Hello World'
    role1.store()

    role2 = pub.role_class(name='bar')
    role2.emails = ['bar@localhost', 'baz@localhost']
    role2.store()

    workflow = Workflow(name='wf roles')
    st1 = workflow.add_status('Status1', 'st1')
    item = st1.add_action('sendmail')
    item.to = ['_receiver', '_other']
    item.subject = 'Foobar'
    item.body = 'Hello'
    workflow.roles['_other'] = 'Other Function'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_roles = {'_receiver': role1.id, '_other': role2.id}
    formdef.workflow_id = workflow.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    emails.empty()
    item.perform(formdata)
    get_response().process_after_jobs()
    assert emails.count() == 1
    assert emails.get('Foobar')
    assert set(emails.get('Foobar')['email_rcpt']) == {'foo@localhost', 'bar@localhost', 'baz@localhost'}

    workflow.roles['_slug-with-dash'] = 'Dashed Function'
    workflow.store()
    formdef.workflow_roles['_slug-with-dash'] = role1.id
    formdef.store()
    substvars = formdata.get_substitution_variables()
    assert substvars.get('form_role_other_name') == 'bar'
    assert substvars.get('form_role_slug_with_dash_name') == 'foo'
    assert substvars.get('form_role_slug_with_dash_details') == 'Hello World'


def test_criticality(pub):
    FormDef.wipe()

    workflow = Workflow(name='criticality')
    workflow.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.workflow_id = workflow.id
    formdef.store()

    item = ModifyCriticalityWorkflowStatusItem()

    formdata = formdef.data_class()()
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'yellow'

    formdata = formdef.data_class()()
    item.mode = MODE_INC
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'yellow'
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'red'
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'red'

    item.mode = MODE_DEC
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'yellow'
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'green'
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'green'

    item.mode = MODE_SET
    item.absolute_value = 2
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'red'
    item.absolute_value = 0
    item.perform(formdata)
    assert formdata.get_criticality_level_object().name == 'green'


def test_geolocate_action_enable_geolocation(two_pubs):
    # switch to a workflow with geolocation
    formdef = FormDef()
    formdef.name = 'foo'
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()

    workflow = Workflow(name='wf')
    st1 = workflow.add_status('Status1', 'st1')
    item = st1.add_action('geolocate')
    item.method = 'address_string'
    item.address_string = '{{form_var_string}}, paris, france'
    workflow.store()

    formdef.change_workflow(workflow)
    assert formdef.geolocations

    conn, cur = sql.get_connection_and_cursor()
    assert column_exists_in_table(cur, formdef.table_name, 'geoloc_base')
    conn.commit()
    cur.close()

    # change to current workflow
    workflow = Workflow(name='wf2')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.store()

    formdef = FormDef()
    formdef.name = 'bar'
    formdef.fields = []
    formdef.workflow = workflow
    formdef.store()
    assert not formdef.geolocations

    item = st1.add_action('geolocate')
    item.method = 'address_string'
    item.address_string = '{{form_var_string}}, paris, france'
    workflow.store()
    get_response().process_after_jobs()

    formdef.refresh_from_storage()
    assert formdef.geolocations

    conn, cur = sql.get_connection_and_cursor()
    assert column_exists_in_table(cur, formdef.table_name, 'geoloc_base')
    conn.commit()
    cur.close()


def test_geolocate_address(two_pubs):
    formdef = FormDef()
    formdef.geolocations = {'base': 'bla'}
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='string'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '169 rue du chateau'}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = GeolocateWorkflowStatusItem()
    item.method = 'address_string'
    item.address_string = '[form_var_string], paris, france'

    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (
            None,
            200,
            force_bytes(json.dumps([{'lat': '48.8337085', 'lon': '2.3233693'}])),
            None,
        )
        item.perform(formdata)
        assert 'https://nominatim.entrouvert.org/search' in http_get_page.call_args[0][0]
        assert urllib.parse.quote('169 rue du chateau, paris') in http_get_page.call_args[0][0]
        assert int(formdata.geolocations['base']['lat']) == 48
        assert int(formdata.geolocations['base']['lon']) == 2

    two_pubs.load_site_options()
    two_pubs.site_options.set('options', 'nominatim_key', 'KEY')
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (
            None,
            200,
            force_bytes(json.dumps([{'lat': '48.8337085', 'lon': '2.3233693'}])),
            None,
        )
        item.perform(formdata)
        assert 'https://nominatim.entrouvert.org/search' in http_get_page.call_args[0][0]
        assert urllib.parse.quote('169 rue du chateau, paris') in http_get_page.call_args[0][0]
        assert 'key=KEY' in http_get_page.call_args[0][0]
        assert int(formdata.geolocations['base']['lat']) == 48
        assert int(formdata.geolocations['base']['lon']) == 2

    two_pubs.load_site_options()
    two_pubs.site_options.set('options', 'geocoding_service_url', 'http://example.net/')
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (
            None,
            200,
            force_bytes(json.dumps([{'lat': '48.8337085', 'lon': '2.3233693'}])),
            None,
        )
        item.perform(formdata)
        assert 'http://example.net/?q=' in http_get_page.call_args[0][0]

    two_pubs.site_options.set('options', 'geocoding_service_url', 'http://example.net/?param=value')
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (
            None,
            200,
            force_bytes(json.dumps([{'lat': '48.8337085', 'lon': '2.3233693'}])),
            None,
        )
        item.perform(formdata)
        assert 'http://example.net/?param=value&' in http_get_page.call_args[0][0]

    # check for invalid ezt
    item.address_string = '[if-any], paris, france'
    formdata.geolocations = None
    item.perform(formdata)
    assert formdata.geolocations == {}
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert (
        logged_error.summary
        == 'error in template for address string [syntax error in ezt template: unclosed block at line 1 and column 24]'
    )
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.exception_class == 'TemplateError'
    assert (
        logged_error.exception_message
        == 'syntax error in ezt template: unclosed block at line 1 and column 24'
    )
    two_pubs.loggederror_class.wipe()

    # check for None
    item.address_string = '=None'
    formdata.geolocations = None
    item.perform(formdata)
    assert formdata.geolocations == {}

    # check for nominatim returning an empty result set
    item.address_string = '[form_var_string], paris, france'
    formdata.geolocations = None
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (None, 200, force_bytes(json.dumps([])), None)
        item.perform(formdata)
        assert formdata.geolocations == {}

    # check for nominatim bad json
    formdata.geolocations = None
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.return_value = (None, 200, b'bad json', None)
        item.perform(formdata)
        assert formdata.geolocations == {}

    # check for nominatim connection error
    formdata.geolocations = None
    with mock.patch('wcs.wf.geolocate.http_get_page') as http_get_page:
        http_get_page.side_effect = ConnectionError('some error')
        item.perform(formdata)
        assert formdata.geolocations == {}
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'error calling geocoding service [some error]'
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.exception_class == 'ConnectionError'
    assert logged_error.exception_message == 'some error'


def test_geolocate_image(pub):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.geolocations = {'base': 'bla'}
    formdef.fields = [
        FileField(id='3', label='File', type='file', varname='file'),
    ]
    formdef.store()

    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
        upload.receive([fd.read()])

    formdata = formdef.data_class()()
    formdata.data = {'3': upload}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = GeolocateWorkflowStatusItem()
    item.method = 'photo_variable'

    item.photo_variable = '=form_var_file_raw'
    item.perform(formdata)
    assert int(formdata.geolocations['base']['lat']) == -1
    assert int(formdata.geolocations['base']['lon']) == 6

    # invalid expression
    formdata.geolocations = None
    item.photo_variable = '=1/0'
    item.perform(formdata)
    assert formdata.geolocations == {}

    # invalid type
    formdata.geolocations = None
    item.photo_variable = '="bla"'
    item.perform(formdata)
    assert formdata.geolocations == {}

    # invalid photo
    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as fd:
        upload.receive([fd.read()])
    formdata.data = {'3': upload}
    formdata.geolocations = None
    item.perform(formdata)
    assert formdata.geolocations == {}


def test_geolocate_map(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.geolocations = {'base': 'bla'}
    formdef.fields = [
        MapField(id='2', label='Map', type='map', varname='map'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'2': '48.8337085;2.3233693'}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = GeolocateWorkflowStatusItem()
    item.method = 'map_variable'
    item.map_variable = '=form_var_map'

    item.perform(formdata)
    assert int(formdata.geolocations['base']['lat']) == 48
    assert int(formdata.geolocations['base']['lon']) == 2

    # invalid data
    formdata.geolocations = None
    item.map_variable = '=form_var'
    item.perform(formdata)
    assert formdata.geolocations == {}

    # invalid data
    formdata.geolocations = None
    formdata.data = {'2': '48.8337085'}
    item.map_variable = '=form_var_map'
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'error geolocating from map variable'
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.exception_class == 'ValueError'
    assert logged_error.exception_message == 'not enough values to unpack (expected 2, got 1)'


def test_geolocate_overwrite(pub):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.geolocations = {'base': 'bla'}
    formdef.fields = [
        MapField(id='2', label='Map', type='map', varname='map'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'2': '48.8337085;2.3233693'}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = GeolocateWorkflowStatusItem()
    item.method = 'map_variable'
    item.map_variable = '=form_var_map'

    item.perform(formdata)
    assert int(formdata.geolocations['base']['lat']) == 48
    assert int(formdata.geolocations['base']['lon']) == 2

    formdata.data = {'2': '48.8337085;3.3233693'}
    item.perform(formdata)
    assert int(formdata.geolocations['base']['lat']) == 48
    assert int(formdata.geolocations['base']['lon']) == 3

    formdata.data = {'2': '48.8337085;4.3233693'}
    item.overwrite = False
    item.perform(formdata)
    assert int(formdata.geolocations['base']['lat']) == 48
    assert int(formdata.geolocations['base']['lon']) == 3


@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
def test_transform_to_pdf():
    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as instream:
        outstream = transform_to_pdf(instream)
        assert outstream is not False
        assert outstream.read(10).startswith(b'%PDF-')


@pytest.mark.parametrize(
    'template_name', ['template-with-image.odt', 'template-with-image-django-syntax.odt']
)
def test_export_to_model_image(pub, template_name):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        FileField(id='3', label='File', type='file', varname='image'),
    ]
    formdef.store()

    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
        image_data = fd.read()
    upload.receive([image_data])

    formdata = formdef.data_class()()
    formdata.data = {'3': upload}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = ExportToModel()
    item.convert_to_pdf = False
    item.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), '..', template_name)
    with open(template_filename, 'rb') as fd:
        template = fd.read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    item.model_file = UploadedFile(pub.app_dir, None, upload)
    item.attach_to_history = True

    item.perform(formdata)

    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
    with zipfile.ZipFile(formdata.evolution[-1].parts[0].get_file_path(), mode='r') as zfile:
        zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
    # check the image has been replaced by the one from the formdata
    assert zinfo.file_size == len(image_data)

    # check with missing data or wrong kind of data
    for field_value in (None, 'wrong kind'):
        formdata = formdef.data_class()()
        formdata.data = {'3': field_value}
        formdata.just_created()
        formdata.store()
        pub.substitutions.feed(formdata)

        item.perform(formdata)

        with zipfile.ZipFile(formdata.evolution[-1].parts[0].get_file_path(), mode='r') as zfile:
            zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
        # check the original image has been left
        assert zinfo.file_size == 580

    item.filename = 'formulaire-{{form_number}}/2.odt'
    item.perform(formdata)
    assert formdata.evolution[-1].parts[-1].base_filename == 'formulaire-%s-%s-2.odt' % (
        formdef.id,
        formdata.id,
    )


def test_export_to_model_qrcode(pub):
    FormDef.wipe()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = ExportToModel()
    item.convert_to_pdf = False
    item.method = 'non-interactive'
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-with-qrcode.odt')
    with open(template_filename, 'rb') as fd:
        template = fd.read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    item.model_file = UploadedFile(pub.app_dir, None, upload)
    item.attach_to_history = True

    item.perform(formdata)

    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
    with zipfile.ZipFile(formdata.evolution[-1].parts[0].get_file_path(), mode='r') as zfile:
        # base template use a jpg images and export_to_model does not rename it
        # event when content is PNG, but it still works inside LibreOffice
        # which ignores the filename extension.
        image_filename = [name for name in zfile.namelist() if name.endswith('.jpg')][0]
        with zfile.open(image_filename, 'r') as image_fd:
            img = Image.open(image_fd)
            assert (
                zbar_decode_qrcode(img, symbols=[ZBarSymbol.QRCODE])[0].data.decode()
                == 'http://example.net/backoffice/management/baz/1/'
            )


def test_export_to_model_backoffice_field(pub):
    wf = Workflow(name='email with attachments')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo1', label='bo field 1', type='file', varname='backoffice_file1'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()
    formdef = FormDef()
    formdef.name = 'foo-export-to-bofile'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='string'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = ExportToModel()
    item.method = 'non-interactive'
    item.convert_to_pdf = False
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
    with open(template_filename, 'rb') as fd:
        template = fd.read()
    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    item.model_file = UploadedFile(pub.app_dir, None, upload)
    item.parent = st1
    item.backoffice_filefield_id = 'bo1'
    item.perform(formdata)

    assert 'bo1' in formdata.data
    fbo1 = formdata.data['bo1']
    assert fbo1.base_filename == 'template.odt'
    assert fbo1.content_type == 'application/octet-stream'
    with zipfile.ZipFile(fbo1.get_file()) as zfile:
        assert b'foo-export-to-bofile' in zfile.read('content.xml')

    # no more 'bo1' backoffice field: do nothing
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)
    # id is not bo1:
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo2', label='bo field 2', type='file'),
    ]
    item.perform(formdata)
    assert formdata.data == {}
    # field is not a field file:
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string'),
    ]
    item.perform(formdata)
    assert formdata.data == {}
    # no field at all:
    wf.backoffice_fields_formdef.fields = []
    item.perform(formdata)
    assert formdata.data == {}


def test_export_to_model_django_template(pub):
    formdef = FormDef()
    formdef.name = 'foo-export-to-template-with-django'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='string'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = ExportToModel()
    item.method = 'non-interactive'
    item.attach_to_history = True
    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-django.odt')
    with open(template_filename, 'rb') as fd:
        template = fd.read()
    upload = QuixoteUpload('/foo/template-django.odt', content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    item.model_file = UploadedFile(pub.app_dir, None, upload)
    item.convert_to_pdf = False
    item.perform(formdata)

    with open(formdata.evolution[0].parts[0].get_file_path(), 'rb') as fd:
        with zipfile.ZipFile(fd) as zout:
            new_content = zout.read('content.xml')
    assert b'>foo-export-to-template-with-django<' in new_content

    formdef.name = 'Name with a \' simple quote'
    formdef.store()
    item.perform(formdata)

    with open(formdata.evolution[0].parts[1].get_file_path(), 'rb') as fd:
        with zipfile.ZipFile(fd) as zout:
            new_content = zout.read('content.xml')
    assert b'>Name with a \' simple quote<' in new_content

    formdef.name = 'A <> name'
    formdef.store()
    item.perform(formdata)

    with open(formdata.evolution[0].parts[2].get_file_path(), 'rb') as fd:
        with zipfile.ZipFile(fd) as zout:
            new_content = zout.read('content.xml')
    assert b'>A &lt;&gt; name<' in new_content


def test_export_to_model_xml(two_pubs):
    pub = two_pubs
    LoggedError = pub.loggederror_class
    if LoggedError:
        LoggedError.wipe()

    formdef = FormDef()
    formdef.name = 'foo-export-to-template-with-django'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='string'),
    ]
    formdef.store()
    formdata = formdef.data_class()()
    formdata.data = {'1': 'écho'}
    formdata.just_created()
    formdata.store()

    # good XML
    item = ExportToModel()
    item.method = 'non-interactive'
    item.attach_to_history = True

    def run(template, filename='/foo/template.xml', content_type='application/xml'):
        upload = QuixoteUpload(filename, content_type=content_type)
        upload.fp = io.BytesIO()
        upload.fp.write(template.encode())
        upload.fp.seek(0)
        item.model_file = UploadedFile(pub.app_dir, None, upload)
        item.convert_to_pdf = False
        pub.substitutions.reset()
        pub.substitutions.feed(formdata)
        item.perform(formdata)
        with open(formdata.evolution[0].parts[-1].get_file_path()) as fd:
            return fd.read()

    # good XML
    assert run(template='<a>{{ form_var_string }}</a>') == '<a>écho</a>'
    assert (
        run(template='<a>{{ form_var_string }}</a>', content_type='application/octet-stream') == '<a>écho</a>'
    )
    assert run(template='<a>{{ form_var_string }}</a>', filename='/foo/template.svg') == '<a>écho</a>'

    # unknown file format
    with pytest.raises(UploadValidationError):
        run(
            template='<a>{{ form_var_string }}</a>',
            filename='/foo/template.txt',
            content_type='application/octet-stream',
        )

    # malformed XML
    assert not LoggedError or LoggedError.count() == 0
    assert run(template='<a>{{ form_var_string }}<a>') == '<a>écho<a>'
    # on error in the XML correctness no exception is raised but an error is logged
    assert not LoggedError or LoggedError.count() == 1


@pytest.mark.parametrize('filename', ['template-form-details.odt', 'template-form-details-no-styles.odt'])
def test_export_to_model_form_details_section(pub, filename):
    BlockDef.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.digest_template = 'X{{foobar_var_foo}}Y'
    block.fields = [
        StringField(id='123', required=True, label='Test', type='string', varname='foo'),
        StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foo-export-details'
    formdef.fields = [
        PageField(id='1', label='Page 1', type='page'),
        TitleField(id='2', label='Title', type='title'),
        SubtitleField(id='3', label='Subtitle', type='subtitle'),
        StringField(id='4', label='String', type='string', varname='string'),
        EmailField(id='5', label='Email', type='email'),
        TextField(id='6', label='Text', type='text'),
        BoolField(id='8', label='Bool', type='bool'),
        FileField(id='9', label='File', type='file'),
        DateField(id='10', label='Date', type='date'),
        ItemField(id='11', label='Item', type='item', items=['foo', 'bar']),
        TableField(id='12', label='Table', type='table', columns=['a', 'b'], rows=['c', 'd']),
        PageField(id='13', label='Empty Page', type='page'),
        TitleField(id='14', label='Empty Title', type='title'),
        StringField(id='15', label='Empty String', type='string', varname='invisiblestr'),
        BlockField(id='16', label='Block Field', type='block:foobar'),
        ItemsField(id='17', label='Items', type='items', items=['foo', 'bar']),
    ]
    formdef.store()
    formdef.data_class().wipe()
    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
        upload.receive([fd.read()])
    formdata = formdef.data_class()()
    formdata.data = {
        '4': 'string',
        '5': 'foo@localhost',
        '6': 'para1\npara2',
        '8': False,
        '9': upload,
        '10': time.strptime('2015-05-12', '%Y-%m-%d'),
        '11': 'foo',
        '12': [['1', '2'], ['3', '4']],
        # value from test_block_digest in tests/form_pages/test_block.py
        '16': {
            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
            'schema': {'123': 'string', '234': 'string'},
        },
        '16_display': 'XfooY, Xfoo2Y',
    }
    formdata.just_created()
    formdata.store()
    pub.substitutions.feed(formdata)

    item = ExportToModel()
    item.method = 'non-interactive'
    item.attach_to_history = True
    template_filename = os.path.join(os.path.dirname(__file__), '..', filename)
    with open(template_filename, 'rb') as fd:
        template = fd.read()
    upload = QuixoteUpload(filename, content_type='application/octet-stream')
    upload.fp = io.BytesIO()
    upload.fp.write(template)
    upload.fp.seek(0)
    item.model_file = UploadedFile(pub.app_dir, None, upload)
    item.convert_to_pdf = False
    item.perform(formdata)

    with open(formdata.evolution[0].parts[0].get_file_path(), 'rb') as fd:
        with zipfile.ZipFile(fd) as zout:
            new_content = force_text(zout.read('content.xml'))
    # section content has been removed
    assert 'Titre de page' not in new_content
    assert 'Titre' not in new_content
    assert 'Libell&#233; de champ' not in new_content
    assert 'Valeur de champ' not in new_content
    # and new content has been inserted
    assert '>Page 1<' in new_content
    assert '>Title<' in new_content
    assert '>Subtitle<' in new_content
    assert '<text:span>string</text:span>' in new_content
    assert '>para1<' in new_content
    assert '>para2<' in new_content
    assert '<text:span>No</text:span>' in new_content
    assert 'xlink:href="http://example.net/foo-export-details/1/download?f=9"' in new_content
    assert '>test.jpeg</text:a' in new_content
    assert '>2015-05-12<' in new_content
    assert 'Invisible' not in new_content
    assert new_content.count('/table:table-cell') == 8
    assert 'XfooY, Xfoo2Y' in new_content

    if filename == 'template-form-details-no-styles.odt':
        with open(formdata.evolution[0].parts[0].get_file_path(), 'rb') as fd:
            with zipfile.ZipFile(fd) as zout:
                new_styles = force_text(zout.read('styles.xml'))
        assert 'Field_20_Label' in new_styles


@pytest.mark.parametrize('formdef_class', [FormDef, CardDef])
def test_global_timeouts(two_pubs, formdef_class):
    pub = two_pubs
    CardDef.wipe()
    FormDef.wipe()
    Workflow.wipe()

    workflow = Workflow(name='global-timeouts')
    workflow.possible_status = Workflow.get_default_workflow().possible_status[:]
    workflow.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    action = workflow.add_global_action('Timeout Test')
    action.add_action('modify_criticality')
    trigger = action.append_trigger('timeout')
    trigger.anchor = 'creation'
    workflow.store()

    formdef = formdef_class()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    formdata1 = formdef.data_class()()
    formdata1.just_created()
    formdata1.store()

    # delay isn't set yet, no crash
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    # delay didn't expire yet, no change
    trigger.timeout = '2'
    workflow.store()

    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    formdata1.receipt_time = time.localtime(time.time() - 3 * 86400)
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'

    # make sure it's not triggered a second time
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'

    # change id so it's triggered again
    trigger.id = 'XXX1'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'red'
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'red'

    # reset formdata to initial state
    formdata1.store()

    trigger.anchor = '1st-arrival'
    trigger.anchor_status_first = None
    workflow.store()

    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    formdata1.evolution[-1].time = time.localtime(time.time() - 3 * 86400)
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'

    formdata1.store()  # reset

    # bad (obsolete) status: do nothing
    trigger.anchor_status_first = 'wf-foobar'
    workflow.store()
    formdata1.evolution[-1].time = time.localtime(time.time() - 3 * 86400)
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    trigger.anchor = 'latest-arrival'
    trigger.anchor_status_latest = None
    workflow.store()

    formdata1.evolution[-1].time = time.localtime()
    formdata1.store()
    formdata1.jump_status('new')
    formdata1.evolution[-1].time = time.localtime(time.time() - 7 * 86400)
    formdata1.jump_status('accepted')
    formdata1.jump_status('new')
    formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)

    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # limit trigger to formdata with "accepted" status
    trigger.anchor_status_latest = 'wf-accepted'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    # limit trigger to formdata with "new" status
    trigger.anchor_status_latest = 'wf-new'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # bad (obsolete) status: do nothing
    trigger.anchor_status_latest = 'wf-foobar'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    # check trigger is not run on finalized formdata
    formdata1.jump_status('finished')
    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
    formdata1.store()
    trigger.anchor = 'creation'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    # check trigger is run on finalized formdata when anchor status is an
    # endpoint
    formdata1.jump_status('finished')
    formdata1.evolution[-1].last_jump_datetime = None
    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
    formdata1.store()
    trigger.anchor = 'latest-arrival'
    trigger.anchor_status_latest = 'wf-finished'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # check "finalized" anchor
    trigger.anchor = 'finalized'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # use python expression as anchor
    #  timestamp
    formdata1.jump_status('new')
    formdata1.evolution[-1].time = time.localtime(time.time() - 4 * 86400)
    formdata1.evolution[-1].last_jump_datetime = None
    formdata1.store()

    trigger.anchor = 'python'
    trigger.anchor_expression = repr(time.time())
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    trigger.anchor = 'python'
    trigger.anchor_expression = repr(time.time() - 10 * 86400)
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    #  datetime object
    trigger.anchor = 'python'
    trigger.anchor_expression = (
        'datetime.datetime(%s, %s, %s, %s, %s)'
        % (datetime.datetime.now() - datetime.timedelta(days=10)).timetuple()[:5]
    )
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    #  datetime object
    trigger.anchor = 'python'
    trigger.anchor_expression = (
        'datetime.date(%s, %s, %s)' % (datetime.datetime.now() - datetime.timedelta(days=10)).timetuple()[:3]
    )
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # string object
    trigger.anchor = 'python'
    trigger.anchor_expression = (
        '"%04d-%02d-%02d"' % (datetime.datetime.now() - datetime.timedelta(days=10)).timetuple()[:3]
    )
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # invalid variable
    trigger.anchor = 'python'
    trigger.anchor_expression = 'Ellipsis'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    # invalid expression
    trigger.anchor = 'python'
    trigger.anchor_expression = 'XXX'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'
    formdata1.store()

    # django template
    trigger.anchor = 'template'
    trigger.anchor_template = '{{ form_receipt_date|date:"Y-m-d" }}'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # django template
    trigger.anchor = 'template'
    trigger.anchor_template = '{{ form_receipt_date|date:"Y-m-d" }}'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # django template (with local date format)
    trigger.anchor = 'template'
    trigger.anchor_template = '{{ form_receipt_date }}'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # django template (with local date/time format)
    trigger.anchor = 'template'
    trigger.anchor_template = '{{ form_receipt_datetime }}'
    workflow.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()

    # django template (from form_option_)
    workflow.variables_formdef = WorkflowVariablesFieldsFormDef(workflow=workflow)
    workflow.variables_formdef.fields = [
        DateField(id='4', label='Date', type='date', varname='date'),
    ]
    trigger.anchor = 'template'
    trigger.anchor_template = '{{ form_option_date }}'
    workflow.store()
    formdef.workflow_options = {
        'date': time.strptime('2015-05-12', '%Y-%m-%d'),
    }
    formdef.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'
    formdata1.store()


def test_global_timeouts_latest_arrival(two_pubs):
    pub = two_pubs
    FormDef.wipe()
    Workflow.wipe()

    workflow = Workflow(name='global-timeouts')
    workflow.possible_status = Workflow.get_default_workflow().possible_status[:]
    workflow.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    action = workflow.add_global_action('Timeout Test')
    action.add_action('modify_criticality')
    trigger = action.append_trigger('timeout')
    trigger.anchor = 'latest-arrival'
    trigger.anchor_status_latest = 'wf-new'
    trigger.timeout = '2'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    formdata1 = formdef.data_class()()
    formdata1.just_created()
    formdata1.store()

    formdata1.jump_status('new')
    # enter in status 8 days ago
    formdata1.evolution[-1].time = time.localtime(time.time() - 8 * 86400)
    # but get a new comment 1 day ago
    formdata1.evolution.append(Evolution())
    formdata1.evolution[-1].time = time.localtime(time.time() - 1 * 86400)
    formdata1.evolution[-1].comment = 'plop'
    formdata1.store()
    pub.apply_global_action_timeouts()
    # no change
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'

    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
    formdata1.store()
    pub.apply_global_action_timeouts()
    # change
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'

    # check it applies even after the status has been left
    formdata1 = formdef.data_class()()
    formdata1.just_created()
    formdata1.store()
    formdata1.jump_status('new')
    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
    formdata1.store()
    formdata1.jump_status('accepted')
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'yellow'

    # but not if an endpoint has been reached
    formdata1 = formdef.data_class()()
    formdata1.just_created()
    formdata1.store()
    formdata1.jump_status('new')
    formdata1.evolution[-1].time = time.localtime(time.time() - 5 * 86400)
    formdata1.store()
    formdata1.jump_status('accepted')
    formdata1.jump_status('finished')
    formdata1.store()
    pub.apply_global_action_timeouts()
    assert formdef.data_class().get(formdata1.id).get_criticality_level_object().name == 'green'


def test_profile(two_pubs):
    User = two_pubs.user_class
    user = User()
    user.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.user_id = user.id
    formdata.data = {'1': 'bar@localhost'}

    item = UpdateUserProfileStatusItem()
    item.fields = [{'field_id': '__email', 'value': '=form_var_foo'}]
    item.perform(formdata)
    assert User.get(user.id).email == 'bar@localhost'

    formdata.data = {'1': 'Plop'}
    item.fields = [{'field_id': '__name', 'value': '=form_var_foo'}]
    item.perform(formdata)
    assert User.get(user.id).name == 'Plop'

    item.fields = [{'field_id': '__name', 'value': 'dj{{form_var_foo}}'}]
    item.perform(formdata)
    assert User.get(user.id).name == 'djPlop'
    item.fields = [{'field_id': '__name', 'value': 'ezt[form_var_foo]'}]
    item.perform(formdata)
    assert User.get(user.id).name == 'eztPlop'

    from wcs.admin.settings import UserFieldsFormDef

    formdef = UserFieldsFormDef(two_pubs)
    formdef.fields = [
        StringField(id='3', label='test', type='string', varname='plop'),
        DateField(id='4', label='Date', type='date', varname='bar'),
    ]
    formdef.store()

    item.fields = [{'field_id': 'plop', 'value': '=form_var_foo'}]
    item.perform(formdata)
    assert User.get(user.id).form_data.get('3') == 'Plop'
    assert not User.get(user.id).form_data.get('4')

    # check transmission to IdP
    get_publisher().cfg['sp'] = {'idp-manage-user-attributes': True}
    get_publisher().cfg['idp'] = {'xxx': {'metadata_url': 'http://idp.example.net/idp/saml2/metadata'}}
    get_publisher().write_cfg()

    user = User.get(user.id)
    user.name_identifiers = ['xyz']
    user.store()

    for date_value in (
        '20/03/2018',
        '=utils.make_date("20/03/2018")',
        '=utils.make_date("20/03/2018").timetuple()',
    ):

        # check local value
        item.fields = [{'field_id': 'bar', 'value': date_value}]
        item.perform(formdata)
        assert User.get(user.id).form_data.get('3') == 'Plop'
        assert User.get(user.id).form_data.get('4').tm_year == 2018

        with mock.patch('wcs.wf.profile.http_patch_request') as http_patch_request:
            http_patch_request.return_value = (None, 200, '', None)
            get_response().process_after_jobs()
            assert http_patch_request.call_count == 1
            assert http_patch_request.call_args[0][1] == '{"bar": "2018-03-20"}'

    for date_value in ('baddate', '', {}, [], None):
        # reset date to a known value
        user.form_data['4'] = datetime.datetime.now().timetuple()
        user.store()
        year = User.get(user.id).form_data.get('4').tm_year
        # perform action
        item.fields = [{'field_id': 'bar', 'value': date_value}]
        item.perform(formdata)
        if date_value not in (None, ''):  # bad value : do nothing
            assert User.get(user.id).form_data.get('4').tm_year == year
        else:  # empty value : empty field
            assert User.get(user.id).form_data.get('4') is None

        with mock.patch('wcs.wf.profile.http_patch_request') as http_patch_request:
            http_patch_request.return_value = (None, 200, '', None)
            get_response().process_after_jobs()
            assert http_patch_request.call_count == 1
            if date_value not in (None, ''):  # bad value : do nothing
                assert http_patch_request.call_args[0][1] == '{}'
            else:  # empty value : null field
                assert http_patch_request.call_args[0][1] == '{"bar": null}'

    # out of http request/response cycle (cron, after_job)
    two_pubs._set_request(None)
    item.fields = [{'field_id': 'bar', 'value': '01/01/2020'}]
    with mock.patch('wcs.wf.profile.http_patch_request') as http_patch_request:
        http_patch_request.return_value = (None, 200, '', None)
        item.perform(formdata)
        assert User.get(user.id).form_data.get('4').tm_year == 2020
        assert http_patch_request.call_count == 1
        assert http_patch_request.call_args[0][1] == '{"bar": "2020-01-01"}'


def test_set_backoffice_field(http_requests, two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='1st backoffice field', type='string', varname='backoffice_blah'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='00', label='String', type='string', varname='string'),
        StringField(id='01', label='Other string', type='string', varname='other'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'00': 'HELLO'}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1

    assert item.render_as_line() == 'Backoffice Data'
    assert item.get_jump_label('plop') == 'Backoffice Data'
    item.label = 'label'
    assert item.render_as_line() == 'Backoffice Data (label)'
    assert item.get_jump_label('plop') == 'Backoffice Data "label"'

    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') is None

    item.fields = [{'field_id': 'bo1', 'value': '=form_var_string'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO'

    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_string }} WORLD'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO WORLD'

    item.fields = [{'field_id': 'bo1', 'value': '[form_var_string] GOODBYE'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO GOODBYE'

    item.fields = [{'field_id': 'bo1', 'value': '{{ form.var.string }} LAZY'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO LAZY'

    item.fields = [{'field_id': 'bo1', 'value': '=form.var.string'}]  # lazy python
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO'

    item.fields = [{'field_id': 'bo1', 'value': '=vars().get("form_var_string") + " PLOP"'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO PLOP'

    item.fields = [
        {'field_id': 'bo1', 'value': '=vars().get("form_var_other") or vars().get("form_var_string")'}
    ]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'HELLO'

    item.fields = [{'field_id': 'bo1', 'value': '=vars().get("form_var_bouh", "X") + " PLOP"'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'X PLOP'

    item.fields = [{'field_id': 'bo1', 'value': None}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] is None

    item.fields = [{'field_id': 'bo1', 'value': ''}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] is None

    # check a value computed as the empty string is stored as an empty string, not None
    item.fields = [{'field_id': 'bo1', 'value': '=""'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ''
    item.fields = [{'field_id': 'bo1', 'value': '{{ does_not_exist }}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ''

    assert two_pubs.loggederror_class.count() == 0

    item.fields = [{'field_id': 'bo1', 'value': '= ~ invalid python ~'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Failed to compute Python expression'
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.expression == ' ~ invalid python ~'
    assert logged_error.expression_type == 'python'
    assert logged_error.exception_class == 'SyntaxError'
    assert logged_error.exception_message == 'invalid syntax (<string>, line 1)'

    two_pubs.loggederror_class.wipe()
    item.fields = [{'field_id': 'bo1', 'value': '{% if bad django %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Failed to compute template'
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.expression == '{% if bad django %}'
    assert logged_error.expression_type == 'template'
    assert logged_error.exception_class == 'TemplateError'
    assert logged_error.exception_message.startswith('syntax error in Django template')


def test_set_backoffice_field_map(http_requests, two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        MapField(id='bo1', label='1st backoffice field', type='map', varname='backoffice_blah'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        MapField(id='1', label='Map1', type='map', varname='map1'),
        MapField(id='2', label='Map2', type='map', varname='map2'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '42;10', '2': None}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1

    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_map1|default:"" }}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') == '42;10'

    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_map2|default:"" }}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') is None

    assert two_pubs.loggederror_class.count() == 0

    item.fields = [{'field_id': 'bo1', 'value': 'invalid value'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert (
        logged_error.summary
        == "Failed to set Map field (bo1), error: invalid coordinates 'invalid value' (missing ;)"
    )
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.exception_class == 'SetValueError'
    assert logged_error.exception_message == "invalid coordinates 'invalid value' (missing ;)"
    two_pubs.loggederror_class.wipe()

    item.fields = [{'field_id': 'bo1', 'value': 'XXX;YYY'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == "Failed to set Map field (bo1), error: invalid coordinates 'XXX;YYY'"
    assert logged_error.formdata_id == str(formdata.id)
    assert logged_error.exception_class == 'SetValueError'
    assert logged_error.exception_message == "invalid coordinates 'XXX;YYY'"
    two_pubs.loggederror_class.wipe()


def test_set_backoffice_field_decimal(http_requests, two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='1st backoffice field', type='string', varname='backoffice_blah'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='String', type='string', varname='string'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '1000'}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{{ "100"|decimal }}'}]
    item.perform(formdata)
    assert formdef.data_class().get(formdata.id).data['bo1'] == '100'

    formdata.store()  # reset
    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_string|decimal }}'}]
    item.perform(formdata)
    assert formdef.data_class().get(formdata.id).data['bo1'] == '1000'


def test_set_backoffice_field_file(http_requests, two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        FileField(id='bo1', label='1st backoffice field', type='file', varname='backoffice_file'),
        StringField(id='bo2', label='2nd backoffice field', type='string'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        FileField(id='00', label='File', type='file', varname='file'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '=locals().get("form_var_file_raw")'}]

    # the file does not exist
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] is None

    # store a PiclableUpload
    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
        image_with_gps_data = fd.read()
    upload.receive([image_with_gps_data])

    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'test.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert formdata.data['bo1'].get_content() == image_with_gps_data
    assert formdata.data['bo1'].get_base64_content() == base64.encodebytes(image_with_gps_data)

    # check with template string
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file_raw}}'}]
    item.perform(formdata)

    assert formdata.data['bo1'].base_filename == 'test.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert formdata.data['bo1'].get_content() == image_with_gps_data

    # check with template string, without _raw
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file}}'}]
    item.perform(formdata)

    assert formdata.data['bo1'].base_filename == 'test.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert formdata.data['bo1'].get_content() == image_with_gps_data

    # check |strip_metadata filter
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file|strip_metadata}}'}]
    item.perform(formdata)

    assert formdata.data['bo1'].base_filename == 'test.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert b'JFIF' in formdata.data['bo1'].get_content()
    assert b'<exif:XResolution>' not in formdata.data['bo1'].get_content()

    # check |rename_file filter
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file|rename_file:"foobar.jpeg"}}'}]
    item.perform(formdata)

    assert formdata.data['bo1'].base_filename == 'foobar.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert formdata.data['bo1'].get_content() == image_with_gps_data

    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file|rename_file:"foobar.$ext"}}'}]
    item.perform(formdata)

    assert formdata.data['bo1'].base_filename == 'foobar.jpeg'
    assert formdata.data['bo1'].content_type == 'image/jpeg'
    assert formdata.data['bo1'].get_content() == image_with_gps_data

    # check |rename_file with invalid input
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()

    two_pubs.substitutions.feed(formdata)

    item.fields = [{'field_id': 'bo1', 'value': '{{"xxx"|rename_file:"foobar.jpeg"}}'}]
    item.perform(formdata)

    assert 'bo1' not in formdata.data

    # check with a template string, into a string field
    two_pubs.substitutions.feed(formdata)
    item.fields = [{'field_id': 'bo2', 'value': '{{form_var_file}}'}]
    item.perform(formdata)

    assert formdata.data['bo2'] == 'test.jpeg'

    # check with template string and missing file
    formdata = formdef.data_class()()
    formdata.data = {'00': None}
    formdata.just_created()
    formdata.store()

    assert formdata.data.get('bo1') is None

    # check stripping metadata
    two_pubs.substitutions.feed(formdata)
    item.fields = [
        {
            'field_id': 'bo1',
            'value': '=utils.attachment(form_var_file_raw,'
            + ' filename="new_"+form_var_file,'
            + ' content_type="my content type",'
            + ' strip_metadata=True)',
        }
    ]
    formdata = formdef.data_class()()
    formdata.data = {'00': upload}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'new_test.jpeg'
    assert formdata.data['bo1'].content_type == 'my content type'
    assert formdata.data['bo1'].qfilename != formdata.data['00'].qfilename
    assert formdata.data['00'].get_content().find(b'<exif:XResolution>')
    assert not Image or formdata.data['bo1'].get_content().find(b'<exif:XResolution>') == -1

    upload2 = PicklableUpload('test2.odt', 'application/vnd.oasis.opendocument.text')
    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as fd:
        template_odt = fd.read()
    upload2.receive([template_odt])
    formdata = formdef.data_class()()
    formdata.data = {'00': upload2}
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'new_test2.odt'
    assert formdata.data['bo1'].content_type == 'my content type'
    assert formdata.data['bo1'].get_content() == template_odt
    assert formdata.data['bo1'].qfilename == formdata.data['00'].qfilename

    # check storing response as attachment
    two_pubs.substitutions.feed(formdata)
    item = WebserviceCallStatusItem()
    item.url = 'http://remote.example.net/xml'
    item.varname = 'xxx'
    item.response_type = 'attachment'
    item.record_errors = True
    item.perform(formdata)
    attachment = formdata.evolution[-1].parts[-1]
    assert isinstance(attachment, AttachmentEvolutionPart)
    assert attachment.base_filename == 'xxx.xml'
    assert attachment.content_type == 'text/xml'

    formdata = formdef.data_class().get(formdata.id)
    two_pubs.substitutions.feed(formdata)
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '=attachments.xxx'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'xxx.xml'

    # check storing a file from an assembled dictionary
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '={"content": "hello world", "filename": "hello.txt"}'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'hello.txt'
    assert formdata.data['bo1'].get_content() == b'hello world'

    # check resetting a value
    for value in ('', None):
        item = SetBackofficeFieldsWorkflowStatusItem()
        item.parent = st1
        item.fields = [{'field_id': 'bo1', 'value': value}]
        item.perform(formdata)

        formdata = formdef.data_class().get(formdata.id)
        assert formdata.data['bo1'] is None

    # set from base64 content
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [
        {'field_id': 'bo1', 'value': '={"b64_content": "SEVMTE8gV09STEQ=", "filename": "hello.txt"}'}
    ]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'].base_filename == 'hello.txt'
    assert formdata.data['bo1'].get_content() == b'HELLO WORLD'

    hello_world = formdata.data['bo1']
    # check wrong value
    for value in ('="HELLO"', 'BAD'):
        formdata.data['bo1'] = hello_world
        formdata.store()

        item = SetBackofficeFieldsWorkflowStatusItem()
        item.parent = st1
        item.fields = [{'field_id': 'bo1', 'value': value}]

        two_pubs.loggederror_class.wipe()
        item.perform(formdata)

        formdata = formdef.data_class().get(formdata.id)
        assert formdata.data['bo1'].base_filename == 'hello.txt'
        assert formdata.data['bo1'].get_content() == b'HELLO WORLD'
        assert two_pubs.loggederror_class.count() == 1
        logged_error = two_pubs.loggederror_class.select()[0]
        assert logged_error.summary.startswith('Failed to convert')
        assert logged_error.formdata_id == str(formdata.id)
        assert logged_error.exception_class == 'ValueError'

    # check wrong field
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo3', 'value': '=form_var_file_raw'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo3') is None


def test_set_backoffice_field_item(two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        ItemField(
            id='bo1',
            label='1st backoffice field',
            type='item',
            varname='backoffice_item',
            items=['a', 'b', 'c'],
        ),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': 'a'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'a'
    assert formdata.data['bo1_display'] == 'a'

    datasource = {'type': 'formula', 'value': repr([('a', 'aa'), ('b', 'bb'), ('c', 'cc')])}

    wf.backoffice_fields_formdef.fields = [
        ItemField(
            id='bo1',
            label='1st backoffice field',
            type='item',
            varname='backoffice_item',
            data_source=datasource,
        ),
    ]
    wf.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': 'a'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'a'
    assert formdata.data['bo1_display'] == 'aa'

    datasource = {
        'type': 'formula',
        'value': repr([{'id': 'a', 'text': 'aa', 'more': 'aaa'}, {'id': 'b', 'text': 'bb', 'more': 'bbb'}]),
    }

    wf.backoffice_fields_formdef.fields = [
        ItemField(
            id='bo1',
            label='1st backoffice field',
            type='item',
            varname='backoffice_item',
            data_source=datasource,
        ),
    ]
    wf.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': 'a'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'a'
    assert formdata.data['bo1_display'] == 'aa'
    assert formdata.data['bo1_structured'] == {'id': 'a', 'more': 'aaa', 'text': 'aa'}

    # check when assigning using the display value
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': 'aa'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'a'
    assert formdata.data['bo1_display'] == 'aa'
    assert formdata.data['bo1_structured'] == {'id': 'a', 'more': 'aaa', 'text': 'aa'}

    # check with unknown value
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': 'foobar'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == 'foobar'
    assert formdata.data.get('bo1_display') is None
    assert formdata.data.get('bo1_structured') is None


def test_set_backoffice_field_card_item(two_pubs):
    CardDef.wipe()
    Workflow.wipe()
    FormDef.wipe()

    carddef = CardDef()
    carddef.name = 'items'
    carddef.digest_templates = {'default': '{{form_var_name}}'}
    carddef.fields = [
        StringField(id='0', label='string', varname='name'),
        StringField(id='1', label='string', varname='attr'),
    ]
    carddef.store()
    carddef.data_class().wipe()
    for i, value in enumerate(['foo', 'bar', 'baz']):
        carddata = carddef.data_class()()
        carddata.data = {
            '0': value,
            '1': 'attr%s' % i,
        }
        carddata.just_created()
        carddata.store()
    latest_carddata = carddata
    latest_carddata_id = carddata.id
    ds = {'type': 'carddef:%s' % carddef.url_name}

    wf = Workflow(name='xxx')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.workflow_id = wf.id
    formdef.fields = [
        ItemField(id='0', label='string', type='item', data_source=ds, display_disabled_items=True)
    ]
    formdef.store()

    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        ItemField(
            id='bo1', label='1st backoffice field', type='item', varname='backoffice_item', data_source=ds
        ),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': str(latest_carddata_id)}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == str(latest_carddata_id)
    assert formdata.data['bo1_display'] == 'baz'
    assert formdata.data['bo1_structured']['attr'] == 'attr2'

    # reset, and get by display id value
    formdata.data = {}
    formdata.store()
    item.fields = [{'field_id': 'bo1', 'value': latest_carddata.get_display_id()}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == str(latest_carddata_id)
    assert formdata.data['bo1_display'] == 'baz'
    assert formdata.data['bo1_structured']['attr'] == 'attr2'

    # reset, and get by text value
    formdata.data = {}
    formdata.store()
    item.fields = [{'field_id': 'bo1', 'value': 'bar'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] != str(latest_carddata_id)
    assert formdata.data['bo1_display'] == 'bar'
    assert formdata.data['bo1_structured']['attr'] == 'attr1'

    # reset, with unknown value
    two_pubs.loggederror_class.wipe()
    formdata.data = {}
    formdata.store()
    item.fields = [{'field_id': 'bo1', 'value': 'xxx'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') is None  # invalid value is not stored
    assert formdata.data.get('bo1_display') is None
    assert formdata.data.get('bo1_structured') is None
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary.startswith('Failed to convert')

    # reset, and get empty value
    formdata.data = {}
    formdata.store()
    item.fields = [{'field_id': 'bo1', 'value': ''}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] is None
    assert formdata.data.get('bo1_display') is None
    assert formdata.data.get('bo1_structured') is None


def test_set_backoffice_field_items(two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        ItemsField(
            id='bo1',
            label='1st backoffice field',
            type='items',
            varname='backoffice_item',
            items=['a', 'b', 'c'],
        ),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=['a', 'b']"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ['a', 'b']
    assert formdata.data['bo1_display'] == 'a, b'

    datasource = {'type': 'formula', 'value': repr([('a', 'aa'), ('b', 'bb'), ('c', 'cc')])}

    wf.backoffice_fields_formdef.fields = [
        ItemsField(
            id='bo1',
            label='1st backoffice field',
            type='items',
            varname='backoffice_item',
            data_source=datasource,
        ),
    ]
    wf.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=['a', 'b']"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ['a', 'b']
    assert formdata.data['bo1_display'] == 'aa, bb'

    datasource = {
        'type': 'formula',
        'value': repr(
            [
                {'id': 'a', 'text': 'aa', 'more': 'aaa'},
                {'id': 'b', 'text': 'bb', 'more': 'bbb'},
                {'id': 'c', 'text': 'cc', 'more': 'ccc'},
            ]
        ),
    }

    wf.backoffice_fields_formdef.fields = [
        ItemsField(
            id='bo1',
            label='1st backoffice field',
            type='items',
            varname='backoffice_item',
            data_source=datasource,
        ),
    ]
    wf.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=['a', 'c']"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ['a', 'c']
    assert formdata.data['bo1_display'] == 'aa, cc'
    assert len(formdata.data['bo1_structured']) == 2
    assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
    assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']

    # from formdata field
    formdef.fields = [
        ItemsField(id='1', label='field', type='items', varname='items', data_source=datasource),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': ['a', 'c']}
    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=form_var_items_raw"}]
    item.perform(formdata)

    assert formdata.data['bo1'] == ['a', 'c']
    assert formdata.data['bo1_display'] == 'aa, cc'
    assert len(formdata.data['bo1_structured']) == 2
    assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
    assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']

    # with a template
    formdata = formdef.data_class()()
    formdata.data = {'1': ['a', 'c']}
    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.just_created()
    formdata.store()
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)

    item.fields = [{'field_id': 'bo1', 'value': "{{form_var_items_raw}}"}]
    item.perform(formdata)

    # using a single int
    datasource = {
        'type': 'formula',
        'value': repr(
            [
                {'id': 1, 'text': 'aa', 'more': 'aaa'},
                {'id': 2, 'text': 'bb', 'more': 'bbb'},
                {'id': 3, 'text': 'cc', 'more': 'ccc'},
            ]
        ),
    }

    wf.backoffice_fields_formdef.fields = [
        ItemsField(
            id='bo1',
            label='1st backoffice field',
            type='items',
            varname='backoffice_item',
            data_source=datasource,
        ),
    ]
    wf.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=2"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ['2']
    assert formdata.data['bo1_display'] == 'bb'
    assert len(formdata.data['bo1_structured']) == 1

    # using an invalid value
    formdata.data = {}
    formdata.store()
    two_pubs.loggederror_class.wipe()
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=Ellipsis"}]
    item.perform(formdata)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert 'Failed to convert' in logged_error.summary

    # using a string with multiple values
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "1|3"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == ['1', '3']
    assert formdata.data['bo1_display'] == 'aa, cc'
    assert len(formdata.data['bo1_structured']) == 2
    assert {'id': 1, 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
    assert {'id': 3, 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']


def test_set_backoffice_field_date(two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        DateField(id='bo1', label='1st backoffice field', type='date', varname='backoffice_date'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = []
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "=utils.today()"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert datetime.date(*formdata.data['bo1'][:3]) == datetime.date.today()

    formdata.data['bo1'] = None
    formdata.store()
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% now "j/n/Y" %}'}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert datetime.date(*formdata.data['bo1'][:3]) == datetime.date.today()

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': "23/3/2017"}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert datetime.date(*formdata.data['bo1'][:3]) == datetime.date(2017, 3, 23)

    # invalid values => do nothing
    assert two_pubs.loggederror_class.count() == 0
    for value in ('plop', '={}', '=[]'):
        item = SetBackofficeFieldsWorkflowStatusItem()
        item.parent = st1
        item.fields = [{'field_id': 'bo1', 'value': value}]

        two_pubs.loggederror_class.wipe()
        item.perform(formdata)
        formdata = formdef.data_class().get(formdata.id)
        assert datetime.date(*formdata.data['bo1'][:3]) == datetime.date(2017, 3, 23)
        assert two_pubs.loggederror_class.count() == 1
        assert two_pubs.loggederror_class.select()[0].summary.startswith('Failed to convert')

    # None : empty date
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': None}]
    item.perform(formdata)

    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] is None


def test_set_backoffice_field_boolean(two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        BoolField(id='bo1', label='1st backoffice field', type='bool', varname='backoffice_bool'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [BoolField(id='1', label='field', type='bool', varname='foo')]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': True}
    formdata.just_created()
    formdata.store()
    get_publisher().substitutions.feed(formdata)

    for value in ('=True', '=form_var_foo_raw', '{{ form_var_foo_raw }}', 'True', 'Yes', 'true', 'yes'):
        item = SetBackofficeFieldsWorkflowStatusItem()
        item.parent = st1
        item.fields = [{'field_id': 'bo1', 'value': value}]
        item.perform(formdata)
        formdata = formdef.data_class().get(formdata.id)
        assert formdata.data['bo1'] is True
        formdata.data['bo1'] = None
        formdata.store()

    for value in ('=False', '=not(form_var_foo_raw)', 'False', 'plop', ''):
        item = SetBackofficeFieldsWorkflowStatusItem()
        item.parent = st1
        item.fields = [{'field_id': 'bo1', 'value': value}]
        item.perform(formdata)
        formdata = formdef.data_class().get(formdata.id)
        assert formdata.data['bo1'] is False
        formdata.data['bo1'] = None
        formdata.store()


def test_set_backoffice_field_str_time_filter(two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='1st backoffice field', type='str', varname='backoffice_str'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [StringField(id='1', label='field', type='str', varname='foo')]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': '09:00'}
    formdata.just_created()
    formdata.store()
    get_publisher().substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_foo|time:"H:i:s" }}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == '09:00:00'
    formdata.data['bo1'] = None
    formdata.store()

    # |time will yield the default django reprentation
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{{ form_var_foo|time }}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == '9 a.m.'
    formdata.data['bo1'] = None
    formdata.store()


def test_set_backoffice_field_block(two_pubs):
    BlockDef.wipe()
    Workflow.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.digest_template = 'X{{foobar_var_foo}}Y'
    block.fields = [
        StringField(id='123', required=True, label='Test', type='string', varname='foo'),
        StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        BlockField(id='bo1', label='1st backoffice field', type='block:foobar'),
        StringField(id='bo2', label='2nd backoffice field', type='string'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='foo'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    # value from test_block_digest in tests/test_form_pages.py
    formdata.data = {
        '1': {
            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
            'schema': {'123': 'string', '234': 'string'},
        },
        '1_display': 'XfooY, Xfoo2Y',
    }
    formdata.just_created()
    formdata.store()
    get_publisher().substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_foo_raw}}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == formdata.data['1']
    assert formdata.data['bo1_display'] == formdata.data['1_display']

    # without _raw suffix
    formdata = formdef.data_class()()
    # value from test_block_digest in tests/test_form_pages.py
    formdata.data = {
        '1': {
            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
            'schema': {'123': 'string', '234': 'string'},
        },
        '1_display': 'XfooY, Xfoo2Y',
    }
    formdata.just_created()
    formdata.store()
    get_publisher().substitutions.reset()
    get_publisher().substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [
        {'field_id': 'bo1', 'value': '{{form_var_foo}}'},
        {'field_id': 'bo2', 'value': '{{form_var_foo}}'},
    ]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == formdata.data['1']
    assert formdata.data['bo1_display'] == formdata.data['1_display']
    assert formdata.data['bo2'] == formdata.data['1_display']


def test_set_backoffice_field_block_template_tag(two_pubs):
    BlockDef.wipe()
    Workflow.wipe()
    FormDef.wipe()

    block = BlockDef()
    block.name = 'foobar'
    block.digest_template = 'X{{foobar_var_foo}}Y'
    block.fields = [
        StringField(id='123', required=True, label='Test', type='string', varname='foo'),
        StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
    ]
    block.store()

    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    st1 = wf.add_status('Status1')
    wf.backoffice_fields_formdef.fields = [
        BlockField(id='bo1', label='1st backoffice field', max_items=3, type='block:foobar'),
    ]
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='test', varname='foo'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'plop'}
    formdata.just_created()
    formdata.store()
    get_publisher().substitutions.feed(formdata)

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value foo=form_var_foo bar="xxx" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'plop', '234': 'xxx'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XplopY'

    # override
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value foo=form_var_foo bar="yyy" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'plop', '234': 'yyy'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XplopY'

    # append
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value append=True foo="zzz" bar=form_var_foo %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'plop', '234': 'yyy'}, {'123': 'zzz', '234': 'plop'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XplopY, XzzzY'

    # merge (into last row)
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value merge=True foo="AAA" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'plop', '234': 'yyy'}, {'123': 'AAA', '234': 'plop'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XplopY, XAAAY'

    # merge (into given row)
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value merge=0 foo="BBB" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'BBB', '234': 'yyy'}, {'123': 'AAA', '234': 'plop'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XBBBY, XAAAY'

    # merge with indexerror (ignored)
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value merge=50 foo="CCC" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [{'123': 'BBB', '234': 'yyy'}, {'123': 'AAA', '234': 'plop'}],
        'schema': {'123': 'string', '234': 'string'},
    }
    assert formdata.data['bo1_display'] == 'XBBBY, XAAAY'

    # "item" subfield, make sure raw and display and structured values are stored
    datasource = {
        'type': 'formula',
        'value': repr(
            [
                {'id': 'a', 'text': 'aa', 'more': 'aaa'},
                {'id': 'b', 'text': 'bb', 'more': 'bbb'},
                {'id': 'c', 'text': 'cc', 'more': 'ccc'},
            ]
        ),
    }
    block.fields.append(
        ItemField(id='345', label='Test3', type='item', varname='item', data_source=datasource)
    )
    block.store()
    wf.backoffice_fields_formdef.fields[0]._block = None  # remove cache

    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value item="b" %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [
            {'345': 'b', '345_display': 'bb', '345_structured': {'id': 'b', 'text': 'bb', 'more': 'bbb'}}
        ],
        'schema': {'123': 'string', '234': 'string', '345': 'item'},
    }
    assert formdata.data['bo1_display'] == 'XNoneY'

    # append to invalid existing value (should not happen)
    formdata.data['bo1'] = {'invalid': 'value'}
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1
    item.fields = [{'field_id': 'bo1', 'value': '{% block_value item="b" append=True %}'}]
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data['bo1'] == {
        'data': [
            {'345': 'b', '345_display': 'bb', '345_structured': {'id': 'b', 'text': 'bb', 'more': 'bbb'}}
        ],
        'schema': {'123': 'string', '234': 'string', '345': 'item'},
    }
    assert formdata.data['bo1_display'] == 'XNoneY'


def test_set_backoffice_field_immediate_use(http_requests, two_pubs):
    Workflow.wipe()
    FormDef.wipe()
    wf = Workflow(name='xxx')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='1st backoffice field', type='string', varname='backoffice_blah'),
        StringField(id='bo2', label='2nd backoffice field', type='string', varname='backoffice_barr'),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='00', label='String', type='string', varname='string'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'00': 'HELLO'}
    formdata.just_created()
    formdata.store()
    item = SetBackofficeFieldsWorkflowStatusItem()
    item.parent = st1

    item.fields = [
        {'field_id': 'bo1', 'value': '{{form_var_string}}'},
    ]
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') == 'HELLO'

    item.fields = [
        {'field_id': 'bo1', 'value': 'WORLD'},
    ]
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') == 'WORLD'

    item.fields = [
        {'field_id': 'bo1', 'value': 'X{{form_var_string}}X'},
        {'field_id': 'bo2', 'value': "Y{{form_var_backoffice_blah}}Y"},
    ]
    two_pubs.substitutions.reset()
    two_pubs.substitutions.feed(formdata)
    item.perform(formdata)
    formdata = formdef.data_class().get(formdata.id)
    assert formdata.data.get('bo1') == 'XHELLOX'
    assert formdata.data.get('bo2') == 'YXHELLOXY'


def test_redirect_to_url(pub):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'bar'}

    item = RedirectToUrlWorkflowStatusItem()
    assert item.render_as_line() == 'Web Redirection (not configured)'
    item.url = 'https://www.example.net/?foo=[form_var_foo]'
    assert item.render_as_line() == 'Web Redirection (to https://www.example.net/?foo=[form_var_foo])'
    pub.substitutions.feed(formdata)
    assert item.perform(formdata) == 'https://www.example.net/?foo=bar'

    item.url = 'https://www.example.net/?django={{ form_var_foo }}'
    assert item.render_as_line() == 'Web Redirection (to https://www.example.net/?django={{ form_var_foo }})'
    pub.substitutions.feed(formdata)
    assert item.perform(formdata) == 'https://www.example.net/?django=bar'

    item.url = '[if-any nada]https://www.example.net/[end]'
    pub.substitutions.feed(formdata)
    assert item.perform(formdata) is None

    item.url = '{% if nada %}https://www.example.net/{% endif %}'
    pub.substitutions.feed(formdata)
    assert item.perform(formdata) is None


def test_workflow_jump_condition_migration(pub):
    workflow = Workflow(name='jump condition migration')
    st1 = workflow.add_status('Status1', 'st1')

    jump = st1.add_action('jump', id='_jump')
    workflow.store()

    reloaded_workflow = Workflow.get(workflow.id)
    assert reloaded_workflow.possible_status[0].items[0].condition is None

    jump.condition = 'foobar'
    workflow.store()
    reloaded_workflow = Workflow.get(workflow.id)
    assert reloaded_workflow.possible_status[0].items[0].condition == {'type': 'python', 'value': 'foobar'}


def test_workflow_action_condition(two_pubs):
    two_pubs._set_request(None)  # to avoid after jobs
    workflow = Workflow(name='jump condition migration')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.store()

    role = two_pubs.role_class(name='bar1')
    role.store()

    user = two_pubs.user_class()
    user.roles = [role.id]
    user.store()

    choice = st1.add_action('choice', id='_x')
    choice.by = [role.id]
    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    formdef.workflow_id = workflow.id
    formdef.store()

    formdef.data_class().wipe()

    formdata1 = formdef.data_class()()
    formdata1.data = {'1': 'foo'}
    formdata1.just_created()
    formdata1.store()

    formdata2 = formdef.data_class()()
    formdata2.data = {'2': 'bar'}
    formdata2.just_created()
    formdata2.store()

    assert formdata1.get_actions_roles() == {role.id}
    assert formdata2.get_actions_roles() == {role.id}

    assert len(FormDef.get(formdef.id).data_class().get_actionable_ids([role.id])) == 2

    choice.condition = {'type': 'python', 'value': 'form_var_foo == "foo"'}
    workflow.store()

    with two_pubs.substitutions.temporary_feed(formdata1):
        assert FormDef.get(formdef.id).data_class().get(formdata1.id).get_actions_roles() == {role.id}
    with two_pubs.substitutions.temporary_feed(formdata2):
        assert FormDef.get(formdef.id).data_class().get(formdata2.id).get_actions_roles() == set()

    assert len(FormDef.get(formdef.id).data_class().get_actionable_ids([role.id])) == 1

    # check with a formdef condition
    choice.condition = {'type': 'python', 'value': 'form_name == "test"'}
    workflow.store()
    assert len(FormDef.get(formdef.id).data_class().get_actionable_ids([role.id])) == 0

    choice.condition = {'type': 'python', 'value': 'form_name == "baz"'}
    workflow.store()
    assert len(FormDef.get(formdef.id).data_class().get_actionable_ids([role.id])) == 2

    # bad condition
    two_pubs.loggederror_class.wipe()
    choice.condition = {'type': 'python', 'value': 'foobar == barfoo'}
    workflow.store()
    assert len(FormDef.get(formdef.id).data_class().get_actionable_ids([role.id])) == 0
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.occurences_count > 1  # should be 2... == 12 with pickle, 4 with sql
    assert logged_error.summary == 'Failed to evaluate condition'
    assert logged_error.exception_class == 'NameError'
    assert logged_error.exception_message == "name 'foobar' is not defined"
    assert logged_error.expression == 'foobar == barfoo'
    assert logged_error.expression_type == 'python'


def test_workflow_field_migration(pub):
    Workflow.wipe()
    wf = Workflow(name='wf with backoffice field')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string', in_listing=True),
    ]
    wf.add_status('Status1')
    wf.store()

    wf = Workflow.get(wf.id)
    assert wf.backoffice_fields_formdef.fields[0].display_locations == ['validation', 'summary', 'listings']


def test_aggregation_email(pub, emails):
    Workflow.wipe()
    pub.role_class.wipe()
    AggregationEmail.wipe()

    role = pub.role_class(name='foobar')
    role.emails = ['foobar@localhost']
    role.emails_to_members = False
    role.store()

    workflow = Workflow(name='aggregation-email')
    workflow.possible_status = Workflow.get_default_workflow().possible_status[:]
    aggregation = workflow.possible_status[1].add_action('aggregationemail', prepend=True)
    assert aggregation.get_line_details() == 'not completed'
    aggregation.to = [role.id]
    assert aggregation.get_line_details() == 'to foobar'
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.fields = []
    formdef.workflow_id = workflow.id
    formdef.store()
    formdef.data_class().wipe()

    for i in range(5):
        formdata = formdef.data_class()()
        formdata.data = {}
        formdata.store()
        formdata.just_created()
        formdata.perform_workflow()
        assert AggregationEmail.count() == 1

    send_aggregation_emails(pub)
    assert AggregationEmail.count() == 0
    assert 'New arrivals' in emails.emails
    for i in range(5):
        assert (
            'http://example.net/foobar/%s/status (New)' % (i + 1) in emails.emails['New arrivals']['payload']
        )

    emails.empty()
    send_aggregation_emails(pub)
    assert 'New arrivals' not in emails.emails

    role.emails = []
    role.emails_to_members = True
    role.store()

    user = pub.user_class(name='bar')
    user.email = 'bar@localhost'
    user.roles = [role.id]
    user.store()

    formdata.perform_workflow()
    assert AggregationEmail.count() == 1

    send_aggregation_emails(pub)
    assert AggregationEmail.count() == 0
    assert 'New arrivals' in emails.emails
    assert (
        'http://example.net/foobar/%s/status (New)' % formdata.id in emails.emails['New arrivals']['payload']
    )


def test_call_external_workflow_with_evolution_linked_object(two_pubs):
    FormDef.wipe()
    CardDef.wipe()

    external_wf = Workflow(name='External Workflow')
    st1 = external_wf.add_status(name='New')
    action = external_wf.add_global_action('Delete', 'delete')
    action.add_action('remove')
    trigger = action.append_trigger('webservice')
    trigger.identifier = 'delete'
    external_wf.store()

    external_formdef = FormDef()
    external_formdef.name = 'External Form'
    external_formdef.fields = [
        StringField(id='0', label='string', varname='form_string'),
    ]
    external_formdef.workflow = external_wf
    external_formdef.store()

    external_carddef = CardDef()
    external_carddef.name = 'External Card'
    external_carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    external_carddef.workflow = external_wf
    external_carddef.store()

    wf = Workflow(name='External actions')
    st1 = wf.add_status('Create external formdata')
    create_formdata = st1.add_action('create_formdata', id='_create_form')
    create_formdata.label = 'create linked form'
    create_formdata.formdef_slug = external_formdef.url_name
    create_formdata.varname = 'created_form'
    mappings = [Mapping(field_id='0', expression='{{ form_var_string }}')]
    create_formdata.mappings = mappings

    create_carddata = st1.add_action('create_carddata', id='_create_card')
    create_carddata.label = 'create linked card'
    create_carddata.formdef_slug = external_carddef.url_name
    create_carddata.varname = 'created_card'
    create_carddata.mappings = mappings

    global_action = wf.add_global_action('Delete external linked object', 'delete')
    action = global_action.add_action('external_workflow_global_action')
    action.slug = 'formdef:%s' % external_formdef.url_name
    action.trigger_id = 'action:%s' % trigger.identifier
    wf.store()

    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [
        StringField(id='0', label='string', varname='string'),
    ]
    formdef.workflow = wf
    formdef.store()

    assert external_formdef.data_class().count() == 0
    assert external_carddef.data_class().count() == 0

    formdata = formdef.data_class()()
    formdata.data = {'0': 'test form'}
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()

    assert external_formdef.data_class().count() == 1
    assert external_carddef.data_class().count() == 1

    # remove external formdata
    perform_items([action], formdata)
    assert external_formdef.data_class().count() == 0
    assert external_carddef.data_class().count() == 1

    # formdata is already deleted: cannot find it again, no problem
    perform_items([action], formdata)

    # try remove an unexisting carddef: do nothing
    unused_carddef = CardDef()
    unused_carddef.name = 'External Card (not used)'
    unused_carddef.fields = []
    unused_carddef.workflow = external_wf
    unused_carddef.store()
    action.slug = 'carddef:%s' % unused_carddef.url_name
    wf.store()
    perform_items([action], formdata)
    assert external_formdef.data_class().count() == 0
    assert external_carddef.data_class().count() == 1
    # remove the right carddef
    action.slug = 'carddef:%s' % external_carddef.url_name
    wf.store()
    perform_items([action], formdata)
    assert external_formdef.data_class().count() == 0
    assert external_carddef.data_class().count() == 0


def test_call_external_workflow_with_data_sourced_object(two_pubs, admin_user):
    FormDef.wipe()
    CardDef.wipe()

    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.add_status('status')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    update_action = carddef_wf.add_global_action('Update', 'ac1')
    update_action.add_action('set-backoffice-fields')
    setbo = update_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ form_var_bo|default:"0"|add:1 }}'}]
    trigger = update_action.append_trigger('webservice')
    trigger.identifier = 'update'

    delete = carddef_wf.add_global_action('Delete', 'delete')
    delete.add_action('remove')
    trigger = delete.append_trigger('webservice')
    trigger.identifier = 'delete'
    carddef_wf.store()

    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.digest_templates = {'default': '{{ form_var_card_string }}'}
    carddef.workflow = carddef_wf
    carddef.store()

    carddata = carddef.data_class()()
    carddata.data = {'0': 'Text'}
    carddata.just_created()
    carddata.store()

    wf = Workflow(name='External actions')
    wf.add_status('Action')

    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.trigger_id = 'action:update'

    delete_global_action = wf.add_global_action('Delete external linked object', 'delete')
    delete_action = delete_global_action.add_action('external_workflow_global_action')
    delete_action.slug = 'carddef:%s' % carddef.url_name
    delete_action.trigger_id = 'action:delete'
    wf.store()

    datasource = {'type': 'carddef:%s' % carddef.url_name}
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [
        ItemField(id='0', label='Card', type='item', varname='card', data_source=datasource),
        EmailField(id='1', label='Email', varname='email'),
    ]
    formdef.workflow = wf
    formdef.store()

    assert carddef.data_class().count() == 1

    formdata = formdef.data_class()()
    formdata.data = {'0': '1', '1': 'foo@example.com'}
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()

    perform_items([update_action], formdata)
    assert carddef.data_class().count() == 1
    data = carddef.data_class().select()[0]
    assert data.data['bo0'] == '1'

    assert data.evolution[-1].parts[-1].event == 'global-external-workflow'
    resp = login(get_app(pub), username='admin', password='admin').get(data.get_backoffice_url() + 'inspect')
    # check tracing link is correct:
    assert '/global-actions/ac1/items/1/' in resp.text

    perform_items([update_action], formdata)
    data = carddef.data_class().select()[0]
    assert data.data['bo0'] == '2'

    perform_items([delete_action], formdata)
    assert carddef.data_class().count() == 0

    # linked object is removed: no problem
    perform_items([delete_action], formdata)


def test_call_external_workflow_with_items_data_sourced_object(two_pubs, admin_user):
    FormDef.wipe()
    CardDef.wipe()

    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.add_status('status')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    update_action = carddef_wf.add_global_action('Update', 'ac1')
    update_action.add_action('set-backoffice-fields')
    setbo = update_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ form_var_bo|default:"0"|add:1 }}'}]
    trigger = update_action.append_trigger('webservice')
    trigger.identifier = 'update'
    carddef_wf.store()

    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.digest_templates = {'default': '{{ form_var_card_string }}'}
    carddef.workflow = carddef_wf
    carddef.store()

    carddata = carddef.data_class()()
    carddata.data = {'0': 'Text1'}
    carddata.just_created()
    carddata.store()

    carddata2 = carddef.data_class()()
    carddata2.data = {'0': 'Text2'}
    carddata2.just_created()
    carddata2.store()

    carddata3 = carddef.data_class()()
    carddata3.data = {'0': 'Text3'}
    carddata3.just_created()
    carddata3.store()

    wf = Workflow(name='External actions')
    wf.add_status('Action')

    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.trigger_id = 'action:update'

    wf.store()

    datasource = {'type': 'carddef:%s' % carddef.url_name}
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [
        ItemsField(id='0', label='Cards', type='items', varname='cards', data_source=datasource),
        EmailField(id='1', label='Email', varname='email'),
    ]
    formdef.workflow = wf
    formdef.store()

    assert carddef.data_class().count() == 3

    formdata = formdef.data_class()()
    formdata.data = {'0': [str(carddata.id), str(carddata3.id)], '1': 'foo@example.com'}
    formdata.data['0_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
    formdata.data['0_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()

    perform_items([update_action], formdata)
    assert carddef.data_class().count() == 3
    carddata.refresh_from_storage()
    assert carddata.data['bo0'] == '1'
    carddata2.refresh_from_storage()
    assert not carddata2.data.get('bo0')
    carddata3.refresh_from_storage()
    assert carddata3.data['bo0'] == '1'

    perform_items([update_action], formdata)
    carddata.refresh_from_storage()
    assert carddata.data['bo0'] == '2'
    carddata2.refresh_from_storage()
    assert not carddata2.data.get('bo0')
    carddata3.refresh_from_storage()
    assert carddata3.data['bo0'] == '2'


def test_call_external_workflow_with_parent_object(pub):
    FormDef.wipe()
    CardDef.wipe()

    # carddef workflow, with global action to increment a counter in its
    # backoffice fields.
    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    increment_global_action = carddef_wf.add_global_action('Update')
    increment_global_action.add_action('set-backoffice-fields')
    setbo = increment_global_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ form_var_bo|default:"0"|add:1 }}'}]
    trigger = increment_global_action.append_trigger('webservice')
    trigger.identifier = 'update'
    carddef_wf.store()

    # associated carddef
    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.workflow = carddef_wf
    carddef.store()

    # and sample carddata
    carddata = carddef.data_class()()
    carddata.data = {'0': 'Text'}
    carddata.store()

    # formdef workflow that will trigger the global action
    wf = Workflow(name='External actions')
    wf.add_status('Action')

    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.trigger_id = 'action:update'
    wf.store()

    # associated formdef
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [EmailField(id='1', label='Email', varname='email')]
    formdef.workflow = wf
    formdef.store()

    # and formdata
    formdata = formdef.data_class()()
    formdata.data = {'1': 'foo@example.com'}
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()

    # run, against no parent
    perform_items([update_action], formdata)
    card = carddef.data_class().get(carddata.id)
    assert not card.data.get('bo0')  # not called

    # other parent
    formdata.submission_context = {
        'orig_object_type': 'formdef',
        'orig_formdata_id': str(formdata.id),
        'orig_formdef_id': str(formdef.id),
    }
    formdata.store()
    perform_items([update_action], formdata)
    card = carddef.data_class().get(carddata.id)
    assert not card.data.get('bo0')  # not called

    # appropriate parent
    formdata.submission_context = {
        'orig_object_type': 'carddef',
        'orig_formdata_id': str(carddata.id),
        'orig_formdef_id': str(carddef.id),
    }
    formdata.store()
    perform_items([update_action], formdata)
    card = carddef.data_class().get(carddata.id)
    assert card.data['bo0'] == '1'  # got called


def test_call_external_workflow_use_caller_variable(pub):
    FormDef.wipe()
    CardDef.wipe()

    # carddef workflow, with global action to set a value in a backoffice field
    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    global_action = carddef_wf.add_global_action('Update')
    global_action.add_action('set-backoffice-fields')
    setbo = global_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ caller_form_var_email }}'}]
    trigger = global_action.append_trigger('webservice')
    trigger.identifier = 'update'
    carddef_wf.store()

    # associated carddef
    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.workflow = carddef_wf
    carddef.store()

    # and sample carddata
    carddata = carddef.data_class()()
    carddata.data = {'0': 'Text'}
    carddata.store()

    # formdef workflow that will trigger the global action
    wf = Workflow(name='External actions')
    wf.add_status('Action')

    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.trigger_id = 'action:update'
    wf.store()

    # associated formdef
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [EmailField(id='1', label='Email', varname='email')]
    formdef.workflow = wf
    formdef.store()

    # and formdata
    formdata = formdef.data_class()()
    formdata.data = {'1': 'foo@example.com'}
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()

    # run, against no parent
    perform_items([update_action], formdata)
    card = carddef.data_class().get(carddata.id)
    assert not card.data.get('bo0')  # not called

    # appropriate parent
    formdata.submission_context = {
        'orig_object_type': 'carddef',
        'orig_formdata_id': str(carddata.id),
        'orig_formdef_id': str(carddef.id),
    }
    formdata.store()
    perform_items([update_action], formdata)
    card = carddef.data_class().get(carddata.id)
    assert card.data['bo0'] == 'foo@example.com'  # got called


def test_call_external_workflow_manual_targeting(two_pubs):
    FormDef.wipe()
    CardDef.wipe()

    # carddef workflow, with global action to increment a counter in its
    # backoffice fields.
    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.add_status(name='New')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    global_action = carddef_wf.add_global_action('Update')
    global_action.add_action('set-backoffice-fields')
    setbo = global_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ form_var_bo|default:"0"|add:1 }}'}]
    trigger = global_action.append_trigger('webservice')
    trigger.identifier = 'update'
    carddef_wf.store()

    # associated carddef
    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.workflow = carddef_wf
    carddef.store()
    carddef.data_class().wipe()

    # and sample carddatas
    for i in range(1, 4):
        carddata = carddef.data_class()()
        carddata.data = {'0': 'Text %s' % i}
        carddata.store()

    # formdef workflow that will trigger the global action
    wf = Workflow(name='External actions')
    st1 = wf.add_status('Action')

    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.target_mode = 'manual'
    update_action.target_id = None  # not configured
    update_action.trigger_id = 'action:update'

    # and create carddata
    create_carddata = st1.add_action('create_carddata', id='_create_card')
    create_carddata.label = 'create linked card'
    create_carddata.formdef_slug = carddef.url_name
    create_carddata.varname = 'created_card'
    create_carddata.mappings = [Mapping(field_id='0', expression='{{ form_var_string }}')]

    wf.store()

    # associated formdef
    datasource = {'type': 'carddef:%s' % carddef.url_name}
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = [
        ItemField(id='0', label='Card', type='item', varname='card', data_source=datasource),
        StringField(id='1', label='string', varname='string'),
    ]
    formdef.workflow = wf
    formdef.store()

    # and formdata
    formdata = formdef.data_class()()
    formdata.data = {
        '0': '3',  # set from datasource
        '1': '1',
    }
    # set parent
    formdata.submission_context = {
        'orig_object_type': 'carddef',
        'orig_formdata_id': '2',
        'orig_formdef_id': str(carddef.id),
    }
    formdata.store()
    formdata.just_created()
    formdata.perform_workflow()
    assert carddef.data_class().count() == 4
    assert carddef.data_class().get(1).data['bo0'] is None
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None
    # linked carddata
    assert carddef.data_class().get(4).data['0'] == '1'

    # target not configured
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] is None
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None
    assert two_pubs.loggederror_class.count() == 0

    # configure target
    update_action.target_id = '{{ form_var_string }}'  # == '1'
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None

    # target not found
    update_action.target_id = '42{{ form_var_string }}'  # == '421'
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Could not find targeted "Data" object by id 421'

    # slug not or badly configured
    update_action.target_id = '{{ form_var_string }}'  # == '1'
    update_action.slug = None
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'  # not changed
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None
    update_action.slug = 'foo'
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'  # not changed
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None


def test_form_update_after_backoffice_fields(sql_pub):
    wf = Workflow(name='wf with backoffice field')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string'),
    ]
    wf.add_status('Status1')
    wf.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.fields = [
        StringField(id='1', label='field 1', type='string'),
    ]
    formdef.workflow_id = wf.id
    formdef.store()

    conn, cur = sql.get_connection_and_cursor()
    assert column_exists_in_table(cur, formdef.table_name, 'fbo1')

    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo1', label='bo field 1', type='string'),
        StringField(id='bo2', label='bo field 2', type='string'),
    ]
    wf.backoffice_fields_formdef.store()
    get_response().process_after_jobs()
    assert column_exists_in_table(cur, formdef.table_name, 'fbo2')

    # remove first and add third field
    wf.backoffice_fields_formdef.fields = [
        StringField(id='bo2', label='bo field 2', type='string'),
        StringField(id='bo3', label='bo field 3', type='string'),
    ]
    wf.backoffice_fields_formdef.store()
    get_response().process_after_jobs()
    assert not column_exists_in_table(cur, formdef.table_name, 'fbo1')
    assert column_exists_in_table(cur, formdef.table_name, 'fbo3')

    conn.commit()
    cur.close()


def test_call_external_workflow_manual_queryset_targeting(two_pubs):
    FormDef.wipe()
    CardDef.wipe()

    # carddef workflow, with global action to increment a counter in its
    # backoffice fields.
    carddef_wf = Workflow(name='Carddef Workflow')
    carddef_wf.add_status(name='New')
    carddef_wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(carddef_wf)
    carddef_wf.backoffice_fields_formdef.fields = [
        StringField(id='bo0', varname='bo', type='string', label='bo variable'),
    ]
    global_action = carddef_wf.add_global_action('Update')
    global_action.add_action('set-backoffice-fields')
    setbo = global_action.items[0]
    setbo.fields = [{'field_id': 'bo0', 'value': '{{ form_var_bo|default:"0"|add:1 }}'}]
    trigger = global_action.append_trigger('webservice')  # external call
    trigger.identifier = 'update'
    carddef_wf.store()

    # associated carddef
    carddef = CardDef()
    carddef.name = 'Data'
    carddef.fields = [
        StringField(id='0', label='string', varname='card_string'),
    ]
    carddef.workflow = carddef_wf
    carddef.store()
    carddef.data_class().wipe()

    # and sample carddatas
    for i in range(1, 5):
        carddata = carddef.data_class()()
        carddata.data = {'0': 'Text %s' % i}
        carddata.store()
        carddata.just_created()
        carddata.store()

    # formdef workflow that will trigger the global action
    wf = Workflow(name='External actions')
    wf.add_status('Blah')
    update_global_action = wf.add_global_action('Update linked object data')
    update_action = update_global_action.add_action('external_workflow_global_action')
    update_action.slug = 'carddef:%s' % carddef.url_name
    update_action.target_mode = 'manual'
    update_action.target_id = None  # not configured
    update_action.trigger_id = 'action:update'
    wf.store()

    # associated formdef
    formdef = FormDef()
    formdef.name = 'External action form'
    formdef.fields = []
    formdef.workflow = wf
    formdef.store()

    # and formdata
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.store()
    formdata.just_created()

    # target not configured
    perform_items([update_action], formdata)
    assert carddef.data_class().count() == 4
    assert carddef.data_class().get(1).data['bo0'] is None
    assert carddef.data_class().get(2).data['bo0'] is None
    assert carddef.data_class().get(3).data['bo0'] is None
    assert carddef.data_class().get(4).data['bo0'] is None

    # target all cards
    update_action.target_id = '{{cards|objects:"%s"}}' % carddef.url_name
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'
    assert carddef.data_class().get(2).data['bo0'] == '1'
    assert carddef.data_class().get(3).data['bo0'] == '1'
    assert carddef.data_class().get(4).data['bo0'] == '1'
    status_part = [x for x in formdata.evolution[-1].parts if isinstance(x, ManyExternalCallsPart)][0]
    assert status_part.running is False
    assert status_part.is_hidden() is True
    assert '4 processed' in str(status_part.view())
    assert set(status_part.processed_ids) == {x.get_display_id() for x in carddef.data_class().select()}

    # target some cards
    update_action.target_id = (
        '{{cards|objects:"%s"|filter_by:"card_string"|filter_value:"Text 2"}}' % carddef.url_name
    )
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'
    assert carddef.data_class().get(2).data['bo0'] == '2'
    assert carddef.data_class().get(3).data['bo0'] == '1'
    assert carddef.data_class().get(4).data['bo0'] == '1'

    # target a single formdata
    update_action.target_id = (
        '{{cards|objects:"%s"|filter_by:"card_string"|filter_value:"Text 2"|first}}' % carddef.url_name
    )
    wf.store()
    perform_items([update_action], formdata)
    assert carddef.data_class().get(1).data['bo0'] == '1'
    assert carddef.data_class().get(2).data['bo0'] == '3'
    assert carddef.data_class().get(3).data['bo0'] == '1'
    assert carddef.data_class().get(4).data['bo0'] == '1'

    # mismatch in target
    carddef2 = CardDef()
    carddef2.name = 'Other data'
    carddef2.fields = []
    carddef2.workflow = carddef_wf
    carddef2.store()

    update_action.slug = 'carddef:%s' % carddef2.url_name
    update_action.target_id = (
        '{{cards|objects:"%s"|filter_by:"card_string"|filter_value:"Text 2"}}' % carddef.url_name
    )
    wf.store()
    perform_items([update_action], formdata)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Mismatch in target objects: expected "Other data", got "Data"'

    # mismatch in target, with formdata
    two_pubs.loggederror_class.wipe()
    update_action.target_id = (
        '{{cards|objects:"%s"|filter_by:"card_string"|filter_value:"Text 2"|first}}' % carddef.url_name
    )
    wf.store()
    perform_items([update_action], formdata)
    assert two_pubs.loggederror_class.count() == 1
    logged_error = two_pubs.loggederror_class.select()[0]
    assert logged_error.summary == 'Mismatch in target object: expected "Other data", got "Data"'


def test_conditional_jump_vs_tracing(pub):
    workflow = Workflow(name='wf')
    st1 = workflow.add_status('Status1', 'st1')
    workflow.add_status('Status2', 'st2')
    comment = st1.add_action('register-comment')
    comment.comment = 'hello world'
    jump1 = st1.add_action('jump')
    jump1.parent = st1
    jump1.condition = {'type': 'django', 'value': 'False'}
    jump1.status = 'wf-st2'
    jump2 = st1.add_action('jump')
    jump2.parent = st1
    jump2.status = 'wf-st2'
    workflow.store()

    formdef = FormDef()
    formdef.name = 'baz'
    formdef.workflow = workflow
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()
    perform_items(st1.items, formdata)
    formdata.refresh_from_storage()
    assert formdata.evolution[0].parts[-1].actions[0][1:] == ('register-comment', str(comment.id))
    assert formdata.evolution[0].parts[-1].actions[1][1:] == ('jump', str(jump2.id))


def test_remove_tracking_code(two_pubs):
    formdef = FormDef()
    formdef.name = 'baz'
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()
    formdata = formdef.data_class()()
    formdata.store()
    code = get_publisher().tracking_code_class()
    code.formdata = formdata
    formdata.refresh_from_storage()

    assert formdata.tracking_code
    assert get_publisher().tracking_code_class().count() == 1

    item = RemoveTrackingCodeWorkflowStatusItem()
    item.perform(formdata)
    assert not formdata.tracking_code
    assert get_publisher().tracking_code_class().count() == 0
    item.perform(formdata)  # do not crash if no tracking_code

    item.replace = True
    item.perform(formdata)
    assert formdata.tracking_code
    assert get_publisher().tracking_code_class().count() == 1
    tracking_code_orig = formdata.tracking_code
    item.perform(formdata)
    assert formdata.tracking_code
    assert formdata.tracking_code != tracking_code_orig
    assert get_publisher().tracking_code_class().count() == 1

    # cannot replace if formdef not handles tracking code
    formdef.enable_tracking_codes = False
    formdef.store()
    item.perform(formdata)
    assert not formdata.tracking_code
    assert get_publisher().tracking_code_class().count() == 0