wcs/tests/test_backoffice_pages.py

# -*- coding: utf-8 -*-
import datetime
import json
import os
import re
import shutil
import time
import hashlib
import random
import xml.etree.ElementTree as ET
import zipfile

import mock
import pytest

try:
    import xlwt
except ImportError:
    xlwt = None

from django.utils import six
from django.utils.six import StringIO, BytesIO
from django.utils.six.moves.urllib import parse as urllib
from django.utils.six.moves.urllib import parse as urlparse

from quixote import cleanup, get_publisher
from wcs.qommon import ods
from wcs.api_utils import sign_url
from wcs.qommon import errors, sessions
from wcs.qommon.form import PicklableUpload
from wcs.qommon.ident.password_accounts import PasswordAccount
from wcs.qommon.http_request import HTTPRequest
from wcs.roles import Role
from wcs.workflows import (Workflow, CommentableWorkflowStatusItem,
        ChoiceWorkflowStatusItem, EditableWorkflowStatusItem,
        DisplayMessageWorkflowStatusItem,
        JumpOnSubmitWorkflowStatusItem, WorkflowCriticalityLevel,
        WorkflowBackofficeFieldsFormDef)
from wcs.wf.backoffice_fields import SetBackofficeFieldsWorkflowStatusItem
from wcs.wf.dispatch import DispatchWorkflowStatusItem
from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
from wcs.wf.jump import JumpWorkflowStatusItem
from wcs.wf.wscall import WebserviceCallStatusItem
from wcs.wf.redirect_to_url import RedirectToUrlWorkflowStatusItem
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
from wcs.wf.resubmit import ResubmitWorkflowStatusItem
from wcs.carddef import CardDef
from wcs.categories import Category
from wcs.formdef import FormDef
from wcs.logged_errors import LoggedError
from wcs import fields
from wcs.wscalls import NamedWsCall

from utilities import (get_app, login, create_temporary_pub,
        clean_temporary_pub)


def pytest_generate_tests(metafunc):
    if 'pub' in metafunc.fixturenames:
        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates'], indirect=True)

@pytest.fixture
def pub(request, emails):
    pub = create_temporary_pub(
            sql_mode=bool('sql' in request.param),
            templates_mode=bool('templates' in request.param)
            )

    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
    pub.set_app_dir(req)
    pub.cfg['identification'] = {'methods': ['password']}
    pub.cfg['language'] = {'language': 'en'}
    pub.write_cfg()
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    fd.write('''
[api-secrets]
coucou = 1234
''')
    fd.close()

    return pub

def create_user(pub, is_admin=False):
    if pub.user_class.select(lambda x: x.name == 'admin'):
        user1 = pub.user_class.select(lambda x: x.name == 'admin')[0]
        user1.is_admin = is_admin
        user1.roles = [x.id for x in Role.select() if x.name == 'foobar']
        user1.store()
        return user1
    user1 = pub.user_class(name='admin')
    user1.email = 'admin@localhost'
    user1.is_admin = is_admin
    user1.store()

    account1 = PasswordAccount(id='admin')
    account1.set_password('admin')
    account1.user_id = user1.id
    account1.store()

    Role.wipe()
    role = Role(name='foobar')
    role.store()

    user1.roles = [role.id]
    user1.store()

    return user1

def create_superuser(pub): return create_user(pub, is_admin=True)

def create_environment(pub, set_receiver=True):
    pub.session_manager.session_class.wipe()
    Workflow.wipe()
    Category.wipe()
    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form title'
    if set_receiver:
        formdef.workflow_roles = {'_receiver': 1}

    datasource = {'type': 'formula',
            'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])}

    formdef.fields = [
            fields.StringField(id='1', label='1st field', type='string',
                display_locations=['validation', 'summary', 'listings']),
            fields.ItemField(id='2', label='2nd field', type='item',
                items=['foo', 'bar', 'baz'],
                display_locations=['validation', 'summary', 'listings']),
            fields.ItemField(id='3', label='3rd field', type='item',
                data_source=datasource, varname='foo'),
        ]

    formdef.store()
    formdef.data_class().wipe()
    for i in range(50):
        formdata = formdef.data_class()()
        formdata.just_created()
        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
        formdata.data = {'1': 'FOO BAR %d' % i}
        if i%4 == 0:
            formdata.data['2'] = 'foo'
            formdata.data['2_display'] = 'foo'
            formdata.data['3'] = 'A'
            formdata.data['3_display'] = 'aa'
        elif i%4 == 1:
            formdata.data['2'] = 'bar'
            formdata.data['2_display'] = 'bar'
            formdata.data['3'] = 'B'
            formdata.data['3_display'] = 'bb'
        else:
            formdata.data['2'] = 'baz'
            formdata.data['2_display'] = 'baz'
            formdata.data['3'] = 'C'
            formdata.data['3_display'] = 'cc'
        if i%3 == 0:
            formdata.jump_status('new')
        else:
            formdata.jump_status('finished')
        formdata.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'XXX', '2': 'foo', '2_display': 'foo'}
    formdata.status = 'draft'
    formdata.store()

    formdef = FormDef()
    if set_receiver:
        formdef.workflow_roles = {'_receiver': 1}
    formdef.name = 'other form'
    formdef.fields = []
    formdef.store()
    formdef.data_class().wipe()
    for i in range(20):
        formdata = formdef.data_class()()
        formdata.just_created()
        formdata.receipt_time = datetime.datetime(2014, 1, 1).timetuple()
        formdata.jump_status('new')
        formdata.store()

def teardown_module(module):
    clean_temporary_pub()

def test_backoffice_unlogged(pub):
    create_superuser(pub)
    resp = get_app(pub).get('/backoffice/', status=302)
    assert resp.location == 'http://example.net/login/?next=http%3A%2F%2Fexample.net%2Fbackoffice%2F'

def test_backoffice_home(pub):
    create_superuser(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert 'Management' in resp.text
    assert 'Forms Workshop' in resp.text
    assert 'Workflows Workshop' in resp.text

def test_backoffice_role_user(pub):
    create_user(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert 'Management' in resp.text
    assert not 'Forms Workshop' in resp.text
    assert not 'Workflows Workshop' in resp.text

    pub.cfg['admin-permissions'] = {'forms': [x.id for x in Role.select()]}
    pub.write_cfg()
    resp = app.get('/backoffice/')
    assert 'Management' in resp.text
    assert 'Forms Workshop' in resp.text
    assert not 'Workflows Workshop' in resp.text

    pub.cfg['admin-permissions'] = {'workflows': [x.id for x in Role.select()]}
    pub.write_cfg()
    resp = app.get('/backoffice/')
    assert 'Management' in resp.text
    assert not 'Forms Workshop' in resp.text
    assert 'Workflows Workshop' in resp.text

    # check role id int->str migration
    pub.cfg['admin-permissions'] = {'workflows': [int(x.id) for x in Role.select()]}
    pub.write_cfg()
    resp = app.get('/backoffice/')
    assert 'Management' in resp.text
    assert not 'Forms Workshop' in resp.text
    assert 'Workflows Workshop' in resp.text

def test_backoffice_forms(pub):
    create_superuser(pub)
    create_environment(pub, set_receiver=False)

    # 1st time with user not handling those forms
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    resp = resp.click('Management', index=0)
    resp = resp.follow()
    assert not 'Forms in your care' in resp.text
    assert re.findall('Other Forms.*form-title', resp.text)

    # 2nd time with user set as receiver of the forms
    create_environment(pub, set_receiver=True)
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    resp = resp.click('Management', index=0)
    resp = resp.follow()
    assert 'Forms in your care' in resp.text
    assert '17 open on 50' in resp.text

    # disable form, make sure it's still displayed
    formdef = FormDef.get_by_urlname('form-title')
    formdef.disabled = True
    formdef.store()
    resp = app.get('/backoffice/management/forms')
    assert 'form-title' in resp.text
    assert '17 open on 50' in resp.text

    formdef.disabled = False
    formdef.store()

    # add an extra status to workflow and move a few formdatas to it, they
    # should then be marked as open but not waiting for actions.
    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    st1 = workflow.add_status('Status1')
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.timeout = 86400
    jump.status = 'finished'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow = workflow
    formdef.store()

    for i, formdata in enumerate(formdef.data_class().select(order_by='id')):
        if formdata.status == 'wf-new' and i % 2:
            formdata.status = 'wf-%s' % st1.id
            formdata.store()

    resp = app.get('/backoffice/')
    resp = resp.click('Management', index=0)
    resp = resp.follow()
    assert 'Forms in your care' in resp.text
    assert '9 open on 50' in resp.text

def test_backoffice_listing(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 17

    # check status filter <select>
    resp = app.get('/backoffice/management/form-title/')
    resp.forms[0]['filter'] = 'all'
    resp = resp.forms[0].submit()
    if pub.is_using_postgresql():
        assert resp.text.count('data-link') == 20
    else:
        # not using sql -> no pagination
        assert resp.text.count('data-link') == 50

    # check status filter <select>
    resp = app.get('/backoffice/management/form-title/')
    resp.forms[0]['filter'] = 'done'
    resp = resp.forms[0].submit()
    if pub.is_using_postgresql():
        assert resp.text.count('data-link') == 20
        resp = resp.click('Next Page')
        assert resp.text.count('data-link') == 13
    else:
        assert resp.text.count('data-link') == 33

    # add an extra status to workflow and move a few formdatas to it, they
    # should then be marked as open but not waiting for actions.
    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    st1 = workflow.add_status('Status1')
    st1.id = 'plop'
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.timeout = 86400
    jump.status = 'finished'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow = workflow
    formdef.store()

    for i, formdata in enumerate(formdef.data_class().select(order_by='id')):
        if formdata.status == 'wf-new' and i % 2:
            formdata.status = 'wf-%s' % st1.id
            formdata.store()

    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 9
    resp.forms[0]['filter'] = 'pending'
    resp = resp.forms[0].submit()
    assert resp.text.count('data-link') == 17

    # check status forced as endpoints are not part of the "actionable" list.
    workflow = Workflow.get_default_workflow()
    workflow.id = '3'
    st1 = workflow.add_status('Status1')
    st1.id = 'plop'
    st1.forced_endpoint = False
    again = ChoiceWorkflowStatusItem()
    again.id = '_again'
    again.label = 'Again'
    again.by = ['_receiver']
    again.status = st1.id
    st1.items.append(again)
    again.parent = st1
    workflow.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow = workflow
    formdef.store()
    formdef.data_class().rebuild_security()

    for i, formdata in enumerate(formdef.data_class().select(order_by='id')):
        if formdata.status == 'wf-new' and i % 2:
            formdata.status = 'wf-%s' % st1.id
            formdata.store()

    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 17
    resp.forms[0]['filter'] = 'pending'
    resp = resp.forms[0].submit()
    assert resp.text.count('data-link') == 17

    # mark status as an endpoint
    st1.forced_endpoint = True
    workflow.store()
    formdef.data_class().rebuild_security()

    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 9
    resp.forms[0]['filter'] = 'pending'
    resp = resp.forms[0].submit()
    assert resp.text.count('data-link') == 9

def test_backoffice_listing_pagination(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 17

    resp = app.get('/backoffice/management/form-title/?limit=5')
    assert resp.text.count('data-link') == 5
    assert '<div id="page-links">' in resp.text

    resp = resp.click(re.compile('^2$')) # second page
    assert resp.text.count('data-link') == 5
    assert resp.form['offset'].value == '5'

    resp = resp.click(re.compile('^3$')) # third page
    assert resp.text.count('data-link') == 5
    assert resp.form['offset'].value == '10'

    resp = resp.click(re.compile('^4$')) # fourth page
    assert resp.text.count('data-link') == 2
    assert resp.form['offset'].value == '15'

    with pytest.raises(IndexError): # no fifth page
        resp = resp.click(re.compile('^5$'))

    resp = resp.click(re.compile('^10$')) # per page: 10
    assert resp.text.count('data-link') == 10

    resp = resp.click(re.compile('^20$')) # per page: 20
    assert resp.text.count('data-link') == 17

    # try an overbound offset
    resp = app.get('/backoffice/management/form-title/?limit=5&offset=30')
    resp = resp.follow()
    assert resp.form['offset'].value == '0'

def test_backoffice_listing_order(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    ids = []
    for i, formdata in enumerate(formdef.data_class().select(order_by='id')):
        if formdata.status in ('wf-new', 'wf-accepted'):
            ids.append(formdata.id)
            formdata.receipt_time = datetime.datetime(2015, 1, 1, 10, i).timetuple()
            # ordered with odd-numbered ids then even-numbered ids
            formdata.evolution[-1].time = datetime.datetime(2015, 2, 1, 10 + i % 2, i).timetuple()
            formdata.store()

    inversed_receipt_time_order = list(reversed([str(x) for x in sorted(ids)]))
    last_update_time_order = [str(x) for x in sorted(ids, key=lambda x: int(x) if int(x) % 2 else int(x)+1000)]

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 17
    ids = [x.strip('/') for x in re.findall(r'data-link="(.*?)"', resp.text)]
    assert ids == inversed_receipt_time_order

    resp = app.get('/backoffice/management/form-title/?order_by=receipt_time')
    assert resp.text.count('data-link') == 17
    ids = [x.strip('/') for x in re.findall(r'data-link="(.*?)"', resp.text)]
    assert ids == list(reversed(inversed_receipt_time_order))

    resp = app.get('/backoffice/management/form-title/?order_by=last_update_time')
    assert resp.text.count('data-link') == 17
    ids = [x.strip('/') for x in re.findall(r'data-link="(.*?)"', resp.text)]
    assert ids == last_update_time_order

    resp = app.get('/backoffice/management/form-title/?order_by=-last_update_time')
    assert resp.text.count('data-link') == 17
    ids = [x.strip('/') for x in re.findall(r'data-link="(.*?)"', resp.text)]
    assert ids == list(reversed(last_update_time_order))

    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
        pub.site_options.set('options', 'default-sort-order', '-last_update_time')
        fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
        pub.site_options.write(fd)
        fd.close()

    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('data-link') == 17
    ids = [x.strip('/') for x in re.findall(r'data-link="(.*?)"', resp.text)]
    assert ids == list(reversed(last_update_time_order))

def test_backoffice_legacy_urls(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/form-title/')
    assert resp.location == 'http://example.net/backoffice/management/form-title/'
    resp = app.get('/backoffice/form-title/1/')
    assert resp.location == 'http://example.net/backoffice/management/form-title/1/'
    resp = app.get('/backoffice/form-title/listing/?bla')
    assert resp.location == 'http://example.net/backoffice/management/form-title/listing/?bla'
    resp = app.get('/backoffice/form-title/listing/foo?bla')
    assert resp.location == 'http://example.net/backoffice/management/form-title/listing/foo?bla'
    resp = app.get('/backoffice/not-form-title/', status=404)

def test_backoffice_columns(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('</th>') == 8 # six columns
    resp.forms[0]['1'].checked = False
    assert not 'submission_channel' in resp.forms[0].fields
    assert 'last_update_time' in resp.forms[0].fields
    resp = resp.forms[0].submit()
    assert resp.text.count('</th>') == 7 # fixe columns
    assert resp.text.count('data-link') == 17 # 17 rows
    assert resp.text.count('FOO BAR') == 0 # no field 1 column

def test_backoffice_channel_column(pub):
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'welco_url', 'xxx')
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    pub.site_options.write(fd)
    fd.close()

    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('</th>') == 8 # six columns
    resp.forms[0]['submission_channel'].checked = True
    resp = resp.forms[0].submit()
    assert resp.text.count('</th>') == 9 # seven columns
    assert resp.text.count('data-link') == 17 # 17 rows
    assert resp.text.count('<td>Web</td>') == 17

def test_backoffice_submission_agent_column(pub):
    user = create_user(pub)
    create_environment(pub)

    agent = pub.user_class(name='agent')
    agent.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert not 'submission_agent' in resp.form.fields

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    assert resp.text.count('</th>') == 8 # six columns
    resp.form['submission_agent'].checked = True
    resp = resp.form.submit()
    assert resp.text.count('</th>') == 9 # seven columns
    assert resp.text.count('data-link') == 17 # 17 rows
    assert not '>agent<' in resp.text
    resp = resp.click('Export as CSV File')
    assert len(resp.text.splitlines()) == 18 # 17 + header line
    assert not ',agent,' in resp.text

    for formdata in formdef.data_class().select():
        formdata.submission_context = {'agent_id': agent.id}
        formdata.store()

    resp = app.get('/backoffice/management/form-title/')
    resp.form['submission_agent'].checked = True
    resp = resp.form.submit()
    assert resp.text.count('>agent<') == 17

    resp = resp.click('Export as CSV File')
    assert len(resp.text.splitlines()) == 18 # 17 + header line
    assert resp.text.count(',agent,') == 17

def test_backoffice_image_column(pub):
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields.append(fields.FileField(id='4', label='file field', type='file',
                display_locations=['validation', 'summary', 'listings']))
    formdef.store()

    upload = PicklableUpload('test.jpeg', 'image/jpeg')
    jpg = open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
    upload.receive([jpg])
    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
        formdata.data['4'] = upload
        formdata.store()
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert 'download?f=4&thumbnail=1' not in resp.text

def test_backoffice_filter(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert resp.forms[0]['filter-status'].checked == True
    resp.forms[0]['filter-status'].checked = False
    resp.forms[0]['filter-2'].checked = True
    resp = resp.forms[0].submit()
    assert '<select name="filter">' not in resp.text

    resp.forms[0]['filter-2-value'] = 'baz'
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 8
    assert resp.text.count('<td>foo</td>') == 0
    assert resp.text.count('<td>bar</td>') == 0

    resp.forms[0]['filter-start'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-start-value'] = datetime.datetime(2015, 2, 1).strftime('%Y-%m-%d')
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 0
    resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 8

    # check there's no crash on invalid date values
    resp.forms[0]['filter-start-value'] = 'whatever'
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 8

    # check two-digit years are handled correctly
    resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%y-%m-%d')
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 8

    # check it's also ok for end filter
    resp.forms[0]['filter-end'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-end-value'] = datetime.datetime(2014, 2, 2).strftime('%y-%m-%d')
    resp = resp.forms[0].submit()
    assert resp.text.count('<td>baz</td>') == 0

def test_backoffice_default_filter(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert not 'filter-2-value' in resp.form.fields

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[1].in_filters = True
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    assert 'filter-2-value' in resp.form.fields

    # same check for items field
    formdef.fields.append(
            fields.ItemsField(id='4', label='4th field', type='items', items=['foo', 'bar', 'baz']))
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    assert not 'filter-4-value' in resp.form.fields

    formdef.fields[-1].in_filters = True
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    assert 'filter-4-value' in resp.form.fields

def test_backoffice_bool_filter(pub):
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields.append(fields.BoolField(id='4', label='4th field',
        type='bool',
        display_locations=['validation', 'summary', 'listings']))
    formdef.store()

    for i, formdata in enumerate(formdef.data_class().select()):
        formdata.data['4'] = bool(i%2)
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp.form['filter-4'].checked = True
    resp = resp.form.submit()

    assert resp.form['filter-4-value'].value == ''

    resp.form['filter-4-value'].value = 'true'
    resp = resp.form.submit()
    assert resp.text.count('<td>Yes</td>') > 0
    assert resp.text.count('<td>No</td>') == 0

    resp.form['filter-4-value'].value = 'false'
    resp = resp.form.submit()
    assert resp.text.count('<td>Yes</td>') == 0
    assert resp.text.count('<td>No</td>') > 0

def test_backoffice_items_filter(pub):
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields.append(fields.ItemsField(id='4', label='4th field', type='items',
        items=['a', 'b', 'c', 'd'],
        display_locations=['validation', 'summary', 'listings']))
    formdef.store()

    for i, formdata in enumerate(formdef.data_class().select()):
        if i%4 == 0:
            formdata.data['4'] = ['a', 'b']
            formdata.data['4_display'] = 'a, b'
        elif i%4 == 1:
            formdata.data['4'] = ['b', 'd']
            formdata.data['4_display'] = 'b, d'
        elif i%4 == 2:
            formdata.data['4'] = ['a']
            formdata.data['4_display'] = 'a'
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp.form['filter-4'].checked = True
    resp = resp.form.submit()

    assert resp.form['filter-4-value'].value == ''

    resp.form['filter-4-value'].value = 'a'
    resp = resp.form.submit()
    assert resp.text.count('<td>a, b</td>') > 0
    assert resp.text.count('<td>a</td>') > 0
    assert resp.text.count('<td>b, d</td>') == 0

    resp.form['filter-4-value'].value = 'b'
    resp = resp.form.submit()
    assert resp.text.count('<td>a, b</td>') > 0
    assert resp.text.count('<td>a</td>') == 0
    assert resp.text.count('<td>b, d</td>') > 0

    resp.form['filter-4-value'].value = 'c'
    resp = resp.form.submit()
    assert resp.text.count('<td>a, b</td>') == 0
    assert resp.text.count('<td>a</td>') == 0
    assert resp.text.count('<td>b, d</td>') == 0
    assert resp.text.count('data-link') == 0 # no rows

def test_backoffice_csv(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export as CSV File')
    assert resp.headers['content-type'].startswith('text/')
    assert len(resp.text.splitlines()) == 18 # 17 + header line
    assert len(resp.text.splitlines()[0].split(',')) == 7

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[-1].display_locations = ['validation', 'summary', 'listings']
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export as CSV File')
    assert len(resp.text.splitlines()[0].split(',')) == 9

    # check item fields with datasources get two columns (id & text)
    assert resp.text.splitlines()[0].split(',')[6] == '3rd field'
    assert resp.text.splitlines()[0].split(',')[7] == '' # 3rd field, continue
    assert resp.text.splitlines()[1].split(',')[6] == 'A'
    assert resp.text.splitlines()[1].split(',')[7] == 'aa'

    resp = app.get('/backoffice/management/form-title/')
    resp.forms[0]['filter'] = 'all'
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert len(resp_csv.text.splitlines()) == 51

    # test status filter
    resp.forms[0]['filter'] = 'pending'
    resp.forms[0]['filter-2'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-2-value'] = 'baz'
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert len(resp_csv.text.splitlines()) == 9

    # test criteria filters
    resp.forms[0]['filter-start'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-start-value'] = datetime.datetime(2015, 2, 1).strftime('%Y-%m-%d')
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert len(resp_csv.text.splitlines()) == 1

    resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert len(resp_csv.text.splitlines()) == 9
    assert 'Created' in resp_csv.text.splitlines()[0]

    # test column selection
    resp.form['time'].checked = False
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert 'Created' not in resp_csv.text.splitlines()[0]

def test_backoffice_export_long_listings(pub):
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    for i in range(100):
        formdata = formdef.data_class()()
        formdata.just_created()
        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
        formdata.data = {'1': 'BAZ BAZ %d' % i}
        formdata.jump_status('new')
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export as CSV File')
    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
    resp = resp.follow()
    assert 'completed' in resp.text
    resp = resp.click('Download Export')
    resp_lines = resp.text.splitlines()
    assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,2nd field,Status'
    assert len(resp_lines) == 118
    assert resp_lines[1].split(',')[1].startswith(
            time.strftime('%Y-%m-%d', formdata.receipt_time))
    assert resp_lines[1].split(',')[2].startswith(
            time.strftime('%Y-%m-%d', formdata.last_update_time))

    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export a Spreadsheet')
    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
    job_id = urlparse.parse_qs(urlparse.urlparse(resp.location).query)['job'][0]
    resp = resp.follow()
    assert 'completed' in resp.text
    resp = resp.click('Download Export')
    assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'

    # check afterjob ajax call
    status_resp = app.get('/afterjobs/' + job_id)
    assert status_resp.text == 'completed|completed'

    # check error handling
    app.get('/afterjobs/whatever', status=404)

def test_backoffice_csv_export_channel(pub):
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'welco_url', 'xxx')
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    pub.site_options.write(fd)
    fd.close()

    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp_csv = resp.click('Export as CSV File')
    assert 'Channel' not in resp_csv.text.splitlines()[0]

    # add submission channel column
    resp.form['submission_channel'].checked = True
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert resp_csv.text.splitlines()[0].split(',')[1] == 'Channel'
    assert resp_csv.text.splitlines()[1].split(',')[1] == 'Web'

def test_backoffice_csv_export_anonymised(pub):
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    pub.site_options.write(fd)
    fd.close()

    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp_csv = resp.click('Export as CSV File')
    assert resp_csv.text.splitlines()[0].split(',')[-1] != 'Anonymised'

    # add anonymised column
    resp.form['anonymised'].checked = True
    resp = resp.forms[0].submit()
    resp_csv = resp.click('Export as CSV File')
    assert resp_csv.text.splitlines()[0].split(',')[-1] == 'Anonymised'
    assert resp_csv.text.splitlines()[1].split(',')[-1] == 'No'

def test_backoffice_ods(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export a Spreadsheet')
    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
    assert 'filename=form-title.ods' in resp.headers['content-disposition']
    assert resp.body[:2] == b'PK' # ods has a zip container

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields.append(fields.FileField(id='4', label='file field', type='file',
        display_locations=['validation', 'summary', 'listings']))
    formdef.fields.append(fields.DateField(id='5', label='date field', type='date',
        display_locations=['validation', 'summary', 'listings']))
    formdef.fields.append(fields.StringField(id='6', label='number field', type='string',
        display_locations=['validation', 'summary', 'listings']))
    formdef.fields.append(fields.StringField(id='7', label='phone field', type='string',
        display_locations=['validation', 'summary', 'listings']))
    formdef.fields.append(fields.DateField(id='8', label='very old field', type='date',
        display_locations=['validation', 'summary', 'listings']))
    formdef.fields.append(fields.StringField(id='9', label='string field', type='string',
        display_locations=['validation', 'summary', 'listings']))
    formdef.store()

    formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
    formdata.data['4'] = PicklableUpload('/foo/bar', content_type='text/plain')
    formdata.data['4'].receive([b'hello world'])
    formdata.data['5'] = time.strptime('2015-05-12', '%Y-%m-%d')
    formdata.data['6'] = '12345'
    formdata.data['7'] = '0102030405'
    formdata.data['8'] = time.strptime('1871-03-18', '%Y-%m-%d')
    formdata.data['9'] = 'plop\npl\x1dop'  # with control characters
    formdata.store()

    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Export a Spreadsheet')
    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
    assert 'filename=form-title.ods' in resp.headers['content-disposition']
    assert resp.body[:2] == b'PK' # ods has a zip container

    zipf = zipfile.ZipFile(BytesIO(resp.body))
    ods_sheet = ET.parse(zipf.open('content.xml'))
    # check the ods contains a link to the document
    elem = ods_sheet.findall('.//{%s}a' % ods.NS['text'])[0]
    assert elem.attrib['{%s}href' % ods.NS['xlink']] == 'http://example.net/backoffice/management/form-title/%s/files/4/bar' % formdata.id
    resp = app.get(elem.attrib['{%s}href' % ods.NS['xlink']])
    assert resp.text == 'hello world'

    all_texts = [x.text for x in ods_sheet.findall('.//{%s}table-row//{%s}p' % (ods.NS['table'], ods.NS['text']))]
    created_column = all_texts.index('Created')
    date_column = all_texts.index('date field')
    number_column = all_texts.index('number field')
    phone_column = all_texts.index('phone field')
    old_column = all_texts.index('very old field')
    string_column = all_texts.index('string field')

    for row in ods_sheet.findall('.//{%s}table-row' % ods.NS['table']):
        if row.findall('.//{%s}table-cell/{%s}p' % (
                ods.NS['table'], ods.NS['text']))[0].text == formdata.get_display_id():
            break
    else:
        assert False, 'failed to find data row'

    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
            '{%s}value-type' % ods.NS['office']] == 'date'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
            '{%s}style-name' % ods.NS['table']] == 'DateTime'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
            '{%s}value-type' % ods.NS['office']] == 'date'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
            '{%s}style-name' % ods.NS['table']] == 'Date'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
            '{%s}value-type' % ods.NS['office']] == 'float'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
            '{%s}value' % ods.NS['office']] == '12345'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[phone_column].attrib[
            '{%s}value-type' % ods.NS['office']] == 'string'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
            '{%s}value-type' % ods.NS['office']] == 'date'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
            '{%s}date-value' % ods.NS['office']] == '1871-03-18'
    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[
            string_column].find('{%s}p' % ods.NS['text']).text == 'plop\nplop'

@pytest.mark.skipif('xlwt is None')
def test_backoffice_xls(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert not 'Excel Export' in resp.text

    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
        pub.site_options.set('options', 'legacy-excel-export', 'true')
        fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
        pub.site_options.write(fd)
        fd.close()

    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Excel Export')
    assert resp.headers['content-type'].startswith('application/vnd.ms-excel')

def test_backoffice_statistics(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Statistics')
    assert 'Total number of records: 50' in resp.text
    assert 'New: 17' in resp.text
    assert 'Finished: 33' in resp.text
    assert re.findall('foo.*26.*bar.*26.*bar.*48', resp.text) # percentages
    assert 'Resolution time' in resp.text
    assert 'To Status &quot;New&quot;' in resp.text
    assert 'To Status &quot;Finished&quot;' in resp.text
    assert not '<h2>Filters</h2>' in resp.text

    resp.forms[0]['filter-end-value'] = '2013-01-01'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 0' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert 'End: 2013-01-01' in resp.text

def test_backoffice_multi_actions(pub):
    create_superuser(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert not 'id="multi-actions"' in resp.text

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = ['whatever']

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.store()

    resp = app.get('/backoffice/management/form-title/')
    assert not 'id="multi-actions"' in resp.text

    trigger.roles = [x.id for x in Role.select() if x.name == 'foobar']
    workflow.store()

    resp = app.get('/backoffice/management/form-title/')
    assert 'id="multi-actions"' in resp.text
    ids = []
    for checkbox in resp.forms[0].fields['select[]'][1:6]:
        ids.append(checkbox._value)
        checkbox.checked = True
    resp = resp.forms[0].submit('button-action-1')
    assert '?job=' in resp.location
    resp = resp.follow()
    assert 'Executing task &quot;FOOBAR&quot; on forms' in resp.text
    assert '>completed<' in resp.text
    for id in ids:
        assert formdef.data_class().get(id).status == 'wf-finished'

    draft_ids = [x.id for x in formdef.data_class().select() if x.status == 'draft']
    resp = app.get('/backoffice/management/form-title/')
    assert resp.forms[0].fields['select[]'][0]._value == '_all'
    resp.forms[0].fields['select[]'][0].checked = True
    resp = resp.forms[0].submit('button-action-1')
    for formdata in formdef.data_class().select():
        if formdata.id in draft_ids:
            assert formdata.status == 'draft'
        else:
            assert formdata.status == 'wf-finished'

    for formdata in formdef.data_class().select():
        if formdata.status != 'draft':
            formdata.jump_status('new')
            formdata.store()

    # action for other role
    action2 = workflow.add_global_action('OTHER ACTION')
    jump = action2.append_item('jump')
    jump.status = 'accepted'
    trigger = action2.triggers[0]
    trigger.roles = ['whatever']
    workflow.store()
    resp = app.get('/backoffice/management/form-title/')
    assert 'id="multi-actions"' in resp.text
    assert not 'OTHER ACTION' in resp.text

    # action for function
    trigger.roles = ['_foobar']
    workflow.store()

    resp = app.get('/backoffice/management/form-title/')
    assert 'id="multi-actions"' in resp.text
    assert 'OTHER ACTION' not in resp.text

    workflow.roles['_foobar'] = 'Foobar'
    workflow.store()

    resp = app.get('/backoffice/management/form-title/')
    assert 'id="multi-actions"' in resp.text
    assert 'OTHER ACTION' in resp.text

    # alter some formdata to simulate dispatch action
    stable_ids = []
    for checkbox in resp.forms[0].fields['select[]'][1:6]:
        formdata = formdef.data_class().get(checkbox._value)
        formdata.workflow_roles = {'_foobar': formdef.workflow_roles['_receiver']}
        formdata.store()
        stable_ids.append(formdata.id)

    resp = app.get('/backoffice/management/form-title/')
    assert 'OTHER ACTION' in resp.text

    resp.forms[0].fields['select[]'][0].checked = True  # _all
    resp = resp.forms[0].submit('button-action-2')
    assert '?job=' in resp.location
    resp = resp.follow()
    assert 'Executing task &quot;OTHER ACTION&quot; on forms' in resp.text
    # check only dispatched formdata have been moved by global action executed
    # on all formdatas
    for formdata in formdef.data_class().select():
        if formdata.id in draft_ids:
            assert formdata.status == 'draft'
        elif formdata.id in stable_ids:
            assert formdata.status == 'wf-accepted'
        else:
            assert formdata.status != 'wf-accepted'

def test_backoffice_statistics_status_filter(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Statistics')
    assert 'filter' not in resp.forms[0].fields # status is not displayed by default
    assert not '<h2>Filters</h2>' in resp.text

    # add 'status' as a filter
    resp.forms[0]['filter-status'].checked = True
    resp = resp.forms[0].submit()
    assert 'filter' in resp.forms[0].fields
    assert not '<h2>Filters</h2>' in resp.text

    assert resp.forms[0]['filter'].value == 'all'
    resp.forms[0]['filter'].value = 'pending'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 17' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert 'Status: Pending' in resp.text

    resp.forms[0]['filter'].value = 'done'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 33' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert 'Status: Done' in resp.text

    resp.forms[0]['filter'].value = 'rejected'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 0' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert 'Status: Rejected' in resp.text

    resp.forms[0]['filter'].value = 'all'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 50' in resp.text

def test_backoffice_statistics_status_select(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Statistics')
    assert not 'filter-2-value' in resp.form.fields

    resp.forms[0]['filter-2'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-2-value'].value = 'bar'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 13' in resp.text

    resp.forms[0]['filter-2-value'].value = 'baz'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 24' in resp.text

    resp.forms[0]['filter-2-value'].value = 'foo'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 13' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert '2nd field: foo' in resp.text

    # check it's also possible to get back to the complete list
    resp.forms[0]['filter-2-value'].value = ''
    resp = resp.forms[0].submit()
    assert 'Total number of records: 50' in resp.text

    # check it also works with item fields with a data source
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Statistics')
    resp.forms[0]['filter-3'].checked = True
    resp = resp.forms[0].submit()
    resp.forms[0]['filter-3-value'].value = 'A'
    resp = resp.forms[0].submit()
    assert 'Total number of records: 13' in resp.text
    assert '<h2>Filters</h2>' in resp.text
    assert '3rd field: aa' in resp.text

    # set field to be displayed by default in filters
    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[1].in_filters = True
    formdef.store()
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Statistics')
    assert 'filter-2-value' in resp.form.fields

def test_backoffice_map(pub):
    user = create_user(pub)
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 17

    # check there's no link to map the sidebar
    assert not 'Plot on a Map' in resp.text

    formdef = FormDef.get_by_urlname('form-title')
    formdef.geolocations = {'base': 'Geolocafoobar'}
    formdef.store()
    number31.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
    number31.store()

    resp = app.get('/backoffice/management/form-title/')
    assert 'Plot on a Map' in resp.text
    resp = resp.click('Plot on a Map')
    assert 'data-geojson-url' in resp.text
    assert 'tiles.entrouvert.org/' in resp.text

    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')

    pub.site_options.set('options', 'map-tile-urltemplate', 'https://{s}.tile.example.net/{z}/{x}/{y}.png')
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    pub.site_options.write(fd)
    fd.close()

    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click('Plot on a Map')
    assert 'tile.example.net/' in resp.text

def test_backoffice_geojson(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields.append(fields.MapField(id='4', label='4th field', type='map'))
    formdef.fields.append(fields.MapField(id='5', label='5th field', type='string'))
    form_class = formdef.data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/geojson', status=404)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.geolocations = {'base': 'Geolocafoobar'}
    formdef.store()
    number31 = formdef.data_class().get(number31.id)
    number31.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
    number31.store()

    resp = app.get('/backoffice/management/form-title/geojson?1=on&4=on&5=on')
    assert len(resp.json['features']) == 1
    assert resp.json['features'][0]['geometry']['coordinates'] == [2.32, 48.83]
    assert 'status_colour' in resp.json['features'][0]['properties']
    assert resp.json['features'][0]['properties']['status_name'] == 'New'
    assert resp.json['features'][0]['properties']['status_colour'] == '#66FF00'
    assert resp.json['features'][0]['properties']['view_label'] == 'View'
    assert 'display_fields' in resp.json['features'][0]['properties']
    assert len(resp.json['features'][0]['properties']['display_fields']) == 1

    resp = app.get('/backoffice/management/form-title/geojson?filter=pending&filter-status=on')
    assert len(resp.json['features']) == 1

    resp = app.get('/backoffice/management/form-title/geojson?filter=done&filter-status=on')
    assert len(resp.json['features']) == 0


def test_backoffice_handling(pub):
    create_user(pub)
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 17

    # check sidebar links are ok
    assert 'Statistics' in resp.text
    assert 'Export' in resp.text
    app.get('/backoffice/management/form-title/stats', status=200)
    app.get('/backoffice/management/form-title/csv', status=200)
    app.get('/backoffice/management/form-title/ods', status=200)
    app.get('/backoffice/management/form-title/json', status=200)

    # click on a formdata
    resp = resp.click(href='%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    resp.forms[0]['comment'] = 'HELLO WORLD'
    resp = resp.forms[0].submit('button_accept')
    resp = resp.follow()
    assert FormDef.get_by_urlname('form-title').data_class().get(number31.id).status == 'wf-accepted'
    assert 'HELLO WORLD' in resp.text

def test_backoffice_handling_global_action(pub):
    create_user(pub)
    create_environment(pub)

    formdef = FormDef()
    formdef.name = 'test global action'
    formdef.fields = []

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    register_comment = action.append_item('register-comment')
    register_comment.comment = 'HELLO WORLD GLOBAL ACTION'
    jump = action.append_item('jump')
    jump.status = 'finished'
    trigger = action.triggers[0]
    trigger.roles = [x.id for x in Role.select() if x.name == 'foobar']

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata.id))
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')

    resp = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata.id))
    assert 'HELLO WORLD GLOBAL ACTION' in resp.text
    assert formdef.data_class().get(formdata.id).status == 'wf-finished'

def test_backoffice_global_remove_action(pub):
    create_user(pub)
    create_environment(pub)

    formdef = FormDef()
    formdef.name = 'test global remove'
    formdef.fields = []

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    action = workflow.add_global_action('FOOBAR')
    remove = action.append_item('remove')
    trigger = action.triggers[0]
    trigger.roles = [x.id for x in Role.select() if x.name == 'foobar']

    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 1}
    formdef.store()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata.id))
    assert 'remove' in resp.text
    assert 'button-action-1' in resp.form.fields
    resp = resp.form.submit('button-action-1')
    resp = resp.follow()
    assert resp.request.url == 'http://example.net/backoffice/management/test-global-remove/'
    assert 'The form has been deleted.' in resp.text

def test_backoffice_submission_context(pub):
    user = create_user(pub)
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 17

    # click on a formdata
    resp = resp.click(href='%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text

    # check there's nothing in the sidebar
    assert not 'Channel' in resp.text

    number31.submission_channel = 'mail'
    number31.user_id = user.id
    number31.submission_context = {
            'mail_url': 'http://www.example.com/test.pdf',
            'thumbnail_url': 'http://www.example.com/thumbnail.png',
            'comments': 'test_backoffice_submission_context',
            'summary_url': 'http://www.example.com/summary',
            'agent_id': user.id,
    }
    number31.store()
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click(href='%s/' % number31.id)
    assert 'Channel' in resp.text
    assert 'http://www.example.com/thumbnail.png' in resp.text
    assert 'http://www.example.com/test.pdf' in resp.text
    assert 'Associated User' in resp.text
    assert 'test_backoffice_submission_context' in resp.text
    assert 'http://www.example.com/summary' in resp.text
    assert 'by %s' % user.get_display_name() in resp.text

def test_backoffice_geolocation_info(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.geolocations = {'base': 'Geolocafoobar'}
    formdef.store()
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 17

    # click on a formdata
    resp = resp.click(href='%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text

    # check there's nothing in the sidebar
    assert not 'Geolocation' in resp.text

    number31.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
    number31.store()
    resp = app.get('/backoffice/management/form-title/')
    resp = resp.click(href='%s/' % number31.id)
    assert 'Geolocafoobar' in resp.text
    assert 'class="qommon-map"' in resp.text
    assert 'data-init-lng="2.32"' in resp.text
    assert 'data-init-lat="48.83' in resp.text

def test_backoffice_info_text(pub):
    create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='info texts')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = ['_submitter', '_receiver']
    commentable.button_label = 'CLICK ME!'
    st1.items.append(commentable)
    commentable.parent = st1

    commentable2 = CommentableWorkflowStatusItem()
    commentable2.id = '_commentable2'
    commentable2.by = ['_submitter']
    commentable2.button_label = 'CLICK ME2!'
    st1.items.append(commentable2)
    commentable2.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    assert 'CLICK ME!' in resp.text
    assert not 'CLICK ME2!' in resp.text
    assert not 'backoffice-description' in resp.text

    # add an info text to the status
    st1.backoffice_info_text = '<p>Foo</p>'
    workflow.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert 'backoffice-description' in resp.text
    assert '<p>Foo</p>' in resp.text

    # add an info text to the button
    commentable.backoffice_info_text = '<p>Bar</p>'
    workflow.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert 'backoffice-description' in resp.text
    assert '<p>Foo</p>' in resp.text
    assert '<p>Bar</p>' in resp.text

    # info text is not visible if form is locked
    second_user = pub.user_class(name='foobar')
    second_user.roles = Role.keys()
    second_user.store()
    account = PasswordAccount(id='foobar')
    account.set_password('foobar')
    account.user_id = second_user.id
    account.store()
    app2 = login(get_app(pub), username='foobar', password='foobar')
    resp = app2.get('/backoffice/management/form-title/%s/' % number31.id)
    assert 'Be warned forms of this user are also being looked' in resp.text
    assert not 'backoffice-description' in resp.text
    assert not 'CLICK ME!' in resp.text
    assert not 'CLICK ME2!' in resp.text

    # remove info text from the status
    st1.backoffice_info_text = None
    workflow.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert 'backoffice-description' in resp.text
    assert not '<p>Foo</p>' in resp.text
    assert '<p>Bar</p>' in resp.text

    # add info text to second button
    commentable2.backoffice_info_text = '<p>Baz</p>'
    workflow.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert 'backoffice-description' in resp.text
    assert not '<p>Foo</p>' in resp.text
    assert '<p>Bar</p>' in resp.text
    assert not '<p>Baz</p>' in resp.text

    # remove info text from first button
    commentable.backoffice_info_text = None
    workflow.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert not 'backoffice-description' in resp.text

def test_backoffice_handling_post_dispatch(pub):
    # check a formdata that has been dispatched to another role is accessible
    # by an user with that role.
    user1 = create_user(pub)
    role = Role(name='foobaz')
    role.store()
    user1.roles = [role.id]
    user1.store()
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))

    # check there's no access at the moment
    resp = app.get('/backoffice/management/').follow()
    assert not 'form-title/' in resp.text
    resp = app.get('/backoffice/management/form-title/', status=403)
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id, status=403)

    # emulate a dispatch (setting formdata.workflow_roles), receiver of that
    # formdata is now the local role we gave to the user.
    formdata31 = form_class.get(number31.id)
    formdata31.workflow_roles = {'_receiver': role.id}
    formdata31.store()

    # check listing is accessible, with a single item
    resp = app.get('/backoffice/management/').follow()
    assert 'form-title/' in resp.text
    resp = app.get('/backoffice/management/form-title/', status=200)
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 1

    # check statistics and exports are also available
    assert 'Statistics' in resp.text
    assert 'Export' in resp.text
    app.get('/backoffice/management/form-title/stats', status=200)
    app.get('/backoffice/management/form-title/csv', status=200)
    app.get('/backoffice/management/form-title/ods', status=200)
    app.get('/backoffice/management/form-title/json', status=200)

    # check formdata is accessible, and that it's possible to perform an action
    # on it.
    resp = resp.click(href='%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    resp.forms[0]['comment'] = 'HELLO WORLD'
    resp = resp.forms[0].submit('button_accept')
    resp = resp.follow()
    assert FormDef.get_by_urlname('form-title').data_class().get(number31.id).status == 'wf-accepted'
    assert 'HELLO WORLD' in resp.text

def test_global_statisticspub(pub):
    create_superuser(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    resp = resp.click('Global statistics')
    assert 'Total count: 70' in resp.text

    resp.forms[0]['start'] = '2014-01-01'
    resp.forms[0]['end'] = '2014-12-31'
    resp = resp.forms[0].submit()
    assert 'Total count: 20' in resp.text

def test_backoffice_submission(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert not 'Submission' in resp.text
    app.get('/backoffice/submission/', status=403)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    resp = app.get('/backoffice/')
    assert 'Submission' in resp.text
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    # going back to first page, to check
    resp = resp.form.submit('previous')
    assert resp.form['f1'].value == 'test submission'
    resp = resp.form.submit('submit')

    # final submit
    resp = resp.form.submit('submit')

    formdata_no = resp.location.split('/')[-2]
    data_class = formdef.data_class()
    assert data_class.get(formdata_no).data['1'] == 'test submission'
    assert data_class.get(formdata_no).data['2'] == 'baz'
    assert data_class.get(formdata_no).status == 'wf-new'
    assert data_class.get(formdata_no).user is None
    assert data_class.get(formdata_no).backoffice_submission is True

    resp = resp.follow() # get to the formdata page

    formdata_count = data_class.count()

    # test submission when agent is not receiver
    formdef.workflow_roles = {}
    formdef.store()
    resp = app.get('/backoffice/submission/')
    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit') # to validation screen
    resp = resp.form.submit('submit') # final submit
    assert resp.location == 'http://example.net/backoffice/submission/'
    resp = resp.follow() # should go back to submission screen

    assert data_class.count() == formdata_count + 1

    # test redirection on cancel
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/backoffice/submission/'

    # test submission when agent is not receiver but there's a redirect action
    # in the workflow.
    formdef = FormDef.get_by_urlname('form-title')
    wf = Workflow(name='dispatch')
    st1 = wf.add_status('Status1')
    item = RedirectToUrlWorkflowStatusItem()
    item.id = '_redirect'
    item.url = 'http://www.example.org/'
    st1.items.append(item)
    item.parent = st1
    wf.store()
    formdef.workflow_id = wf.id
    formdef.store()

    resp = app.get('/backoffice/submission/')
    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit') # to validation screen
    resp = resp.form.submit('submit') # final submit
    assert resp.location == 'http://www.example.org/'


def test_backoffice_submission_with_tracking_code(pub):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/')

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    # final submit
    validation_resp_body = resp.text
    resp = resp.form.submit('submit')

    formdata_no = resp.location.split('/')[-2]
    data_class = formdef.data_class()
    formdata = data_class.get(formdata_no)
    assert formdata.tracking_code in validation_resp_body

    formdata_location = resp.location
    resp = resp.follow() # get to the formdata page
    # check tracking code is still displayed in formdata page
    assert 'test submission' in resp.text
    assert formdata.tracking_code in resp.text

    # check access by different user
    formdata.submission_context = {'agent_id': '10000'}
    formdata.store()
    resp = app.get(formdata_location)
    assert 'test submission' in resp.text
    assert not formdata.tracking_code in resp.text

    # restore user
    formdata.submission_context = {'agent_id': user.id}
    formdata.store()
    resp = app.get(formdata_location)
    assert formdata.tracking_code in resp.text

    # check access at a later time
    formdata.receipt_time = time.localtime(time.time() - 3600)
    formdata.store()
    resp = app.get(formdata_location)
    assert not formdata.tracking_code in resp.text

def test_backoffice_submission_welco(pub, welco_url):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert not 'Submission' in resp.text
    app.get('/backoffice/submission/', status=403)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    # if it's empty, redirect to welco
    resp = app.get('/backoffice/')
    assert 'Submission' in resp.text
    resp = app.get('/backoffice/submission/')
    assert resp.location == 'http://welco.example.net'

    # if there are pending submissions, display them
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.status = 'draft'
    formdata.backoffice_submission = True
    formdata.submission_context = {'agent_id': user.id}
    formdata.store()

    resp = app.get('/backoffice/')
    assert 'Submission' in resp.text
    resp = app.get('/backoffice/submission/')
    assert 'Submission to complete' in resp.text
    # check agent name is displayed next to pending submission
    assert '(%s)' % user.display_name in resp.text

def test_backoffice_submission_initiated_from_welco(pub, welco_url):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields = [
        fields.PageField(id='0', label='1st PAGE', type='page',
            condition={'type': 'python', 'value': 'form_submission_channel != "counter" and is_in_backoffice'}),
        fields.StringField(id='1', label='Field on 1st page', type='string'),
        fields.PageField(id='2', label='2nd PAGE', type='page'),
        fields.StringField(id='3', label='Field on 2nd page', type='string'),
    ]
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    def post_formdata():
        signed_url = sign_url('http://example.net/api/formdefs/form-title/submit' +
                '?format=json&orig=coucou&email=%s' % urllib.quote(user.email), '1234')
        url = signed_url[len('http://example.net'):]
        resp = get_app(pub).post_json(url, {
                'meta': {
                    'draft': True,
                    'backoffice-submission': True,
                },
                'data': {},
                'context': {
                    'channel': 'counter'
                }
        })
        return resp.json['data']['id']

    resp = app.get('/backoffice/management/form-title/%s/' % post_formdata())
    resp = resp.follow()  # -> /backoffice/submission/form-title/XXX
    resp = resp.follow()  # -> /backoffice/submission/form-title/?mt=XYZ

    # second page should be shown
    pq = resp.pyquery.remove_namespaces()
    assert pq('#steps li.current .label').text() == '2nd PAGE'
    assert 'Field on 2nd page' in resp.text  # and in fields

    # reverse condition
    formdef.fields[0].condition['value'] = 'form_submission_channel == "counter" and is_in_backoffice'
    formdef.store()

    resp = app.get('/backoffice/management/form-title/%s/' % post_formdata())
    resp = resp.follow()  # -> /backoffice/submission/form-title/XXX
    resp = resp.follow()  # -> /backoffice/submission/form-title/?mt=XYZ

    pq = resp.pyquery.remove_namespaces()
    assert pq('#steps li.current .label').text() == '1st PAGE'
    assert 'Field on 1st page' in resp.text  # and in fields

def test_backoffice_submission_with_return_url(pub):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/?ReturnURL=https://example.org')
    resp = resp.follow().follow()
    resp = resp.form.submit('cancel')
    assert resp.location == 'https://example.org'

    resp = app.get('/backoffice/submission/form-title/?ReturnURL=https://example.org')
    resp = resp.follow().follow()
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')  # -> to validation
    resp = resp.form.submit('submit')  # -> to submit
    assert resp.location.startswith('http://example.net/backoffice/management/form-title/')

    # test submission when agent is not receiver
    formdef.workflow_roles = {}
    formdef.store()
    resp = app.get('/backoffice/submission/form-title/?ReturnURL=https://example.org')
    resp = resp.follow().follow()
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')  # -> to validation
    resp = resp.form.submit('submit')  # -> to submit
    assert resp.location == 'https://example.org'

def test_backoffice_parallel_submission(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    app.get('/backoffice/submission/', status=403)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.status = 'draft'
    formdata.backoffice_submission = True
    formdata.submission_context = {'agent_id': user.id}
    formdata.store()

    resp = app.get('/backoffice/submission/')
    assert 'Submission to complete' in resp.text
    resp1 = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp1 = resp1.follow()
    resp2 = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp2 = resp2.follow()
    resp3 = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp3 = resp3.follow()

    resp1.form['f1'] = 'foo'
    resp1.form['f2'] = 'bar'
    resp1.form['f3'] = 'C'
    resp1 = resp1.form.submit('submit') # to validation page

    # also move the second form to the validation page
    resp2.form['f1'] = 'bar'
    resp2.form['f2'] = 'bar'
    resp2.form['f3'] = 'C'
    resp2 = resp2.form.submit('submit') # to validation page

    resp1 = resp1.form.submit('submit') # final validation
    resp1 = resp1.follow()

    resp2 = resp2.form.submit('submit') # final validation
    assert resp2.status_code == 302
    resp2 = resp2.follow()
    assert 'This form has already been submitted.' in resp2.text

    # do the third form from the start
    resp3.form['f1'] = 'baz'
    resp3.form['f2'] = 'bar'
    resp3.form['f3'] = 'C'

    resp_autosave = app.post('/backoffice/submission/form-title/autosave',
            params=resp3.form.submit_fields())
    assert resp_autosave.json['result'] == 'error'
    assert resp_autosave.json['reason'] == 'form has already been submitted'

    resp3 = resp3.form.submit('submit') # to validation page
    assert resp3.status_code == 302
    resp3 = resp3.follow()
    assert 'This form has already been submitted.' in resp3.text

    assert formdef.data_class().get(formdata.id).data['1'] == 'foo'

    # try again, very late.
    resp4 = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp4 = resp4.follow()
    assert 'This form has already been submitted.' in resp4.text

def test_backoffice_submission_dispatch(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='dispatch')
    st1 = wf.add_status('Status1')
    dispatch = DispatchWorkflowStatusItem()
    dispatch.id = '_dispatch'
    dispatch.role_key = '_receiver'
    dispatch.role_id = '2'
    st1.items.append(dispatch)
    dispatch.parent = st1
    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/')

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit') # to validation screen
    resp = resp.form.submit('submit') # final submit
    # should go to the formdata because the formdef is defined as is
    assert resp.location.startswith('http://example.net/backoffice/management/form-title/')

    # remove function from formdef
    formdef.workflow_roles = {}
    formdef.store()

    resp = app.get('/backoffice/submission/')

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit') # to validation screen
    resp = resp.form.submit('submit') # final submit
    # should NOT go to the formdata
    assert resp.location == 'http://example.net/backoffice/submission/'

    # if there's no function but the dispatch sets the right function, should
    # go to the formdata screen
    dispatch.role_id = '1'
    wf.store()

    resp = app.get('/backoffice/submission/')

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit') # to validation screen
    resp = resp.form.submit('submit') # final submit
    # should go to the formdata because the formdata was dispatched to the
    # right role
    assert resp.location.startswith('http://example.net/backoffice/management/form-title/')

def test_backoffice_submission_tracking_code(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    formdef.enable_tracking_codes = True
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    # stop here, don't validate, let user finish it.
    assert data_class.count() == 1
    formdata_no = data_class.select()[0].id
    tracking_code = data_class.select()[0].tracking_code

    assert data_class.get(formdata_no).data['1'] == 'test submission'
    assert data_class.get(formdata_no).data['2'] == 'baz'
    assert data_class.get(formdata_no).status == 'draft'
    assert data_class.get(formdata_no).user is None

    resp = get_app(pub).get('/code/%s/load' % data_class.select()[0].tracking_code)
    resp = resp.follow()
    assert resp.location.startswith('http://example.net/form-title/?mt=')
    resp = resp.follow()
    assert 'Check values then click submit.' in resp.text
    assert 'test submission' in resp.text

def test_backoffice_submission_drafts(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    formdef.enable_tracking_codes = True
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    assert data_class.count() == 1
    formdata = data_class.select()[0]
    formdata_no = formdata.id
    tracking_code = data_class.select()[0].tracking_code

    # stop here, go back to index
    resp = app.get('/backoffice/submission/')
    assert '%s/%s' % (formdef.url_name, formdata_no) in resp.text
    assert '>#%s' % formdata_no in resp.text
    formdata.submission_channel = 'mail'
    formdata.store()
    resp = app.get('/backoffice/submission/')
    assert '>Mail #%s' % formdata_no in resp.text

    # check it can also be accessed using its final URL
    resp2 = app.get('/backoffice/management/%s/%s/' % (formdef.url_name, formdata_no))
    assert resp2.location == 'http://example.net/backoffice/submission/%s/%s' % (
            formdef.url_name, formdata_no)

    resp = resp.click('#%s' % formdata_no)
    resp = resp.follow()
    assert tracking_code in resp.text
    resp = resp.form.submit('previous')
    assert resp.form['f1'].value == 'test submission'

    resp = resp.form.submit('submit')
    assert "Check values then click submit." in resp.text
    resp = resp.form.submit('submit')
    # check it kept the same id
    assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata_no

def test_backoffice_submission_remove_drafts(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    formdef.enable_tracking_codes = True
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()
    pub.tracking_code_class.wipe()
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    assert data_class.count() == 1
    formdata = data_class.select()[0]
    formdata_no = formdata.id
    tracking_code = data_class.select()[0].tracking_code

    # stop here, go back to the index
    resp = app.get('/backoffice/submission/')
    resp = resp.click('#%s' % formdata_no)
    resp = resp.follow()

    # and try to delete the form (but cancel)
    resp = resp.click('Delete this form')
    resp = resp.form.submit('cancel')
    assert resp.location == 'http://example.net/backoffice/submission/'
    assert data_class.count() == 1
    assert pub.tracking_code_class().count() == 1

    # and this time for real
    resp = app.get('/backoffice/submission/')
    resp = resp.click('#%s' % formdata_no)
    resp = resp.follow()
    resp = resp.click('Delete this form')
    resp = resp.form.submit('delete')
    assert resp.location == 'http://example.net/backoffice/submission/'
    assert data_class.count() == 0
    assert pub.tracking_code_class().count() == 0

    # check it's not possible to delete an actual formdata
    formdata = data_class()
    formdata.store()
    resp = app.get('/backoffice/submission/form-title/remove/%s' % formdata.id,
            status=403)

def test_backoffice_submission_live_condition(pub):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    formdef.fields = [
        fields.StringField(type='string', id='1', label='Bar', size='40',
            required=True, varname='bar'),
        fields.StringField(type='string', id='2', label='Foo', size='40',
            required=True, varname='foo',
            condition={'type': 'django', 'value': 'form_var_bar == "bye"'}),
    ]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    resp.form['f1'] = 'hello'
    live_url = resp.html.find('form').attrs['data-live-url']
    assert '/backoffice/' in live_url
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'bye'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']
    resp.form['f1'] = 'hello'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    assert 'name="f1"' in resp.text
    assert 'name="f2"' not in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert '<span class="label">Bar</span>' in resp.text
    assert '<span class="label">Foo</span>' not in resp.text

def test_backoffice_submission_conditional_jump_based_on_bo_field(pub):
    user = create_superuser(pub)
    Workflow.wipe()
    workflow = Workflow(name='form-title')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(
            id='bo0', varname='foo_bovar', type='string', label='bo variable'),
    ]

    st1 = workflow.add_status('Status1', 'st1')
    st2 = workflow.add_status('Status2', 'st2')

    setbo = SetBackofficeFieldsWorkflowStatusItem()
    setbo.parent = st1
    setbo.fields = [{'field_id': 'bo0', 'value': 'go'}]
    st1.items.append(setbo)

    jump = JumpWorkflowStatusItem()
    jump.condition = {'type': 'django', 'value': "form_var_foo_bovar == 'go'"}
    jump.status = 'st2'
    st1.items.append(jump)
    jump.parent = st1

    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.name = 'form-title'
    formdef.fields = [
        fields.TextField(id='1', varname='foo', type='text', label='fo variable'),
        fields.TextField(id='2', varname='var3', type='text', label='n/a',
                         condition={'type': 'django', 'value': 'True'}),
      ]
    formdef.workflow_id = workflow.id
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    formdef.data_class().wipe()

    # check jump condition being True
    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'] = 'foo'
    resp.form['f2'] = 'bar'
    resp = resp.form.submit('submit')  # -> confirmation page
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st2'

    # check jump condition being False
    setbo.fields = [{'field_id': 'bo0', 'value': 'nogo'}]
    workflow.store()
    formdef.data_class().wipe()
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'] = 'foo'
    resp.form['f2'] = 'bar'
    resp = resp.form.submit('submit')  # -> confirmation page
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].status == 'wf-st1'

def test_backoffice_submission_sections(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    formdef.enable_tracking_codes = True
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    data_class = formdef.data_class()
    data_class.wipe()

    resp = app.get('/backoffice/submission/')
    assert not 'Submission to complete' in resp.text
    assert not 'Running submission' in resp.text

    formdata = data_class()
    formdata.data = {}
    formdata.status = 'draft'
    formdata.backoffice_submission = True
    formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
    formdata.store()

    resp = app.get('/backoffice/submission/')
    assert 'Submission to complete' in resp.text
    assert not 'Running submission' in resp.text
    assert '>#%s' % formdata.id in resp.text

    formdata.data = {'1': 'xxx'}
    formdata.store()
    resp = app.get('/backoffice/submission/')
    assert not 'Submission to complete' in resp.text
    assert 'Running submission' in resp.text
    assert '>#%s' % formdata.id in resp.text

def test_backoffice_submission_prefill_user(pub):
    user = create_user(pub)
    create_environment(pub)
    other_user = pub.user_class(name='other user')
    other_user.email = 'other@example.net'
    other_user.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[0].prefill = {'type': 'user', 'value': 'email'}
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.backoffice_submission = True
    formdata.status = 'draft'
    formdata.data = {}
    formdata.submission_channel = 'mail'
    formdata.user_id = other_user.id
    formdata.submission_context = {}
    formdata.store()

    formdata2 = formdef.data_class()()
    formdata2.backoffice_submission = True
    formdata2.status = 'draft'
    formdata2.data = {}
    formdata.submission_channel = 'mail'
    formdata.user_id = None
    formdata2.submission_context = {}
    formdata2.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    assert resp.form['f1'].value == ''

    # restore a draft
    resp = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp = resp.follow()
    # and check it got prefilled with the user from context
    assert resp.form['f1'].value == 'other@example.net'

    # restore another, without user id
    resp = app.get('/backoffice/submission/form-title/%s' % formdata2.id)
    resp = resp.follow()
    # and check it was not prefilled
    assert resp.form['f1'].value == ''

def test_backoffice_submission_prefill_user_via_formula(pub):
    user = create_user(pub)
    create_environment(pub)
    other_user = pub.user_class(name='other user')
    other_user.email = 'other@example.net'
    other_user.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[0].prefill = {'type': 'formula', 'value': 'form_user_email'}
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.backoffice_submission = True
    formdata.status = 'draft'
    formdata.data = {}
    formdata.submission_channel = 'mail'
    formdata.user_id = other_user.id
    formdata.submission_context = {}
    formdata.store()

    formdata2 = formdef.data_class()()
    formdata2.backoffice_submission = True
    formdata2.status = 'draft'
    formdata2.data = {}
    formdata.submission_channel = 'mail'
    formdata.user_id = None
    formdata2.submission_context = {}
    formdata2.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    assert resp.form['f1'].value == ''

    # restore a draft
    resp = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp = resp.follow()
    # and check it got prefilled with the user from context
    assert resp.form['f1'].value == 'other@example.net'

    # restore another, without user id
    resp = app.get('/backoffice/submission/form-title/%s' % formdata2.id)
    resp = resp.follow()
    # and check it was not prefilled
    assert resp.form['f1'].value == ''

def test_backoffice_submission_prefill_user_multiple_pages(pub):
    user = create_user(pub)
    create_environment(pub)
    other_user = pub.user_class(name='other user')
    other_user.email = 'other@example.net'
    other_user.store()

    for enable_tracking_code in (False, True):
        formdef = FormDef.get_by_urlname('form-title')
        formdef.fields = [
            fields.PageField(id='0', label='1st page', type='page'),
            fields.StringField(id='1', label='1st field', type='string', required=False),
            fields.PageField(id='4', label='2nd page', type='page'),
            fields.StringField(id='5', label='field on 2nd page',
                prefill = {'type': 'user', 'value': 'email'}),
        ]
        formdef.backoffice_submission_roles = user.roles[:]
        formdef.enable_tracking_codes = enable_tracking_code
        formdef.store()

        formdef.data_class().wipe()

        formdata = formdef.data_class()()
        formdata.backoffice_submission = True
        formdata.status = 'draft'
        formdata.data = {}
        formdata.submission_channel = 'mail'
        formdata.user_id = other_user.id
        formdata.submission_context = {}
        formdata.store()

        formdata2 = formdef.data_class()()
        formdata2.backoffice_submission = True
        formdata2.status = 'draft'
        formdata2.data = {}
        formdata.submission_channel = 'mail'
        formdata.user_id = None
        formdata2.submission_context = {}
        formdata2.store()

        app = login(get_app(pub))
        resp = app.get('/backoffice/submission/form-title/')
        resp = resp.form.submit('submit')
        assert resp.form['f5'].value == ''

        # restore a draft
        resp = app.get('/backoffice/submission/form-title/%s' % formdata.id)
        resp = resp.follow()
        resp = resp.form.submit('submit')
        # and check it got prefilled with the user from context
        assert resp.form['f5'].value == 'other@example.net'

        # restore another, without user id
        resp = app.get('/backoffice/submission/form-title/%s' % formdata2.id)
        resp = resp.follow()
        resp = resp.form.submit('submit')
        # and check it was not prefilled
        assert resp.form['f5'].value == ''

    # continue with additional tests when drafts are enabled, using autosave

    # restore a draft
    formdata.page_no = 0
    formdata.user_id = other_user.id
    formdata.store()
    resp = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp = resp.follow()
    resp.form['f1'] = 'Hello'
    resp_autosave = app.post('/backoffice/submission/form-title/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().get(formdata.id).user_id == str(other_user.id)
    assert formdef.data_class().get(formdata.id).data['1'] == 'Hello'

    resp = resp.form.submit('submit')
    # and check it got prefilled with the user from context
    assert resp.form['f5'].value == 'other@example.net'

    assert formdef.data_class().get(formdata.id).user_id == str(other_user.id)
    assert formdef.data_class().get(formdata.id).data['1'] == 'Hello'

    formdef.data_class().wipe()
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'] = 'Hello'
    resp_autosave = app.post('/backoffice/submission/form-title/autosave', params=resp.form.submit_fields())
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].user_id is None
    resp = resp.form.submit('submit')
    assert formdef.data_class().count() == 1
    assert formdef.data_class().select()[0].user_id is None

def test_backoffice_submission_multiple_page_restore_on_validation(pub):
    user = create_user(pub)
    create_environment(pub)
    other_user = pub.user_class(name='other user')
    other_user.email = 'other@example.net'
    other_user.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='1st field', type='string', required=False),
        fields.PageField(id='2', label='2nd page', type='page',
            condition={'type': 'python', 'value': 'False'}),
        fields.PageField(id='3', label='3rd page', type='page'),
        fields.StringField(id='5', label='field on 3rd page'),
    ]
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True
    formdef.store()

    formdef.data_class().wipe()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'] = 'foo'
    resp = resp.form.submit('submit')
    resp.form['f5'] = 'bar'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    assert formdef.data_class().count() == 1
    formdata = formdef.data_class().select()[0]
    # restore draft
    resp = app.get('/backoffice/submission/')
    resp = resp.click(href='form-title/%s' % formdata.id)
    resp = resp.follow()
    assert 'Check values then click submit.' in resp.text

def test_backoffice_submission_substitution_vars(pub):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields = [
        fields.PageField(id='0', label='1st page', type='page'),
        fields.StringField(id='1', label='1st field', type='string',
            required=False, varname='foobar'),
        fields.ItemField(id='10', label='2nd field', type='item',
                items=['foo', 'bar', 'baz'], varname='foobar2'),
        fields.PageField(id='4', label='2nd page', type='page'),
        fields.CommentField(id='5', label='X[form_var_foobar]Y[form_var_foobar2_raw]Z', type='comment'),
    ]
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'].value = 'PLOP'
    resp.form['f10'].value = 'bar'
    resp = resp.form.submit('submit')
    assert 'XPLOPYbarZ' in resp.text

    # django-templated comment
    formdef.fields[4] = fields.CommentField(id='5',
            label='dj-{{ form_var_foobar }}-an-{{ form_var_foobar2_raw}}-go', type='comment')
    formdef.store()
    formdef.data_class().wipe()
    resp = app.get('/backoffice/submission/form-title/')
    resp.form['f1'].value = 'foo'
    resp.form['f10'].value = 'bar'
    resp = resp.form.submit('submit')
    assert 'dj-foo-an-bar-go' in resp.text

    formdef.data_class().wipe()

    # same but starting from a draft, as if it was initiated by welco
    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.status = 'draft'
    formdata.backoffice_submission = True
    formdata.store()

    resp = app.get('/backoffice/submission/form-title/%s' % formdata.id)
    resp = resp.follow()
    resp.form['f1'].value = 'PLOP'
    resp.form['f10'].value = 'bar'
    resp = resp.form.submit('submit')
    assert 'dj-PLOP-an-bar-go' in resp.text

def test_backoffice_submission_manual_channel(pub):
    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert not 'Submission' in resp.text
    app.get('/backoffice/submission/', status=403)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()
    resp = app.get('/backoffice/')
    assert 'Submission' in resp.text
    resp = app.get('/backoffice/submission/')
    assert formdef.url_name in resp.text

    resp = resp.click(formdef.name)
    assert resp.form['submission_channel'].attrs['type'] == 'hidden'
    resp.form['submission_channel'] = 'mail'
    resp.form['f1'] = 'test submission'
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text

    # final submit
    resp = resp.form.submit('submit')

    formdata_no = resp.location.split('/')[-2]
    data_class = formdef.data_class()
    assert data_class.get(formdata_no).data['1'] == 'test submission'
    assert data_class.get(formdata_no).data['2'] == 'baz'
    assert data_class.get(formdata_no).status == 'wf-new'
    assert data_class.get(formdata_no).user is None
    assert data_class.get(formdata_no).backoffice_submission is True
    assert data_class.get(formdata_no).submission_channel == 'mail'

def test_backoffice_submission_no_manual_channel_with_welco(pub, welco_url):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/%s/' % formdef.url_name)
    assert 'submission_channel' not in resp.form.fields

def test_backoffice_submission_with_nameid_and_channel(pub, local_user):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[0].prefill = {'type': 'formula', 'value': 'form_user_email'}
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True
    formdef.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/submission/form-title/?NameID=%s&channel=mail' % local_user.name_identifiers[0])
    assert resp.location.startswith('http://example.net/backoffice/submission/form-title/')

    formdata_no = resp.location.split('/')[-1]
    formdata = formdef.data_class().get(formdata_no)
    assert formdata.user_id == str(local_user.id)
    assert formdata.submission_channel == 'mail'
    assert formdata.status == 'draft'

    resp = resp.follow()  # redirect to created draft
    resp = resp.follow()  # redirect to ?mt=

    assert resp.form['f1'].value == local_user.email  # prefill with form_user_email
    resp.form['f2'] = 'baz'
    resp.form['f3'] = 'C'
    resp = resp.form.submit('submit')
    assert 'Check values then click submit.' in resp.text
    # final submit
    resp = resp.form.submit('submit')

    formdata_no = resp.location.split('/')[-2]
    data_class = formdef.data_class()
    formdata = data_class.get(formdata_no)
    assert formdata.user_id == str(local_user.id)
    assert formdata.submission_channel == 'mail'
    assert formdata.status == 'wf-new'

def test_backoffice_wscall_failure_display(http_requests, pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='wscall')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wscall = WebserviceCallStatusItem()
    wscall.id = '_wscall'
    wscall.varname = 'xxx'
    wscall.url = 'http://remote.example.net/xml'
    wscall.action_on_bad_data = ':stop'
    wscall.record_errors = True
    st1.items.append(wscall)
    wscall.parent = st1

    again = ChoiceWorkflowStatusItem()
    again.id = '_again'
    again.label = 'Again'
    again.by = ['_receiver']
    again.status = st1.id
    st1.items.append(again)
    again.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    assert 'Again' in resp.text
    resp = resp.forms[0].submit('button_again')
    resp = resp.follow()
    assert 'Error during webservice call' in resp.text

    number31.user_id = user.id  # change ownership to stay in frontoffice
    number31.store()
    # the failure message shouldn't be displayed in the frontoffice
    resp = app.get('/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    assert not 'Error during webservice call' in resp.text

def test_backoffice_wscall_error_email(http_requests, pub, emails):
    pub.cfg['debug'] = {'error_email': 'errors@localhost.invalid'}
    pub.write_cfg()

    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='wscall')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wscall = WebserviceCallStatusItem()
    wscall.id = '_wscall'
    wscall.varname = 'xxx'
    wscall.url = 'http://remote.example.net/xml'
    wscall.action_on_bad_data = ':stop'
    wscall.record_errors = True
    st1.items.append(wscall)
    wscall.parent = st1

    again = ChoiceWorkflowStatusItem()
    again.id = '_again'
    again.label = 'Again'
    again.by = ['_receiver']
    again.status = st1.id
    st1.items.append(again)
    again.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    assert 'Again' in resp.text
    resp = resp.forms[0].submit('button_again')
    resp = resp.follow()
    assert 'Error during webservice call' in resp.text

    # check email box
    if six.PY2:
        error_email = emails.get('[ERROR] [WSCALL] ValueError: No JSON object could be decoded')
    else:
        error_email = emails.get('[ERROR] [WSCALL] json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)')
    assert '/form-title/%s/' % number31.id in error_email['payload']
    assert error_email['msg']['References']

def test_backoffice_wscall_attachment(http_requests, pub):
    create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='wscall')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.FileField(id='bo1', label='bo field 1', type='file'),
    ]

    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wscall = WebserviceCallStatusItem()
    wscall.id = '_wscall'
    wscall.varname = 'xxx'
    wscall.response_type = 'attachment'
    wscall.backoffice_filefield_id = 'bo1'
    wscall.url = 'http://remote.example.net/xml'
    wscall.action_on_bad_data = ':stop'
    wscall.record_errors = True
    st1.items.append(wscall)
    wscall.parent = st1

    again = ChoiceWorkflowStatusItem()
    again.id = '_again'
    again.label = 'Again'
    again.by = ['_receiver']
    again.status = st1.id
    st1.items.append(again)
    again.parent = st1

    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    assert 'Again' in resp.text
    resp = resp.forms[0].submit('button_again')
    resp = resp.follow()

    # get the two generated files from backoffice: in backoffice fields
    # (wscall.backoffice_filefield_id), and in history
    for index in (0, 1):
        resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
        resp = resp.click('xxx.xml', index=index)
        assert resp.location.endswith('/xxx.xml')
        resp = resp.follow()
        assert resp.content_type == 'text/xml'
        assert resp.text == '<?xml version="1.0"><foo/>'

    formdata = formdef.data_class().get(number31.id)
    assert formdata.evolution[-1].parts[0].orig_filename == 'xxx.xml'
    assert formdata.evolution[-1].parts[0].content_type == 'text/xml'
    assert formdata.get_substitution_variables()['attachments'].xxx.filename == 'xxx.xml'
    resp = app.get(formdata.get_substitution_variables()['attachments'].xxx.url)
    resp = resp.follow()
    assert resp.text == '<?xml version="1.0"><foo/>'

def test_backoffice_wfedit(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    number31.submission_context = {
            'mail_url': 'http://www.example.com/test.pdf',
    }
    number31.store()

    # attach a custom workflow
    workflow = Workflow(name='wfedit')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wfedit = EditableWorkflowStatusItem()
    wfedit.id = '_wfedit'
    wfedit.by = [user.roles[0]]
    st1.items.append(wfedit)
    wfedit.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    resp = resp.form.submit('button_wfedit')
    resp = resp.follow()
    assert 'http://www.example.com/test.pdf' in resp.text # make sure sidebar has details
    assert not 'Tracking Code' in resp.text # make sure it doesn't display a tracking code
    assert resp.form['f1'].value == number31.data['1']
    assert resp.form['f2'].value == number31.data['2']
    assert resp.form['f3'].value == number31.data['3']
    assert 'value="Save Changes"' in resp.text
    resp.form['f2'].value = 'bar'
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert form_class().get(number31.id).data['2'] == 'bar'
    number31.store()

def test_backoffice_wfedit_submission(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.enable_tracking_codes = True

    formdef.fields.insert(0, fields.PageField(id='0', label='1st page', type='page'))
    formdef.fields.append(fields.PageField(id='4', label='2nd page', type='page'))

    formdef.store()
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    number31.backoffice_submission = True
    number31.store()
    formdata_count = form_class.count()

    # attach a custom workflow
    workflow = Workflow(name='wfedit')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wfedit = EditableWorkflowStatusItem()
    wfedit.id = '_wfedit'
    wfedit.by = [user.roles[0]]
    st1.items.append(wfedit)
    wfedit.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text
    resp = resp.form.submit('button_wfedit')
    resp = resp.follow()
    assert resp.form['f1'].value == number31.data['1']
    assert resp.form['f2'].value == number31.data['2']
    assert resp.form['f3'].value == number31.data['3']
    resp.form['f2'].value = 'bar'
    resp = resp.form.submit('submit')
    assert 'value="Save Changes"' in resp.text
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert form_class().get(number31.id).data['2'] == 'bar'
    number31.store()
    assert formdata_count == form_class.count()

def test_backoffice_wfedit_and_backoffice_fields(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='wfedit')
    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
    workflow.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='1st backoffice field',
            type='string', varname='backoffice_blah', required=False),
    ]

    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wfedit = EditableWorkflowStatusItem()
    wfedit.id = '_wfedit'
    wfedit.by = [user.roles[0]]
    st1.items.append(wfedit)
    wfedit.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    number31 = form_class().get(number31.id)
    number31.data['bo1'] = 'plop'
    number31.store()

    app = login(get_app(pub))

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    resp = resp.form.submit('button_wfedit')
    resp = resp.follow()
    resp.form['f2'].value = 'bar'
    resp = resp.form.submit('submit')
    resp = resp.follow()
    assert form_class().get(number31.id).data['bo1'] == 'plop'

def test_backoffice_wfedit_and_data_source_with_user_info(pub):
    user = create_user(pub)
    create_environment(pub)
    formdef = FormDef.get_by_urlname('form-title')

    formdef.fields[2].data_source = {'type': 'json',
            'value': 'https://www.example.invalid/?name_id={% firstof form_user_display_name "XXX" %}'
            }
    formdef.store()

    form_class = formdef.data_class()

    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]

    # attach a custom workflow
    workflow = Workflow(name='wfedit')
    st1 = workflow.add_status('Status1', number31.status.split('-')[1])

    wfedit = EditableWorkflowStatusItem()
    wfedit.id = '_wfedit'
    wfedit.by = [user.roles[0]]
    st1.items.append(wfedit)
    wfedit.parent = st1
    workflow.store()

    formdef.workflow_id = workflow.id
    formdef.store()

    number31 = form_class().get(number31.id)
    number31.user_id = user.id
    number31.store()

    app = login(get_app(pub))

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data = {'data': [{'id': 'A', 'text': 'hello'}, {'id': 'B', 'text': 'world'}]}
        def side_effect(url, *args):
            assert '?name_id=admin' in url
            return StringIO(json.dumps(data))

        urlopen.side_effect = side_effect

        resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
        resp = resp.form.submit('button_wfedit')
        resp = resp.follow()
        assert urlopen.call_count == 1
        resp.form['f3'].value = 'A'
        resp = resp.form.submit('submit')
        assert urlopen.call_count == 2
        resp = resp.follow()

def test_global_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    create_user(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/').follow()
    assert 'Global View' in resp.text
    resp = resp.click('Global View')
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 20
    assert not 'Map View' in resp.text

    resp = app.get('/backoffice/management/listing?limit=500')
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37 # 17 formdef1 + 20 formdef2

    resp = app.get('/backoffice/management/listing?offset=20&limit=20')
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 17

    # try an overbound offset
    resp = app.get('/backoffice/management/listing?offset=40&limit=20')
    resp = resp.follow()
    assert resp.forms['listing-settings']['offset'].value == '0'

    resp = app.get('/backoffice/management/listing')
    resp.forms['listing-settings']['end'] = '2014-02-01'
    resp = resp.forms['listing-settings'].submit()

    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 20
    assert 'http://example.net/backoffice/management/other-form/' in resp.text
    assert not 'http://example.net/backoffice/management/form-title/' in resp.text

    formdef = FormDef.get_by_urlname('form-title')
    last_update_time = formdef.data_class().select(lambda x: not x.is_draft())[0].last_update_time
    # check created and last modified columns
    assert '>2014-01-01 00:00<' in resp.text
    assert time.strftime('>%Y-%m-%d', last_update_time) in resp.text

    # check digest is included
    formdata = formdef.data_class().get(
            re.findall(r'data-link="(.*?)"', app.get('/backoffice/management/listing').text)[0].split('/')[-2])
    formdata.formdef.digest_template = 'digest of number <{{form_number}}>'
    formdata.store()
    assert formdata.get(formdata.id).digest
    resp = app.get('/backoffice/management/listing')
    assert formdata.get_url(backoffice=True) in resp.text
    assert 'digest of number &lt;%s&gt;' % formdata.id_display in resp.text

    # check a Channel column is added when welco is available
    assert not 'Channel' in resp.text

    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'welco_url', 'xxx')
    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
    pub.site_options.write(fd)
    fd.close()

    resp = app.get('/backoffice/management/listing?limit=500')
    formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
    formdata.submission_channel = 'mail'
    formdata.store()
    assert 'Channel' in resp.text
    assert '>Web<' in resp.text
    resp.forms['listing-settings']['submission_channel'] = 'web'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 36
    resp.forms['listing-settings']['submission_channel'] = 'mail'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 1

    resp = app.get('/backoffice/management/listing?limit=500')
    resp.forms['listing-settings']['q'] = 'foo'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 17

    resp = app.get('/backoffice/management/listing?limit=500')
    resp.forms['listing-settings']['status'] = 'waiting'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37
    resp.forms['listing-settings']['status'] = 'open'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37
    resp.forms['listing-settings']['status'] = 'all'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 70
    resp.forms['listing-settings']['status'] = 'done'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 33

    # change role handling a formdef, make sure they do not appear anylonger in
    # the all/done views.
    role = Role(name='whatever')
    role.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_roles = {'_receiver': role.id}
    formdef.store()
    formdef.data_class().rebuild_security()

    resp = app.get('/backoffice/management/listing?limit=500')
    resp.forms['listing-settings']['status'] = 'waiting'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 20
    assert not 'form-title' in resp.text
    resp.forms['listing-settings']['status'] = 'open'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 20
    assert not 'form-title' in resp.text
    resp.forms['listing-settings']['status'] = 'all'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 20
    assert not 'form-title' in resp.text
    resp.forms['listing-settings']['status'] = 'done'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 0
    assert not 'form-title' in resp.text

def test_global_listing_user_label(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    create_user(pub)
    FormDef.wipe()

    from wcs.admin.settings import UserFieldsFormDef
    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.StringField(id='3', label='first_name', type='string'))
    user_formdef.fields.append(fields.StringField(id='4', label='last_name', type='string'))
    user_formdef.store()
    pub.cfg['users']['field_name'] = ['3', '4']
    pub.write_cfg()

    formdef = FormDef()
    formdef.name = 'foobar'
    formdef.url_name = 'foobar'
    formdef.workflow_roles = {'_receiver': 1}
    formdef.fields = [
        fields.StringField(id='1', label='first_name',
        prefill={'type': 'user', 'value': '3'}),
        fields.StringField(id='2', label='last_name',
        prefill={'type': 'user', 'value': '4'}),
    ]
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {'1': 'blah', '2': 'xxx'}
    formdata.just_created()
    formdata.store()
    formdata.jump_status('new')

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/').follow()
    resp = resp.click('Global View')
    assert '<td class="cell-user">blah xxx</td>' in resp.text

def test_management_views_with_no_formdefs(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    create_user(pub)
    create_environment(pub)
    FormDef.wipe()

    from wcs.sql import get_connection_and_cursor, drop_global_views
    conn, cur = get_connection_and_cursor()
    drop_global_views(conn, cur)
    conn.commit()
    cur.close()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/forms')
    assert 'This site is currently empty.' in resp.text
    resp = app.get('/backoffice/management/listing')
    assert 'This site is currently empty.' in resp.text

def test_category_in_global_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/listing?limit=500')
    assert not 'category_id' in resp.forms['listing-settings'].fields

    cat1 = Category(name='cat1')
    cat1.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.category_id = cat1.id
    formdef.store()

    cat2 = Category(name='cat2')
    cat2.store()
    formdef = FormDef.get_by_urlname('other-form')
    formdef.category_id = cat2.id
    formdef.store()

    resp = app.get('/backoffice/management/listing?limit=500')
    assert 'category_id' in resp.forms['listing-settings'].fields
    assert 'management/other-form/' in resp.text
    assert 'management/form-title/' in resp.text

    resp.forms['listing-settings']['category_id'] = cat1.id
    resp = resp.forms['listing-settings'].submit()
    assert not 'management/other-form/' in resp.text
    assert 'management/form-title/' in resp.text

    resp.forms['listing-settings']['category_id'] = cat2.id
    resp = resp.forms['listing-settings'].submit()
    assert 'management/other-form/' in resp.text
    assert not 'management/form-title/' in resp.text

def test_datetime_in_global_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/listing?limit=500')
    resp.forms['listing-settings']['end'] = '01/01/2010'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 0

    resp = app.get('/backoffice/management/listing?limit=500')
    resp.forms['listing-settings']['start'] = '01/01/2010'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37

    resp.forms['listing-settings']['start'] = '01/01/2016'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 0

    resp.forms['listing-settings']['start'] = '01/01/16'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 0

    resp.forms['listing-settings']['start'] = '01/01/10'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37

    resp.forms['listing-settings']['end'] = '01/01/10'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 0

    # check invalid values are simply ignored
    resp.forms['listing-settings']['end'] = 'whatever'
    resp = resp.forms['listing-settings'].submit()
    assert resp.text[resp.text.index('<tbody'):].count('<tr') == 37

def test_global_listing_geojson(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.geolocations = {'base': 'Geolocafoobar'}
    formdef.store()
    for formdata in formdef.data_class().select():
        formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/geojson')
    assert len(resp.json['features']) == 17
    assert resp.json['features'][0]['geometry']['coordinates'] == [2.32, 48.83]
    for feature in resp.json['features']:
        assert feature['properties']['status_colour'] == '#66FF00'
        assert feature['properties']['view_label'] == 'View'
        assert feature['properties']['status_name'] == 'New'
        assert feature['properties']['display_fields']
        assert feature['properties']['display_fields'][0]['label'] == 'Name'
        assert feature['properties']['display_fields'][0]['value'].startswith('form title #')

    resp = app.get('/backoffice/management/geojson?q=aa')
    assert len(resp.json['features']) == 5
    resp = app.get('/backoffice/management/geojson?q=bb')
    assert len(resp.json['features']) == 4
    resp = app.get('/backoffice/management/geojson?q=cc')
    assert len(resp.json['features']) == 8

def test_global_map(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.geolocations = {'base': 'Geolocafoobar'}
    formdef.store()
    for formdata in formdef.data_class().select():
        formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
        formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/listing')
    assert 'Map View' in resp.text
    resp = app.get('/backoffice/management/forms')
    assert 'Map View' in resp.text
    resp = resp.click('Map View')
    assert re.findall(r'data-geojson-url="(.*?)"', resp.text) == ['http://example.net/backoffice/management/geojson?']

    resp = app.get('/backoffice/management/map?q=test')
    assert re.findall(r'data-geojson-url="(.*?)"', resp.text) == ['http://example.net/backoffice/management/geojson?q=test']

def test_formdata_lookup(pub):
    create_user(pub)
    create_environment(pub, set_receiver=False)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.enable_tracking_codes = True
    formdef.store()
    formdata, formdata2 = formdef.data_class().select(order_by='id')[:2]
    code = pub.tracking_code_class()
    code.formdata = formdata

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/').follow()
    assert 'id="lookup-box"' in resp.text
    resp.forms[0]['query'] = formdata.tracking_code
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata.id
    resp = resp.follow()
    assert 'The form has been recorded' in resp.text
    assert 'This form has been accessed via its tracking code' in resp.text

    # check there's no access to other formdata
    app.get('http://example.net/backoffice/management/form-title/%s/' % formdata2.id, status=403)

    resp = app.get('/backoffice/management/').follow()
    resp.forms[0]['query'] = 'AAAAAAAA'
    resp = resp.forms[0].submit()
    assert resp.location == 'http://example.net/backoffice/management/'
    resp = resp.follow().follow()
    assert 'No such tracking code or identifier.' in resp.text

    # check looking up a formdata id
    resp.form['query'] = formdata.get_display_id()
    resp = resp.form.submit()
    assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata.id

    if pub.is_using_postgresql():
        # check looking up on a custom display_id
        formdata.id_display = '999999'
        formdata.store()
        assert formdata.get_display_id() == '999999'
        resp = app.get('/backoffice/management/').follow()
        resp.form['query'] = formdata.get_display_id()
        resp = resp.form.submit()
        assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata.id

        # try it from the global listing
        resp = app.get('/backoffice/management/listing')
        assert 'id="lookup-box"' in resp.text
        resp.forms[0]['query'] = formdata.tracking_code
        resp = resp.forms[0].submit()
        assert resp.location == 'http://example.net/backoffice/management/form-title/%s/' % formdata.id
        resp = resp.follow()
        assert 'The form has been recorded' in resp.text

        resp = app.get('/backoffice/management/listing')
        resp.forms[0]['query'] = 'AAAAAAAA'
        resp = resp.forms[0].submit()
        assert resp.location == 'http://example.net/backoffice/management/listing'
        resp = resp.follow()
        assert 'No such tracking code or identifier.' in resp.text

def test_backoffice_sidebar_user_context(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    assert re.findall('<tbody.*\/tbody>', resp.text, re.DOTALL)[0].count('<tr') == 17

    # click on a formdata
    resp = resp.click(href='%s/' % number31.id)
    assert (' with the number %s.' % number31.get_display_id()) in resp.text

    # check there's nothing in the sidebar
    assert not '/user-pending-forms' in resp.text

    number31.formdef.digest_template = 'digest of number {{form_number}}'
    number31.user_id = user.id
    number31.store()
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert '/user-pending-forms' in resp.text
    user_pending_form_url = re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0]
    partial_resp = app.get(user_pending_form_url)
    assert not number31.get_url(backoffice=True) in partial_resp.text
    assert number31.digest in partial_resp.text
    assert '<span class="formname">%s</span>' % number31.formdef.name in partial_resp.text

    # another item with status = new
    number34 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 33'][0]
    number34.user_id = user.id
    number34.store()

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert '/user-pending-forms' in resp.text
    user_pending_form_url = re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0]
    partial_resp = app.get(user_pending_form_url)
    assert not number31.get_url(backoffice=True) in partial_resp.text
    assert number34.get_url(backoffice=True) in partial_resp.text

    cat1 = Category(name='cat1')
    cat1.store()

    formdef = FormDef.get_by_urlname('other-form')
    formdef.category_id = cat1.id
    formdef.store()
    other_formdata = formdef.data_class().select()[0]
    other_formdata.user_id = user.id
    other_formdata.store()

    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert '/user-pending-forms' in resp.text
    user_pending_form_url = re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0]
    partial_resp = app.get(user_pending_form_url)
    assert number34.get_url(backoffice=True) in partial_resp.text
    assert other_formdata.get_url(backoffice=True) in partial_resp.text
    # categories are displayed, and current formdata category is on top
    assert '>cat1<' in partial_resp.text
    assert '>Misc<' in partial_resp.text
    assert partial_resp.text.index('>Misc<') < partial_resp.text.index('>cat1<')

def test_count_open(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    create_user(pub)

    FormDef.wipe()
    resp = login(get_app(pub)).get('/backoffice/management/count')
    assert resp.json['count'] == 0

    create_environment(pub)
    resp = login(get_app(pub)).get('/backoffice/management/count')
    assert resp.json['count'] == 37

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_roles = {'_receiver': 2} # role the user doesn't have
    formdef.store()
    formdef.data_class().rebuild_security()
    resp = login(get_app(pub)).get('/backoffice/management/count')
    assert resp.json['count'] == 20

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_roles = {'_receiver': 2, '_foobar': 1}
    formdef.store()
    formdef.data_class().rebuild_security()
    resp = login(get_app(pub)).get('/backoffice/management/count')
    assert resp.json['count'] == 20
    resp = login(get_app(pub)).get('/backoffice/management/count?waiting=yes')
    assert resp.json['count'] == 20

    formdef = FormDef.get_by_urlname('form-title')
    workflow = Workflow.get_default_workflow()
    workflow.roles['_foobar'] = 'Foobar'
    workflow.id = '2'
    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.workflow_roles = {'_receiver': 2, '_foobar': '1'}
    formdef.store()
    formdef.data_class().rebuild_security()
    resp = login(get_app(pub)).get('/backoffice/management/count?waiting=no')
    assert resp.json['count'] == 37
    resp = login(get_app(pub)).get('/backoffice/management/count?waiting=yes')
    assert resp.json['count'] == 20
    resp = login(get_app(pub)).get('/backoffice/management/count')
    assert resp.json['count'] == 20

    # check the callback parameter is ignored, that we still get the default
    # criterias when it's set.
    resp = login(get_app(pub)).get('/backoffice/management/count?callback=toto')
    assert "20" in resp.text

def test_count_backoffice_drafts(pub):
    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.store()

    resp = login(get_app(pub)).get('/backoffice/submission/count')
    assert resp.json['count'] == 0

    formdata1, formdata2, formdata3 = formdef.data_class().select()[:3]
    for formdata in (formdata1, formdata2, formdata3):
        formdata.status = 'draft'
        formdata.store()

    resp = login(get_app(pub)).get('/backoffice/submission/count')
    assert resp.json['count'] == 0

    for formdata in (formdata1, formdata2, formdata3):
        formdata.backoffice_submission = True
        formdata.store()

    resp = login(get_app(pub)).get('/backoffice/submission/count')
    assert resp.json['count'] == 3

    formdata1.data = {}
    formdata1.store()

    resp = login(get_app(pub)).get('/backoffice/submission/count?mode=empty')
    assert resp.json['count'] == 1
    resp = login(get_app(pub)).get('/backoffice/submission/count?mode=existing')
    assert resp.json['count'] == 2

def test_menu_json(pub):
    user = create_user(pub)
    create_environment(pub)
    resp = login(get_app(pub)).get('/backoffice/menu.json')
    menu_json_str = resp.text
    assert len(resp.json) == 1
    assert resp.json[0]['slug'] == 'management'
    assert resp.headers['content-type'] == 'application/json'

    resp = login(get_app(pub)).get('/backoffice/menu.json?jsonpCallback=foo')
    assert resp.text == 'foo(%s);' % menu_json_str
    assert resp.headers['content-type'] == 'application/javascript'

def test_per_user_view(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/management/').follow()
    assert 'Per User View' not in resp.text

    if not pub.site_options.has_section('options'):
        pub.site_options.add_section('options')
        pub.site_options.set('options', 'per-user-view', 'true')
        fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
        pub.site_options.write(fd)
        fd.close()

    resp = app.get('/backoffice/management/').follow()
    assert 'Per User View' in resp.text
    resp = resp.click(' User View')
    assert 'Use the search field on the right' in resp.text

    resp.form['q'] = 'admin'
    resp = resp.form.submit()

    assert resp.text.count('<tr') == 2 # header + user

    form_class = FormDef.get_by_urlname('form-title').data_class()
    to_match = []
    for formdata in form_class.select():
        if formdata.data['1'] in ('FOO BAR 30', 'FOO BAR 33'):
            formdata.user_id = user.id
            formdata.store()
            to_match.append('/management/form-title/%s/' % formdata.id)

    resp = app.get('/backoffice/management/users/%s/' % user.id)
    for item in to_match:
        assert item in resp.text
    count_li = resp.text.count('<li')

    # check list items are displayed, without links, if we cannot view them.
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_roles['_receiver'] = 'XXX'
    formdef.store()
    formdef.data_class().rebuild_security()

    resp = app.get('/backoffice/management/users/%s/' % user.id)
    for item in to_match:
        assert not item in resp.text # not linked
    assert resp.text.count('<li') == count_li

    # mark formdef so it's not listed in per-user view
    formdef.skip_from_360_view = True
    formdef.store()

    # check formdatas are no longer part of the page
    resp = app.get('/backoffice/management/users/%s/' % user.id)
    assert resp.text.count('<li') == (count_li - 2)

def test_per_user_view_tracking_code(pub, emails, sms_mocking):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)
    app = login(get_app(pub))

    formdef = FormDef.get_by_urlname('form-title')
    form_class = formdef.data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    number31.user_id = user.id
    number31.store()

    resp = app.get('/backoffice/management/users/%s/' % user.id)
    assert not 'Send tracking code' in resp.text

    formdef.enable_tracking_codes = True
    formdef.store()
    user_view_resp = app.get('/backoffice/management/users/%s/' % user.id)
    assert 'Send tracking code' in user_view_resp.text

    pub.cfg['sms'] = {'mode': 'none'}
    pub.write_cfg()
    resp = user_view_resp.click('Send tracking code')
    assert not 'sms' in resp.form.fields
    assert resp.form['email'].value == user.email

    resp = resp.form.submit()
    resp = resp.follow()
    assert emails.count() == 1
    assert emails.get('Tracking Code reminder')['email_rcpt'] == [user.email]
    assert form_class.get(number31.id).tracking_code in emails.get('Tracking Code reminder')['payload']

    pub.cfg['sms'] = {'mode': 'xx'}
    pub.write_cfg()
    resp = user_view_resp.click('Send tracking code', index=0)
    resp.form['method'].value = 'SMS'
    resp.form['sms'].value = '0123456789'
    resp = resp.form.submit()
    resp = resp.follow()
    assert sms_mocking.sms[-1]['destinations'] == ['0123456789']
    assert form_class.get(number31.id).tracking_code in sms_mocking.sms[-1]['text']

def test_backoffice_resume_folded(pub):
    user = create_user(pub)
    create_environment(pub)
    form_class = FormDef.get_by_urlname('form-title').data_class()
    number31 = [x for x in form_class.select() if x.data['1'] == 'FOO BAR 30'][0]
    app = login(get_app(pub))

    # first access: summary is not folded
    resp = app.get('/backoffice/management/form-title/%s/' % number31.id)
    assert '<div class="section foldable" id="summary">' in resp.text
    # do something: summary is folded
    resp = resp.form.submit('button_commentable')
    resp = resp.follow()
    assert '<div class="section foldable folded" id="summary">' in resp.text

def test_backoffice_backoffice_submission_in_listings(pub):
    create_superuser(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    first_link = re.findall('data-link="(\d+)/?"', resp.text)[0]
    assert not 'backoffice-submission' in resp.text

    formdata = FormDef.get_by_urlname('form-title').data_class().get(first_link)
    formdata.backoffice_submission = True
    formdata.store()

    resp = app.get('/backoffice/management/form-title/')
    assert 'backoffice-submission' in resp.text

def test_backoffice_backoffice_submission_in_global_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    create_superuser(pub)
    create_environment(pub)

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/listing?limit=100')
    assert not 'backoffice-submission' in resp.text

    formdef = FormDef.get_by_urlname('form-title')
    formdata = formdef.data_class().get(
            re.findall(r'data-link="(.*?)"', app.get('/backoffice/management/listing').text)[0].split('/')[-2])
    formdata.backoffice_submission = True
    formdata.store()

    resp = app.get('/backoffice/management/listing?limit=100')
    assert 'backoffice-submission' in resp.text

def test_backoffice_advisory_lock(pub):
    create_superuser(pub)
    create_environment(pub)

    second_user = pub.user_class(name='foobar')
    second_user.roles = Role.keys()
    second_user.store()
    account = PasswordAccount(id='foobar')
    account.set_password('foobar')
    account.user_id = second_user.id
    account.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/')
    first_link = re.findall('data-link="(\d+/)?"', resp.text)[0]
    assert not 'advisory-lock' in resp.text

    app2 = login(get_app(pub), username='foobar', password='foobar')
    resp = app2.get('/backoffice/management/form-title/')
    assert not 'advisory-lock' in resp.text

    resp = app.get('/backoffice/management/form-title/' + first_link)
    resp = app2.get('/backoffice/management/form-title/')
    assert 'advisory-lock' in resp.text
    resp = app.get('/backoffice/management/form-title/')
    assert not 'advisory-lock' in resp.text

    if pub.is_using_postgresql():
        # check global view
        resp = app2.get('/backoffice/management/listing?limit=100')
        assert 'advisory-lock' in resp.text
        resp = app.get('/backoffice/management/listing?limit=100')
        assert not 'advisory-lock' in resp.text

    resp = app.get('/backoffice/management/form-title/' + first_link)
    assert not 'Be warned forms of this user are also being looked' in resp.text
    assert 'button_commentable' in resp.text
    assert len(resp.forms)
    resp = app2.get('/backoffice/management/form-title/' + first_link)
    assert 'Be warned forms of this user are also being looked' in resp.text
    assert not 'button_commentable' in resp.text
    assert len(resp.forms) == 0
    # revisit with first user, no change
    resp = app.get('/backoffice/management/form-title/' + first_link)
    assert not 'Be warned forms of this user are also being looked' in resp.text
    assert 'button_commentable' in resp.text
    # back to second
    resp = app2.get('/backoffice/management/form-title/' + first_link)
    assert 'Be warned forms of this user are also being looked' in resp.text
    assert not 'button_commentable' in resp.text
    resp = resp.click('(unlock actions)')
    resp = resp.follow()
    assert 'Be warned forms of this user are also being looked' in resp.text
    assert 'button_commentable' in resp.text
    assert not '(unlock actions)' in resp.text
    assert len(resp.forms)

    # submit action form
    resp.form['comment'] = 'HELLO'
    resp = resp.form.submit('button_commentable')
    # locks are reset after an action
    assert not 'advisory-lock' in app2.get('/backoffice/management/form-title/')
    assert not 'advisory-lock' in app.get('/backoffice/management/form-title/')
    # but as the current user is redirected to the form, a lock will be
    # acquired (unless the user didn't have actions anymore, but it's not the
    # case here)
    resp = resp.follow()
    assert not 'advisory-lock' in app2.get('/backoffice/management/form-title/')
    assert 'advisory-lock' in app.get('/backoffice/management/form-title/')

def test_backoffice_advisory_lock_related_formdatas(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    pub.session_manager.session_class.wipe()
    user = create_superuser(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdatas = formdef.data_class().select(lambda x: x.status == 'wf-new')

    second_user = pub.user_class(name='foobar')
    second_user.roles = Role.keys()
    second_user.store()
    account = PasswordAccount(id='foobar')
    account.set_password('foobar')
    account.user_id = second_user.id
    account.store()

    third_user = pub.user_class(name='user')
    third_user.store()

    for formdata in formdatas[:2]:
        formdata.user_id = third_user.id
        formdata.store()

    second_formdef = FormDef.get_by_urlname('other-form')
    second_formdef.workflow_roles = {}
    second_formdef.store()
    other_formdatas = second_formdef.data_class().select(lambda x: x.status == 'wf-new')
    other_formdatas[0].user_id = third_user.id
    other_formdatas[0].store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/%s/' % formdatas[0].id)
    assert not 'Be warned forms of this user are also being looked' in resp.text
    app.get(re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0])

    app2 = login(get_app(pub), username='foobar', password='foobar')
    resp2 = app2.get('/backoffice/management/form-title/%s/' % formdatas[0].id)
    assert 'Be warned forms of this user are also being looked' in resp2.text
    app2.get(re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0])

    # another by the same user
    resp2 = app2.get('/backoffice/management/form-title/%s/' % formdatas[1].id)
    assert 'Be warned forms of this user are also being looked' in resp2.text
    app2.get(re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0])

    # another by another user
    resp2 = app2.get('/backoffice/management/form-title/%s/' % formdatas[3].id)
    assert not 'Be warned forms of this user are also being looked' in resp2.text
    app2.get(re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0])

    # check another formdef is only marked as visited if the user has potential
    # actions on it.
    session = pub.session_manager.session_class.select(lambda x: x.user == user.id)[0]
    second_formdef.workflow_roles = {'_receiver': 1}
    second_formdef.store()
    other_formdata = second_formdef.data_class().get(other_formdatas[0].id)
    other_formdata.store() # update concerned_roles

    assert not 'formdata-other-form-%d' % other_formdata.id in session.visiting_objects.keys()
    session.visiting_objects = {}
    session.store()

    resp = app.get('/backoffice/management/form-title/%s/' % formdatas[0].id)
    app.get(re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0])
    session = pub.session_manager.session_class.select(lambda x: x.user == user.id)[0]
    assert 'formdata-other-form-%d' % other_formdata.id in session.visiting_objects.keys()

def test_backoffice_resubmit(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='resubmit')
    st1 = wf.add_status('Status1')
    st2 = wf.add_status('Status2')

    resubmit = ResubmitWorkflowStatusItem()
    resubmit.id = '_resubmit'
    resubmit.by = [user.roles[0]]
    st1.items.append(resubmit)
    resubmit.parent = st1

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1

    register_comment = RegisterCommenterWorkflowStatusItem()
    register_comment.id = '_register'
    register_comment.comment = '<p><a href="[resubmit_formdata_backoffice_url]">resubmitted</a></p>'
    st2.items.append(register_comment)
    register_comment.parent = st2

    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.fields[0].varname = 'foo'
    formdef.workflow_id = wf.id
    formdef.store()

    formdef2 = FormDef()
    formdef2.name = 'form title bis'
    formdef2.backoffice_submission_roles = user.roles[:]
    formdef2.fields = [
        fields.StringField(id='2', label='1st field', type='string', varname='foo')
    ]
    formdef2.store()
    formdef2.data_class().wipe()

    formdata = formdef.data_class()()
    formdata.just_created()
    formdata.data= {'1': 'XXX'}
    formdata.store()

    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    resp.form['resubmit'].value = formdef2.id
    resp = resp.form.submit('button_resubmit')
    resp = resp.follow()
    assert 'resubmitted' in resp.text
    assert formdef2.data_class().select()[0].status == 'draft'
    assert formdef2.data_class().select()[0].data == {'2': 'XXX'}
    resp = resp.click('resubmitted')
    resp = resp.follow()
    resp = resp.follow()
    assert resp.form['f2'].value == 'XXX'
    assert 'Original form' in resp.text
    assert formdata.get_url(backoffice=True) in resp.text

def test_backoffice_workflow_display_form(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = [user.roles[0]]
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields.append(
            fields.StringField(
                id='1', label='Test', varname='str', type='string', required=True))
    display_form.formdef.fields.append(
            # mimick special case: https://dev.entrouvert.org/issues/14691
            # item field displayed as radio buttons, with prefill of a value
            # that doesn't exist.
            fields.ItemField(
                id='2', label='Test2', type='item',
                prefill={'type': 'string', 'value': ''},
                display_mode='radio',
                varname='radio',
                items=['a', 'b', 'c'], required=True))
    status.items.append(display_form)
    display_form.parent = status

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = 'accepted'
    status.items.append(jump)
    jump.parent = status

    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.store()

    for formdata in formdef.data_class().select():
        if formdata.status == 'wf-new':
            break
    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    resp.form['f1'] = 'blah'
    # don't fill required radio button
    resp = resp.form.submit('submit')
    assert formdef.data_class().get(formdata.id).status == 'wf-new'
    assert 'There were errors processing your form.' in resp.text
    resp.form['f2'] = 'c'
    resp = resp.form.submit('submit')
    assert formdef.data_class().get(formdata.id).status == 'wf-accepted'
    assert formdef.data_class().get(formdata.id).workflow_data == {
            'blah_var_str': 'blah', 'blah_var_radio': 'c', 'blah_var_radio_raw': 'c'}

def test_backoffice_workflow_form_with_conditions(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = [user.roles[0]]
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2', type='string', required=True),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.fields[0].varname = 'plop'
    formdef.store()

    for formdata in formdef.data_class().select():
        if formdata.status == 'wf-new':
            break
    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    # check with static condition
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': '0'}),
    ]
    wf.store()

    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    assert 'f2' not in resp.form.fields

    # check condition based on formdata
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'form_var_plop'}),
    ]
    wf.store()

    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'form_var_plop != "xxx"'}),
    ]
    wf.store()

    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields

    # check with live conditions
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.StringField(id='2', label='Test2', varname='str2',
                type='string', required=True,
                condition={'type': 'django', 'value': 'blah_var_str == "xxx"'}),
    ]
    wf.store()

    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True))
    assert 'f1' in resp.form.fields
    assert 'f2' in resp.form.fields
    assert resp.html.find('div', {'data-field-id': '1'}).attrs['data-live-source'] == 'true'
    assert resp.html.find('div', {'data-field-id': '2'}).attrs.get('style') == 'display: none'
    live_url = resp.html.find('form').attrs['data-live-url']
    assert '/backoffice/' in live_url
    resp.form['f1'] = ''
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']

    resp.form['f1'] = 'xxx'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert live_resp.json['result']['2']['visible']

    # check submit doesn't work
    resp = resp.form.submit('submit')
    assert 'There were errors processing your form.' in resp.text

    resp.form['f1'] = 'xxx2'
    live_resp = app.post(live_url, params=resp.form.submit_fields())
    assert live_resp.json['result']['1']['visible']
    assert not live_resp.json['result']['2']['visible']

    # check submit does work when second field is hidden
    resp = resp.form.submit('submit').follow()

    assert formdef.data_class().get(formdata.id).workflow_data == {'blah_var_str': 'xxx2', 'blah_var_str2': None}

def test_backoffice_workflow_form_with_live_data_source(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.store()
    wf = Workflow.get(wf.id)
    status = wf.get_status('new')
    status.items = []
    display_form = FormWorkflowStatusItem()
    display_form.id = '_display_form'
    display_form.by = [user.roles[0]]
    display_form.varname = 'blah'
    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
    display_form.formdef.fields = [
            fields.StringField(id='1', label='Test', varname='str', type='string', required=True),
            fields.ItemField(id='2', label='Test2', varname='str2', type='item', required=True,
                data_source={'type': 'json', 'value': 'https://www.example.invalid/{{ blah_var_str }}'}),
    ]
    status.items.append(display_form)
    display_form.parent = status

    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.fields[0].varname = 'plop'
    formdef.store()

    for formdata in formdef.data_class().select():
        if formdata.status == 'wf-new':
            break

    app = get_app(pub)

    with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
        data1 = {'data': [{'id': 'A', 'text': 'hello'}, {'id': 'B', 'text': 'world'}]}
        data2 = {'data': [{'id': 'C', 'text': 'hello'}, {'id': 'D', 'text': 'world'}]}
        def side_effect(url, *args):
            if 'toto' not in url:
                return StringIO(json.dumps(data1))
            else:
                return StringIO(json.dumps(data2))
        urlopen.side_effect = side_effect

        resp = login(app).get(formdata.get_url(backoffice=True))
        assert 'f1' in resp.form.fields
        assert 'f2' in resp.form.fields
        assert resp.form.fields['f2'][0].options == [(u'A', False, u'hello'), (u'B', False, u'world')]

        live_url = resp.html.find('form').attrs['data-live-url']
        resp.form['f1'] = 'toto'
        live_resp = app.post(live_url + '?modified_field_id=1', params=resp.form.submit_fields())
        assert live_resp.json['result']['2']['items'] == [{u'text': u'hello', u'id': u'C'}, {u'text': u'world', u'id': u'D'}]

def test_backoffice_criticality_in_formdef_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.store()

    formdata1, formdata2, formdata3, formdata4 = [
            x for x in formdef.data_class().select() if x.status == 'wf-new'][:4]

    formdata1.set_criticality_level(1)
    formdata1.store()
    formdata1_str = '>%s<' % formdata1.get_display_id()
    formdata2.set_criticality_level(2)
    formdata2.store()
    formdata2_str = '>%s<' % formdata2.get_display_id()
    formdata3.set_criticality_level(2)
    formdata3.store()
    formdata3_str = '>%s<' % formdata3.get_display_id()
    formdata4_str = '>%s<' % formdata4.get_display_id()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/?order_by=-criticality_level&limit=100')
    assert resp.text.index(formdata1_str) > resp.text.index(formdata2_str)
    assert resp.text.index(formdata1_str) > resp.text.index(formdata3_str)
    assert resp.text.index(formdata1_str) < resp.text.index(formdata4_str)

    resp = app.get('/backoffice/management/form-title/?order_by=criticality_level&limit=100')
    assert resp.text.index(formdata1_str) < resp.text.index(formdata2_str)
    assert resp.text.index(formdata1_str) < resp.text.index(formdata3_str)
    assert resp.text.index(formdata1_str) > resp.text.index(formdata4_str)

def test_backoffice_criticality_in_global_listing(pub):
    if not pub.is_using_postgresql():
        pytest.skip('this requires SQL')
        return

    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.store()

    formdata1, formdata2, formdata3, formdata4 = [
            x for x in formdef.data_class().select() if x.status == 'wf-new'][:4]

    formdata1.set_criticality_level(1)
    formdata1.store()
    formdata1_str = '>%s<' % formdata1.get_display_id()
    formdata2.set_criticality_level(2)
    formdata2.store()
    formdata2_str = '>%s<' % formdata2.get_display_id()
    formdata3.set_criticality_level(2)
    formdata3.store()
    formdata3_str = '>%s<' % formdata3.get_display_id()
    formdata4_str = '>%s<' % formdata4.get_display_id()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/listing?order_by=-criticality_level&limit=100')
    assert resp.text.index(formdata1_str) > resp.text.index(formdata2_str)
    assert resp.text.index(formdata1_str) > resp.text.index(formdata3_str)
    assert resp.text.index(formdata1_str) < resp.text.index(formdata4_str)

    resp = app.get('/backoffice/management/listing?order_by=criticality_level&limit=100')
    assert resp.text.index(formdata1_str) < resp.text.index(formdata2_str)
    assert resp.text.index(formdata1_str) < resp.text.index(formdata3_str)
    assert resp.text.index(formdata1_str) > resp.text.index(formdata4_str)

def test_backoffice_criticality_formdata_view(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow.get_default_workflow()
    wf.id = '2'
    wf.criticality_levels = [
        WorkflowCriticalityLevel(name='green'),
        WorkflowCriticalityLevel(name='yellow'),
        WorkflowCriticalityLevel(name='red'),
    ]
    wf.store()
    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdata = [x for x in formdef.data_class().select() if x.status == 'wf-new'][0]

    formdata.set_criticality_level(1)
    formdata.store()

    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'Criticality Level: yellow' in resp.text

    formdata.set_criticality_level(2)
    formdata.store()
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'Criticality Level: red' in resp.text


class IHateUnicode(object):
    def __unicode__(self):
        raise Exception('HATE!!')

    def __repr__(self):
        return 'ok'

@pytest.fixture
def local_user():
    get_publisher().user_class.wipe()
    user = get_publisher().user_class()
    user.name = 'Jean Darmette'
    user.email = 'jean.darmette@triffouilis.fr'
    user.name_identifiers = ['0123456789']
    user.store()
    return user

def test_inspect_page(pub, local_user):
    create_user(pub)
    create_environment(pub)

    formdef = FormDef.get_by_urlname('form-title')
    formdata = [x for x in formdef.data_class().select() if x.status == 'wf-new'][0]
    # temper with field 3 structured values

    formdata.data['3_structured'] = {
        'unicode': u'uné',
        'str_but_non_utf8': b'\xed\xa0\x00',  # not actually supposed to happen
        'non_unicode_convertible': IHateUnicode(),
        'very_long_string': '0' * 100000,
    }
    formdata.user_id = local_user.id
    formdata.store()

    from wcs.admin.settings import UserFieldsFormDef
    user_formdef = UserFieldsFormDef(pub)
    user_formdef.fields.append(fields.StringField(id='_first_name', label='name', type='string'))
    user_formdef.fields.append(fields.StringField(id='3', label='test', type='string'))
    user_formdef.store()
    local_user.form_data = {'_first_name': 'toto', '3': 'nono'}
    local_user.set_attributes_from_formdata(local_user.form_data)
    local_user.store()

    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True), status=200)
    assert 'Data Inspector' not in resp.text
    resp = login(get_app(pub)).get('%sinspect' % formdata.get_url(backoffice=True), status=403)

    create_user(pub, is_admin=True)
    resp = login(get_app(pub)).get(formdata.get_url(backoffice=True), status=200)
    resp = resp.click('Data Inspector')

    assert '0' * 1000 in resp.text
    assert len(resp.text) < 100000
    pq = resp.pyquery.remove_namespaces()
    assert (pq('[title="form_var_foo_unicode"]')
            .parents('li').children('div.value span')
            .text() == u'uné')
    assert (pq('[title="form_var_foo_non_unicode_convertible"]')
            .parents('li').children('div.value span')
            .text().startswith('ok '))
    if six.PY2:
        assert (pq('[title="form_var_foo_str_but_non_utf8"]')
                .parents('li').children('div.value span')
                .text() == '\'\\xed\\xa0\\x00\'')
    else:
        assert (pq('[title="form_var_foo_str_but_non_utf8"]')
                .parents('li').children('div.value span')
                .text() == "b'\\xed\\xa0\\x00' (<class 'bytes'>)")

    # don't show «unusable» variables
    assert 'form_f1' not in resp.text
    assert 'form_field_' not in resp.text
    assert 'form_user_field_' not in resp.text
    assert 'form_user_f3' not in resp.text
    assert 'form_user_f_' not in resp.text

    # check functions
    assert re.findall('Recipient.*foobar', resp.text)

    role = Role(name='plop')
    role.store()
    formdata.workflow_roles = {'_receiver': role.id}
    formdata.store()
    resp = login(get_app(pub)).get('%sinspect' % formdata.get_url(backoffice=True), status=200)
    assert re.findall('Recipient.*plop', resp.text)

    workflow = Workflow.get_default_workflow()
    workflow.id = None
    workflow.roles.update({'_plop': 'New Function'})
    workflow.store()
    formdef.workflow_id = workflow.id
    formdef.store()
    resp = login(get_app(pub)).get('%sinspect' % formdata.get_url(backoffice=True), status=200)
    assert re.findall('New Function.*unset', resp.text)

    formdata.workflow_roles = {'_receiver': role.id, '_plop': '123'}
    formdata.store()
    resp = login(get_app(pub)).get('%sinspect' % formdata.get_url(backoffice=True), status=200)
    assert re.findall('New Function.*(deleted)', resp.text)

    # test tools
    app = login(get_app(pub))
    resp = app.get('%sinspect' % formdata.get_url(backoffice=True), status=200)
    assert "Test tool" in resp.text

    resp.form['test_mode'] = 'python-condition'

    resp.form['python-condition'] = 'len(form_name) == 10'  # "form title"
    resp = resp.form.submit()
    assert 'Condition result' in resp.text
    assert 'result-true' in resp.text
    assert '<code>True</code>' in resp.text
    resp.form['python-condition'] = 'len(form_name) == 5'
    resp = resp.form.submit()
    assert 'Condition result' in resp.text
    assert 'result-false' in resp.text
    assert '<code>False</code>' in resp.text
    resp.form['python-condition'] = 'form_do_not_exist == 3'
    resp = resp.form.submit()
    assert 'Condition result' not in resp.text
    assert 'Failed to evaluate condition' in resp.text
    assert 'NameError' in resp.text

    resp.form['test_mode'] = 'django-condition'
    resp.form['django-condition'] = 'form_name|length == 10'
    resp = resp.form.submit()
    assert 'Condition result' in resp.text
    assert 'result-true' in resp.text
    resp.form['django-condition'] = 'form_name|length == 5'
    resp = resp.form.submit()
    assert 'Condition result' in resp.text
    assert 'result-false' in resp.text
    resp.form['django-condition'] = 'foo bar'
    resp = resp.form.submit()
    assert 'Condition result' not in resp.text
    assert 'TemplateSyntaxError: Unused' in resp.text

    resp.form['django-condition'] = 'form_name|length == 5'
    ajax_resp = app.post('%sinspect-tool' % formdata.get_url(backoffice=True), params=resp.form.submit_fields())
    assert ajax_resp.text.startswith('<div class="test-tool-result')

    resp.form['test_mode'] = 'template'
    resp.form['template'] = '{{ form_name }}'
    resp = resp.form.submit()
    assert 'Template rendering' in resp.text
    assert '<div class="test-tool-result-plain">form title</div>' in resp.text
    assert 'HTML Source' not in resp.text
    resp.form['template'] = '<p>{{ form_name }}</p>'
    resp = resp.form.submit()
    assert 'Template rendering' in resp.text
    assert '<p>form title</p>' in resp.text
    assert 'HTML Source' in resp.text
    resp.form['template'] = '{% for x in 1 %}ok{% endfor %}'
    resp = resp.form.submit()
    assert 'Failed to evaluate template' in resp.text
    assert 'TypeError' in resp.text
    resp.form['template'] = '{% for x in 1 %}ok{% end %}'
    resp = resp.form.submit()
    assert 'syntax error' in resp.text
    assert 'Invalid block tag' in resp.text
    resp.form['template'] = ''

    resp.form['test_mode'] = 'html_template'
    resp.form['html_template'] = '<p>{{ form_name }}</p>'
    resp = resp.form.submit()
    resp = resp.form.submit()
    assert 'Template rendering' in resp.text
    assert '<p>form title</p>' in resp.text
    assert 'HTML Source' in resp.text
    resp.form['html_template'] = '{% for x in 1 %}ok{% endfor %}'
    resp = resp.form.submit()
    assert 'Failed to evaluate HTML template' in resp.text
    assert 'TypeError' in resp.text
    resp.form['html_template'] = '{% for x in 1 %}ok{% end %}'
    resp = resp.form.submit()
    assert 'syntax error' in resp.text
    assert 'Invalid block tag' in resp.text

def test_workflow_jump_previous(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='jump around')
    #       North
    #     /      \
    # West <----> East
    #   |          |
    #   |         autojump
    #    |         |
    #     \       /
    #       South

    st1 = wf.add_status('North')
    st1.id = 'north'
    st2 = wf.add_status('West')
    st2.id = 'west'
    st3 = wf.add_status('East')
    st3.id = 'east'
    st4 = wf.add_status('Autojump')
    st4.id = 'autojump'
    st5 = wf.add_status('South')
    st5.id = 'south'

    button_by_id = {}

    def add_jump(label, src, dst_id):
        jump = ChoiceWorkflowStatusItem()
        jump.id = str(random.random())
        jump.label = label
        jump.by = ['logged-users']
        jump.status = dst_id
        src.items.append(jump)
        jump.parent = src
        if dst_id != '_previous':
            jump.set_marker_on_status = True
        button_by_id[label] = 'button%s' % jump.id
        return jump

    add_jump('Go West', st1, st2.id)
    add_jump('Go East', st1, st3.id)
    add_jump('Go South', st2, st5.id)
    add_jump('Go Autojump', st3, st4.id)
    add_jump('Go Back', st5, '_previous')

    add_jump('Jump West', st3, st2.id)
    add_jump('Jump East', st2, st3.id)

    jump = JumpWorkflowStatusItem()
    jump.id = '_auto-jump'
    jump.status = st5.id
    st4.items.append(jump)
    jump.parent = st4

    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.data_class().wipe()
    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/%s/' % formdata.id)

    # jump around using buttons
    resp = resp.form.submit(button_by_id['Go West']).follow()  # push (north)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st2.id
    resp = resp.form.submit(button_by_id['Go South']).follow() # push (north, west)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st5.id
    resp = resp.form.submit(button_by_id['Go Back']).follow()  # pop (north)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st2.id
    resp = resp.form.submit(button_by_id['Go South']).follow() # push (north, west)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st5.id
    resp = resp.form.submit(button_by_id['Go Back']).follow()  # pop (north)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st2.id
    resp = resp.form.submit(button_by_id['Jump East']).follow() # push (north, west)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st3.id
    resp = resp.form.submit(button_by_id['Go Autojump']).follow() # push (north, west, east)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st5.id

    # check markers are displayed in /inspect page
    user.is_admin = True
    user.store()
    resp2 = app.get('/backoffice/management/form-title/%s/inspect' % formdata.id)
    assert 'Markers Stack' in resp2.text
    assert '<span class="status">East</span>' in resp2.text
    assert '<span class="status">West</span>' in resp2.text
    assert '<span class="status">North</span>' in resp2.text
    assert resp2.text.find('<span class="status">East</span>') < resp2.text.find('<span class="status">West</span>')
    assert resp2.text.find('<span class="status">West</span>') < resp2.text.find('<span class="status">North</span>')

    resp = resp.form.submit(button_by_id['Go Back']).follow() # pop (north, west)
    assert formdef.data_class().get(formdata.id).status == 'wf-%s' % st3.id

    # and do a last jump using the API
    formdata = formdef.data_class().get(formdata.id)
    formdata.jump_status('_previous') # pop (north)
    assert formdata.status == 'wf-%s' % st2.id

    formdata = formdef.data_class().get(formdata.id)
    formdata.jump_status('_previous') # pop ()
    assert formdata.status == 'wf-%s' % st1.id

def test_workflow_jump_previous_on_submit(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='blah')
    st1 = wf.add_status('North')
    st1.id = 'north'
    st2 = wf.add_status('South')
    st2.id = 'south'

    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = ['_submitter', '_receiver']
    commentable.button_label = 'CLICK ME!'

    st1.items.append(commentable)
    commentable.parent = st1

    jump = JumpOnSubmitWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1
    jump.set_marker_on_status = True

    back = ChoiceWorkflowStatusItem()
    back.id = '_back'
    back.label = 'Back'
    back.by = ['_receiver']
    back.status = '_previous'
    st2.items.append(back)
    back.parent = st2

    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.data_class().wipe()
    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/%s/' % formdata.id)
    resp.form['comment'] = 'HELLO WORLD'
    resp = resp.form.submit('button_commentable')
    resp = resp.follow()

    assert formdef.data_class().get(formdata.id).status == 'wf-south'
    assert formdef.data_class().get(formdata.id).workflow_data['_markers_stack']
    resp = resp.form.submit('button_back')
    resp = resp.follow()

    assert formdef.data_class().get(formdata.id).status == 'wf-north'
    assert not formdef.data_class().get(formdata.id).workflow_data['_markers_stack']

def test_workflow_jump_previous_auto(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='blah')
    st1 = wf.add_status('North')
    st1.id = 'north'
    st2 = wf.add_status('South')
    st2.id = 'south'

    jump = JumpWorkflowStatusItem()
    jump.id = '_auto-jump'
    jump.set_marker_on_status = True
    jump.status = st2.id
    st1.items.append(jump)
    jump.parent = st1

    back = ChoiceWorkflowStatusItem()
    back.id = '_back'
    back.label = 'Back'
    back.by = ['_receiver']
    back.status = '_previous'
    st2.items.append(back)
    back.parent = st2

    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.data_class().wipe()
    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()
    formdata.perform_workflow()
    assert formdata.status == 'wf-south'
    assert formdata.workflow_data['_markers_stack'] == [{'status_id': 'north'}]

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/%s/' % formdata.id)
    resp = resp.form.submit('button_back')
    resp = resp.follow()

    # jumped and got back to north, then re-jump to south again (auto-jump)
    formdata = formdef.data_class().get(formdata.id)
    statuses = [evo.status for evo in formdata.evolution]
    assert statuses == ['wf-north', 'wf-south', 'wf-north', 'wf-south']
    # formdata went through north->south auto-jump again, status and marker are still here
    assert formdata.status == 'wf-south'
    assert formdata.workflow_data['_markers_stack'] == [{'status_id': 'north'}]

def test_backoffice_fields(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='bo fields')
    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
    wf.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='1st backoffice field',
            type='string', varname='backoffice_blah', required=False),
    ]
    st1 = wf.add_status('Status1')
    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow_id = wf.id
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    assert not 'Backoffice Data' in resp.text
    assert not '1st backoffice field' in resp.text

    formdata.data = {'bo1': 'HELLO WORLD'}
    formdata.store()
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'Backoffice Data' in resp.text
    assert '1st backoffice field' in resp.text
    assert 'HELLO WORLD' in resp.text

    wf.backoffice_fields_formdef.fields = [
        fields.StringField(id='bo1', label='1st backoffice field',
            type='string', varname='backoffice_blah', required=True),
    ]
    wf.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'Backoffice Data' in resp.text
    assert 'Not set' in resp.text

def test_backoffice_logged_errors(pub):
    Workflow.wipe()
    workflow = Workflow.get_default_workflow()
    workflow.id = '12'
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.status = 'rejected'
    jump.condition = {'type': 'python', 'value': '1//0'}  # ZeroDivisionError
    st1 = workflow.possible_status[0]
    st1.items.insert(0, jump)
    jump.parent = st1
    workflow.store()

    FormDef.wipe()
    formdef = FormDef()
    formdef.id = '34'
    formdef.workflow = workflow
    formdef.name = 'test'
    formdef.confirmation = False
    formdef.fields = []
    formdef.store()

    LoggedError.wipe()

    create_superuser(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert not 'ZeroDivisionError' in resp.text
    resp = app.get('/backoffice/workflows/%s/' % workflow.id)
    assert not 'ZeroDivisionError' in resp.text

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    app = login(get_app(pub))
    resp = app.get('/backoffice/forms/%s/' % formdef.id)
    assert 'Failed to evaluate condition' in resp.text
    assert 'error ZeroDivisionError' in resp.text
    resp = resp.click('1 error')

    resp = app.get('/backoffice/workflows/%s/' % workflow.id)
    resp2 = resp.click('1 error')
    assert 'Failed to evaluate condition' in resp2.text
    assert 'error ZeroDivisionError' in resp2.text
    resp = resp2.click('Failed to evaluate condition')
    assert 'ZeroDivisionError: integer division or modulo by zero' in resp.text
    assert 'Python Expression: <code>1//0</code>' in resp.text
    assert not 'Acked' in resp.text
    resp = resp.click('Ack').follow()
    assert 'Acked' in resp.text
    assert LoggedError.select()[0].acked is True
    resp = resp.click('Delete').follow()
    assert LoggedError.count() == 0

    pub.cfg.update({'debug': {'error_email': None}})
    pub.write_cfg()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 1

    app = login(get_app(pub))
    resp = app.get('/backoffice/workflows/%s/' % workflow.id)
    assert 'Failed to evaluate condition' in resp2.text
    assert 'error ZeroDivisionError' in resp2.text
    resp2 = resp.click('1 error')
    resp = resp2.click('Failed to evaluate condition')
    assert 'href="http://example.net/backoffice/management/test/' in resp.text

    # very long error string (check it creates a viable tech_id)
    jump.condition = {'type': 'python', 'value': 'x'*500}  # NameError + very long string
    workflow.store()

    app = get_app(pub)
    resp = app.get('/test/')
    resp = resp.form.submit('submit').follow()
    resp = resp.form.submit('submit')
    assert LoggedError.count() == 2

    # remove formdef
    FormDef.wipe()
    resp = resp2.click('Failed to evaluate condition')
    assert not 'href="http://example.net/backoffice/management/test/' in resp.text

def test_backoffice_formdata_named_wscall(http_requests, pub):
    user = create_user(pub)
    create_environment(pub)

    NamedWsCall.wipe()

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': 'http://remote.example.net/json'}
    wscall.store()
    assert wscall.slug == 'hello_world'

    formdef = FormDef()
    formdef.name = 'test'
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.fields = [
        fields.CommentField(id='7', label='X[webservice.hello_world.foo]Y', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub))

    resp = app.get('/backoffice/submission/test/')
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'XbarY'

    # check with publisher variable in named webservice call
    if not pub.site_options.has_section('variables'):
        pub.site_options.add_section('variables')
    pub.site_options.set('variables', 'example_url', 'http://remote.example.net/')
    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
        pub.site_options.write(fd)

    wscall = NamedWsCall()
    wscall.name = 'Hello world'
    wscall.request = {'url': '[example_url]json'}
    wscall.store()

    resp = app.get('/backoffice/submission/test/')
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'XbarY'

    # django-templated URL
    wscall.request = {'url': '{{ example_url }}json'}
    wscall.store()
    resp = app.get('/backoffice/submission/test/')
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'XbarY'

    # webservice call in django template
    formdef.fields = [
        fields.CommentField(id='7', label='dja-{{ webservice.hello_world.foo}}-ngo', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = app.get('/backoffice/submission/test/')
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'dja-bar-ngo'

def test_backoffice_session_var(pub):
    open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w').write('''[options]
query_string_allowed_vars = foo,bar
''')

    user = create_user(pub)
    create_environment(pub)

    formdef = FormDef()
    formdef.name = 'test'
    formdef.backoffice_submission_roles = user.roles[:]
    formdef.fields = [
        fields.CommentField(id='7', label='X[session_var_foo]Y', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()

    app = login(get_app(pub))

    resp = app.get('/backoffice/submission/test/?session_var_foo=bar')
    assert resp.location.endswith('/backoffice/submission/test/')
    resp = resp.follow()
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'XbarY'

    # django template
    formdef.fields = [
        fields.CommentField(id='7', label='d{{ session_var_foo }}o', type='comment'),
    ]
    formdef.store()
    formdef.data_class().wipe()
    resp = app.get('/backoffice/submission/test/?session_var_foo=jang')
    assert resp.location.endswith('/backoffice/submission/test/')
    resp = resp.follow()
    assert resp.html.find('div', {'data-field-id': '7'}).text.strip() == 'django'

def test_backoffice_display_message(pub):
    user = create_user(pub)
    create_environment(pub)

    workflow = Workflow(name='test')
    st1 = workflow.add_status('Status1', 'st1')

    display1 = DisplayMessageWorkflowStatusItem()
    display1.message = 'message-to-all'
    display1.to = []
    st1.items.append(display1)
    display1.parent = st1

    display2 = DisplayMessageWorkflowStatusItem()
    display2.message = 'message-to-submitter'
    display2.to = ['_submitter']
    st1.items.append(display2)
    display2.parent = st1

    display3 = DisplayMessageWorkflowStatusItem()
    display3.message = 'message-to-receiver'
    display3.to = [user.roles[0]]
    st1.items.append(display3)
    display3.parent = st1

    workflow.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow = workflow
    formdef.store()

    formdata = formdef.data_class().select()[0]
    formdata.status = 'wf-st1'
    formdata.store()

    app = login(get_app(pub))
    resp = app.get(formdata.get_url(backoffice=True))

    assert 'message-to-all' in resp.text
    assert 'message-to-submitter' not in resp.text
    assert 'message-to-receiver' in resp.text

    # display first message at the bottom of the page
    display1.position = 'bottom'
    workflow.store()
    resp = app.get(formdata.get_url(backoffice=True))
    assert resp.text.index('message-to-all') > resp.text.index('message-to-receiver')

    # display first message on top of actions
    display1.position = 'actions'
    workflow.store()

    resp = app.get(formdata.get_url(backoffice=True))
    assert not 'message-to-all' in resp.text  # no actions no message

    again = ChoiceWorkflowStatusItem()
    again.id = '_again'
    again.label = 'Again'
    again.by = ['_receiver']
    again.status = st1.id
    st1.items.append(again)
    again.parent = st1
    workflow.store()

    resp = app.get(formdata.get_url(backoffice=True))
    assert 'message-to-all' in resp.text
    assert resp.text.index('message-to-all') > resp.text.index('message-to-receiver')

def test_backoffice_forms_condition_on_button(pub):
    create_superuser(pub)
    create_environment(pub, set_receiver=True)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'
    workflow.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.workflow = workflow
    formdef.store()

    # move some forms from new to accepted
    for i, formdata in enumerate(formdef.data_class().select(lambda x: x.status == 'wf-new')):
        if i % 2:
            formdata.status = 'wf-accepted'
            formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    resp = resp.click('Management', index=0)
    resp = resp.follow()
    assert '17 open on 50' in resp.text

    formdata = [x for x in formdef.data_class().select() if x.status == 'wf-new'][0]
    resp = app.get(formdata.get_url(backoffice=True))
    assert 'button_commentable' in resp.text
    assert 'button_accept' in resp.text
    assert 'button_reject' in resp.text

    # commentable
    workflow.possible_status[1].items[0].condition = {'type': 'python', 'value': 'False'}
    # reject
    workflow.possible_status[1].items[2].condition = {'type': 'python', 'value': 'False'}
    workflow.store()

    resp = app.get(formdata.get_url(backoffice=True))
    assert 'button_commentable' not in resp.text
    assert 'button_accept' in resp.text
    assert 'button_reject' not in resp.text

    formdef = FormDef.get_by_urlname('form-title')
    assert formdef.data_class().get(formdata.id).actions_roles == set(['1'])

    # accept
    workflow.possible_status[1].items[1].condition = {'type': 'python', 'value': 'False'}
    workflow.store()

    resp = app.get(formdata.get_url(backoffice=True))
    assert 'button_commentable' not in resp.text
    assert 'button_accept' not in resp.text
    assert 'button_reject' not in resp.text

    formdef = FormDef.get_by_urlname('form-title')
    assert formdef.data_class().get(formdata.id).actions_roles == set()

    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    resp = resp.click('Management', index=0)
    resp = resp.follow()
    assert '8 open on 50' in resp.text  # only the accepted ones

def test_workflow_inspect_page(pub):
    create_superuser(pub)
    create_environment(pub, set_receiver=True)

    workflow = Workflow.get_default_workflow()
    workflow.id = '2'

    st1 = workflow.add_status('Status1')
    jump = JumpWorkflowStatusItem()
    jump.id = '_jump'
    jump.timeout = '=86400'
    jump.status = 'finished'
    st1.items.append(jump)
    jump.parent = st1
    workflow.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/workflows/%s/inspect' % workflow.id)
    assert '=86400' in resp.text

    jump.timeout = '82800'
    workflow.store()
    resp = app.get('/backoffice/workflows/%s/inspect' % workflow.id)
    assert '23 hours' in resp.text

def test_workflow_comment_required(pub):
    user = create_user(pub)
    create_environment(pub)

    wf = Workflow(name='blah')
    st1 = wf.add_status('Comment')
    st1.id = 'comment'
    commentable = CommentableWorkflowStatusItem()
    commentable.id = '_commentable'
    commentable.by = ['_submitter', '_receiver']
    st1.items.append(commentable)
    commentable.parent = st1
    wf.store()

    formdef = FormDef.get_by_urlname('form-title')
    formdef.data_class().wipe()
    formdef.workflow = wf
    formdef.store()

    formdata = formdef.data_class()()
    formdata.data = {}
    formdata.just_created()
    formdata.store()

    app = login(get_app(pub))
    resp = app.get('/backoffice/management/form-title/%s/' % formdata.id)
    assert 'widget-required' not in resp.text
    resp.form['comment'] = 'HELLO WORLD 1'
    resp = resp.form.submit('button_commentable')
    resp = resp.follow()
    assert 'HELLO WORLD 1' in resp.text
    assert 'widget-required' not in resp.text
    resp.form['comment'] = None
    resp = resp.form.submit('button_commentable')
    resp = resp.follow()

    commentable.required = True
    wf.store()
    resp = app.get('/backoffice/management/form-title/%s/' % formdata.id)
    assert 'widget-required' in resp.text
    resp.form['comment'] = '  '  # spaces == empty
    resp = resp.form.submit('button_commentable')
    assert 'widget-with-error' in resp.text
    resp.form['comment'] = 'HELLO WORLD 2'
    resp = resp.form.submit('button_commentable')
    resp = resp.follow()
    assert 'widget-with-error' not in resp.text
    assert 'HELLO WORLD 2' in resp.text

def test_carddata_management(pub, studio):
    user = create_user(pub)
    app = login(get_app(pub))
    resp = app.get('/backoffice/')
    assert not 'Cards' in resp.text
    carddef = CardDef()
    carddef.name = 'foo'
    carddef.fields = [
        fields.StringField(id='1', label='Test', type='string', varname='foo'),
    ]
    carddef.store()

    resp = app.get('/backoffice/')
    assert not 'Cards' in resp.text

    carddef.backoffice_submission_roles = user.roles
    carddef.store()
    resp = app.get('/backoffice/')
    assert 'Cards' in resp.text

    carddef.backoffice_submission_roles = None
    carddef.workflow_roles = {'_editor': user.roles[0]}
    carddef.store()
    resp = app.get('/backoffice/')
    assert 'Cards' in resp.text

    resp = app.get('/backoffice/data/')
    resp = resp.click('foo')
    assert 'Add' not in resp.text

    carddef.backoffice_submission_roles = user.roles
    carddef.store()

    resp = app.get('/backoffice/data/')
    resp = resp.click('foo')
    assert resp.text.count('<tr') == 1  # header
    assert 'Add' in resp.text
    resp = resp.click('Add')
    resp.form['f1'] = 'blah'
    resp = resp.form.submit('submit')
    assert resp.location.endswith('/backoffice/data/foo/1/')
    resp = resp.follow()
    assert 'Edit Card' in resp.text
    assert 'Delete Card' in resp.text

    resp = app.get('/backoffice/data/')
    resp = resp.click('foo')
    assert resp.text.count('<tr') == 2  # header + row of data