tests/api/test_formdata.py

@@ -177,6 +177,7 @@ def test_formdata(pub, local_user, user, auth):
         fields.BlockField(id='5', label='test', varname='blockdata', block_slug='foobar', max_items=3),
         fields.TextField(id='6', label='rich text', varname='richtext', display_mode='rich'),
         fields.FileField(id='7', label='image file', varname='image_file'),
+        fields.NumericField(id='8', label='numeric value', varname='numeric'),
     ]
     Workflow.wipe()
     workflow = Workflow(name='foo')
@@ -210,6 +211,7 @@ def test_formdata(pub, local_user, user, auth):
         '5_display': 'hello',
         '6': '<script></script><p>foo</p>',
         '7': image_upload,
+        '8': 5.5,
     }
     formdata.data['4_display'] = item_field.store_display_value(formdata.data, item_field.id)
     formdata.data['4_structured'] = item_field.store_structured_value(formdata.data, item_field.id)
@@ -233,7 +235,7 @@ def test_formdata(pub, local_user, user, auth):
 
     assert datetime.datetime.strptime(resp.json['last_update_time'], '%Y-%m-%dT%H:%M:%S')
     assert datetime.datetime.strptime(resp.json['receipt_time'], '%Y-%m-%dT%H:%M:%S')
-    assert len(resp.json['fields']) == 10
+    assert len(resp.json['fields']) == 11
     assert 'foobar' in resp.json['fields']
     assert 'foobar2' not in resp.json['fields']  # foobar2 has no varname, not in json
     assert resp.json['user']['name'] == local_user.name
@@ -262,6 +264,8 @@ def test_formdata(pub, local_user, user, auth):
     assert resp.json['fields']['image_file']['url'].startswith('http://example.net/test/1/download?hash=')
     assert 'thumbnail=1' in resp.json['fields']['image_file']['thumbnail_url']
 
+    assert resp.json['fields']['numeric'] == '5.5'
+
     assert resp.json['workflow']['status']['name'] == 'New'
     assert resp.json['workflow']['status']['first_arrival_datetime']
     assert resp.json['workflow']['status']['latest_arrival_datetime']
@@ -1380,6 +1384,78 @@ def test_api_list_formdata_string_filter(pub, local_user):
         assert len(resp.json) == result
 
 
+def test_api_list_formdata_numeric_filter(pub, local_user):
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    local_user.roles = [role.id]
+    local_user.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.fields = [
+        fields.NumericField(id='2', label='Numeric', varname='numeric'),
+    ]
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(5):
+        formdata = data_class()
+        formdata.data = {
+            '2': '%.2f' % (3.2 + 0.8 * i),
+        }
+        if i == 3:
+            # Empty values
+            formdata.data = {
+                '2': None,
+            }
+        if i == 4:
+            # None values
+            formdata.data = {}
+        formdata.user_id = local_user.id
+        formdata.just_created()
+        formdata.jump_status('new')
+        formdata.store()
+
+    params = [
+        ('eq', '4', 1),
+        ('ne', '4', 4),
+        ('lt', '4', 1),
+        ('lte', '4', 2),
+        ('lt', '4.1', 2),
+        ('lte', '4.1', 2),
+        ('gt', '4', 1),
+        ('gt', '3.9', 2),
+        ('gte', '4', 2),
+        ('in', '4', 1),
+        ('in', '3.2|4', 2),
+        ('in', '4|42', 1),
+        ('in', '4|a', 1),
+        ('in', '4.00|a', 1),
+        ('not_in', '4', 2),
+        ('not_in', '3.2|4', 1),
+        ('not_in', '3.2|42', 2),
+        ('absent', 'on', 2),
+        ('existing', 'on', 3),
+        ('between', '3.1|4.5', 2),
+        ('between', '3.3|4.5', 1),
+        ('between', '4.5|3.1', 2),
+    ]
+    for operator, value, result in params:
+        resp = get_app(pub).get(
+            sign_uri(
+                '/api/forms/test/list?filter-numeric=%s&filter-numeric-operator=%s' % (value, operator),
+                user=local_user,
+            )
+        )
+        assert len(resp.json) == result
+
+
 def test_api_list_formdata_text_filter(pub, local_user):
     pub.role_class.wipe()
     role = pub.role_class(name='test')

tests/api/test_formdef.py

@@ -1,5 +1,6 @@
 import base64
 import datetime
+import decimal
 import json
 import os
 import time
@@ -1227,3 +1228,67 @@ def test_formdef_import_export_unnamed_block(pub, admin_user):
     app.post_json('/api/formdefs/test/submit', formdata_export)
     new_formdata = formdef.data_class().select()[0]
     assert new_formdata.data == formdata.data
+
+
+def test_formdef_submit_numeric_field(pub, local_user):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'test'
+    formdef.fields = [
+        fields.NumericField(id='11', label='numeric', varname='numeric'),
+    ]
+    formdef.store()
+    data_class = formdef.data_class()
+
+    def post(payload):
+        signed_url = sign_url(
+            'http://example.net/api/formdefs/test/submit'
+            + '?format=json&orig=coucou&email=%s' % urllib.parse.quote(local_user.email),
+            '1234',
+        )
+        url = signed_url[len('http://example.net') :]
+        return get_app(pub).post_json(url, payload)
+
+    # valid value as float
+    payload = {
+        'data': {
+            'numeric': 10.5,
+        }
+    }
+    resp = post(payload)
+    assert resp.json['err'] == 0
+    formdata = data_class.get(resp.json['data']['id'])
+    assert formdata.data['11'] == decimal.Decimal('10.5')
+
+    # valid value as string
+    payload = {
+        'data': {
+            'numeric': '10.5',
+        }
+    }
+    resp = post(payload)
+    assert resp.json['err'] == 0
+    formdata = data_class.get(resp.json['data']['id'])
+    assert formdata.data['11'] == decimal.Decimal('10.5')
+
+    # null value
+    payload = {
+        'data': {
+            'numeric': None,
+        }
+    }
+    resp = post(payload)
+    assert resp.json['err'] == 0
+    formdata = data_class.get(resp.json['data']['id'])
+    assert formdata.data['11'] is None
+
+    # invalid value
+    payload = {
+        'data': {
+            'numeric': 'xxx',
+        }
+    }
+    resp = post(payload)
+    assert resp.json['err'] == 0
+    formdata = data_class.get(resp.json['data']['id'])
+    assert formdata.data['11'] is None

tests/backoffice_pages/test_export.py

@@ -372,6 +372,7 @@ def test_backoffice_csv_export_fields(pub):
         fields.DateField(id='456', required=True, label='Test4', varname='date'),
         fields.FileField(id='567', required=True, label='Test5', varname='file'),
         fields.BoolField(id='678', required=True, label='Test6', varname='bool'),
+        fields.NumericField(id='890', label='Test7', varname='numeric'),
     ]
     formdef.workflow_roles = {'_receiver': 1}
     formdef.store()
@@ -389,6 +390,7 @@ def test_backoffice_csv_export_fields(pub):
         '456': time.strptime('2020-04-24', '%Y-%m-%d'),
         '567': upload,
         '678': True,
+        '890': 5.5,
     }
     formdata.just_created()
     formdata.jump_status('new')
@@ -401,6 +403,7 @@ def test_backoffice_csv_export_fields(pub):
         '456': time.strptime('2020-04-25', '%Y-%m-%d'),
         '567': upload,
         '678': False,
+        '890': 2.5,
     }
     formdata.just_created()
     formdata.jump_status('new')
@@ -415,18 +418,20 @@ def test_backoffice_csv_export_fields(pub):
     resp.forms['listing-settings']['456'].checked = True
     resp.forms['listing-settings']['567'].checked = True
     resp.forms['listing-settings']['678'].checked = True
+    resp.forms['listing-settings']['890'].checked = True
     resp = resp.forms['listing-settings'].submit()
     resp_csv = resp.click('Export a Spreadsheet')
     resp_csv.form['format'] = 'csv'
     resp_csv = resp_csv.form.submit('submit')
-    assert resp_csv.text.splitlines()[0].split(',')[-4:] == [
+    assert resp_csv.text.splitlines()[0].split(',')[-5:] == [
         '"Test3"',
         '"Test4"',
         '"Test5"',
         '"Test6"',
+        '"Test7"',
     ]
-    line1 = resp_csv.text.splitlines()[1].split(',')[-4:]
-    line2 = resp_csv.text.splitlines()[2].split(',')[-4:]
+    line1 = resp_csv.text.splitlines()[1].split(',')[-5:]
+    line2 = resp_csv.text.splitlines()[2].split(',')[-5:]
     if line1[0] == '"blah2@example.invalid"':
         line1, line2 = line2, line1
     assert line1 == [
@@ -434,12 +439,14 @@ def test_backoffice_csv_export_fields(pub):
         '"2020-04-24"',
         '"test.jpeg"',
         '"Yes"',
+        '"5.5"',
     ]
     assert line2 == [
         '"blah2@example.invalid"',
         '"2020-04-25"',
         '"test.jpeg"',
         '"No"',
+        '"2.5"',
     ]
 
     # export as ods
@@ -814,6 +821,11 @@ def test_backoffice_ods(pub):
             label='number field with zero',
             display_locations=['validation', 'summary', 'listings'],
         ),
+        fields.NumericField(
+            id='13',
+            label='real numeric field',
+            display_locations=['validation', 'summary', 'listings'],
+        ),
     ]
     formdef.workflow_roles = {'_receiver': 1}
     formdef.store()
@@ -838,6 +850,7 @@ def test_backoffice_ods(pub):
         '10': ' 123,45',
         '11': '1_000_000',
         '12': '0',
+        '13': 234.56,
     }
     formdata.data['4'].receive([b'hello world'])
     formdata.just_created()
@@ -875,6 +888,7 @@ def test_backoffice_ods(pub):
     comma_number_column = all_texts.index('number with comma field')
     not_number_column = all_texts.index('not a number, with underscore')
     zero_number_column = all_texts.index('number field with zero')
+    numeric_column = all_texts.index('real numeric field')
 
     for row in ods_sheet.findall('.//{%s}table-row' % ods.NS['table']):
         if (
@@ -961,6 +975,18 @@ def test_backoffice_ods(pub):
         ]
         == '0'
     )
+    assert (
+        row.findall('.//{%s}table-cell' % ods.NS['table'])[numeric_column].attrib[
+            '{%s}value-type' % ods.NS['office']
+        ]
+        == 'float'
+    )
+    assert (
+        row.findall('.//{%s}table-cell' % ods.NS['table'])[numeric_column].attrib[
+            '{%s}value' % ods.NS['office']
+        ]
+        == '234.56'
+    )
 
 
 def test_backoffice_empty_ods(pub):

tests/backoffice_pages/test_filters.py

@@ -1741,3 +1741,67 @@ def test_backoffice_block_field_filter(pub):
     resp.forms['listing-settings']['filter-0-2-value'].value = '2'
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('<tr') == 1 + 1
+
+
+def test_backoffice_numeric_filter(pub):
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
+        fields.NumericField(
+            id='4', label='4th field', display_locations=['validation', 'summary', 'listings']
+        )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
+        formdata.data['4'] = '123.4' if bool(i % 2) else '315'
+        formdata.just_created()
+        formdata.jump_status('new')
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-4'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+
+    assert resp.forms['listing-settings']['filter-4-value'].value == ''
+
+    resp.forms['listing-settings']['filter-4-value'].value = '123.4'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>123.4</td>') > 0
+    assert resp.text.count('<td>315</td>') == 0
+
+    resp.forms['listing-settings']['filter-4-value'].value = '123,4'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>123.4</td>') > 0
+    assert resp.text.count('<td>315</td>') == 0
+
+    resp.forms['listing-settings']['filter-4-value'].value = '315'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>123.4</td>') == 0
+    assert resp.text.count('<td>315</td>') > 0
+
+    resp.forms['listing-settings']['filter-4-value'].value = '123.4'
+    resp.forms['listing-settings']['filter-4-operator'].value = 'gte'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>123.4</td>') > 0
+    assert resp.text.count('<td>315</td>') > 0
+
+    resp.forms['listing-settings']['filter-4-value'].value = '123.4'
+    resp.forms['listing-settings']['filter-4-operator'].value = 'gt'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<td>123.4</td>') == 0
+    assert resp.text.count('<td>315</td>') > 0

tests/test_fields.py

@@ -1014,6 +1014,32 @@ def test_new_field_type_options(pub):
         ('computed', 'Computed Data', 'computed'),
     ]
 
+    pub.site_options.set('options', 'numeric-field-type', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+    assert fields.get_field_options(blacklisted_types=[]) == [
+        ('string', 'Text (line)', 'string'),
+        ('text', 'Long Text', 'text'),
+        ('email', 'Email', 'email'),
+        ('bool', 'Check Box (single choice)', 'bool'),
+        ('numeric', 'Numeric', 'numeric'),
+        ('file', 'File Upload', 'file'),
+        ('date', 'Date', 'date'),
+        ('item', 'List', 'item'),
+        ('items', 'Multiple choice list', 'items'),
+        ('table-select', 'Table of Lists', 'table-select'),
+        ('tablerows', 'Table with rows', 'tablerows'),
+        ('map', 'Map', 'map'),
+        ('ranked-items', 'Ranked Items', 'ranked-items'),
+        ('', '—', ''),
+        ('title', 'Title', 'title'),
+        ('subtitle', 'Subtitle', 'subtitle'),
+        ('comment', 'Comment', 'comment'),
+        ('page', 'Page', 'page'),
+        ('', '—', ''),
+        ('computed', 'Computed Data', 'computed'),
+    ]
+
 
 def test_block_do_not_pickle_cache(pub):
     FormDef.wipe()

tests/test_sql.py

@@ -1,4 +1,5 @@
 import datetime
+import decimal
 import io
 import os
 import pickle
@@ -71,6 +72,7 @@ def formdef(pub):
         fields.DateField(id='5', label='date'),
         fields.ItemsField(id='6', label='items', items=('apple', 'pear', 'peach', 'apricot')),
         fields.BlockField(id='7', label='block', block_slug='fooblock'),
+        fields.NumericField(id='8', lable='numeric'),
     ]
     formdef.store()
     return formdef
@@ -192,6 +194,17 @@ def test_sql_field_bool(formdef):
     check_sql_field(formdef, '3', True)
 
 
+def test_sql_field_numeric(formdef):
+    check_sql_field(formdef, '8', 6)
+    check_sql_field(formdef, '8', decimal.Decimal('4.5'))
+    check_sql_field(formdef, '8', -2)
+    assert [x.data['8'] for x in formdef.data_class().select(order_by='f8')] == [
+        decimal.Decimal('-2'),
+        decimal.Decimal('4.5'),
+        decimal.Decimal('6'),
+    ]
+
+
 def test_sql_field_item(formdef):
     check_sql_field(formdef, '4', 'apricot', display=True)
 

tests/test_widgets.py

@@ -1,5 +1,6 @@
 import copy
 import datetime
+import decimal
 import os
 import shutil
 
@@ -21,6 +22,7 @@ from wcs.qommon.form import (
     Form,
     MapWidget,
     MiniRichTextWidget,
+    NumericWidget,
     OptGroup,
     PasswordEntryWidget,
     RichTextWidget,
@@ -1567,3 +1569,35 @@ def test_error_templates():
     assert 'data-use-live-server-validation=' in widget_html
     assert PyQuery(widget_html)('#error_test_valueMissing').text() == 'required field'
     assert PyQuery(widget_html)('#error_test_tooLong').text() == 'too many characters (limit is 10)'
+
+
+def test_numeric_widget():
+    widget = NumericWidget('test')
+    mock_form_submission(req, widget, {'test': ' 5 '})
+    assert not widget.has_error()
+    assert widget.parse() == decimal.Decimal(5)
+
+    widget = NumericWidget('test')
+    mock_form_submission(req, widget, {'test': ' 2.5 '})
+    assert not widget.has_error()
+    assert widget.parse() == decimal.Decimal('2.5')
+
+    widget = NumericWidget('test')
+    mock_form_submission(req, widget, {'test': 'xxx'})
+    assert widget.has_error()
+    assert widget.get_error() == 'invalid value'
+
+    widget = NumericWidget('test', min_value=10)
+    mock_form_submission(req, widget, {'test': '5'})
+    assert widget.has_error()
+    assert widget.get_error() == 'You should enter a number greater than or equal to 10.'
+
+    widget = NumericWidget('test', max_value=10)
+    mock_form_submission(req, widget, {'test': '15'})
+    assert widget.has_error()
+    assert widget.get_error() == 'You should enter a number less than or equal to 10.'
+
+    widget = NumericWidget('test', restrict_to_integers=True)
+    mock_form_submission(req, widget, {'test': '5.5'})
+    assert widget.has_error()
+    assert widget.get_error() == 'You should enter a number without a decimal separator.'

tests/workflow/test_all.py

@@ -6,23 +6,15 @@ import os
 import shutil
 import time
 import urllib.parse
-import zipfile
 from unittest import mock
 
 import pytest
 import responses
-from django.utils.encoding import force_str
-from PIL import Image
-from pyzbar.pyzbar import ZBarSymbol
-from pyzbar.pyzbar import decode as zbar_decode_qrcode
 from quixote import cleanup, get_publisher, get_response
-from quixote.http_request import Upload as QuixoteUpload
 
 from wcs import sessions, sql
-from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.fields import (
-    BlockField,
     BoolField,
     CommentField,
     DateField,
@@ -31,17 +23,12 @@ from wcs.fields import (
     ItemField,
     ItemsField,
     MapField,
-    PageField,
     StringField,
-    SubtitleField,
-    TableField,
-    TextField,
-    TitleField,
 )
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon.errors import ConnectionError
-from wcs.qommon.form import Form, UploadedFile
+from wcs.qommon.form import Form
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.qommon.upload_storage import PicklableUpload
@@ -52,7 +39,6 @@ from wcs.wf.create_formdata import Mapping
 from wcs.wf.criticality import MODE_DEC, MODE_INC, MODE_SET, ModifyCriticalityWorkflowStatusItem
 from wcs.wf.dispatch import DispatchWorkflowStatusItem
 from wcs.wf.display_message import DisplayMessageWorkflowStatusItem
-from wcs.wf.export_to_model import ExportToModel, UploadValidationError, transform_to_pdf
 from wcs.wf.external_workflow import ManyExternalCallsPart
 from wcs.wf.form import WorkflowFormFieldsFormDef
 from wcs.wf.geolocate import GeolocateWorkflowStatusItem
@@ -3809,403 +3795,6 @@ def test_geolocate_overwrite(pub):
     assert int(formdata.geolocations['base']['lon']) == 3
 
 
-@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
-def test_transform_to_pdf():
-    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as instream:
-        outstream = transform_to_pdf(instream)
-        assert outstream is not False
-        assert outstream.read(10).startswith(b'%PDF-')
-
-
-@pytest.mark.parametrize(
-    'template_name', ['template-with-image.odt', 'template-with-image-django-syntax.odt']
-)
-def test_export_to_model_image(pub, template_name):
-    formdef = FormDef()
-    formdef.name = 'baz'
-    formdef.fields = [
-        FileField(id='3', label='File', varname='image'),
-    ]
-    formdef.store()
-
-    upload = PicklableUpload('test.jpeg', 'image/jpeg')
-    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
-        image_data = fd.read()
-    upload.receive([image_data])
-
-    formdata = formdef.data_class()()
-    formdata.data = {'3': upload}
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-
-    item = ExportToModel()
-    item.convert_to_pdf = False
-    item.method = 'non-interactive'
-    template_filename = os.path.join(os.path.dirname(__file__), '..', template_name)
-    with open(template_filename, 'rb') as fd:
-        template = fd.read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
-    upload.fp = io.BytesIO()
-    upload.fp.write(template)
-    upload.fp.seek(0)
-    item.model_file = UploadedFile(pub.app_dir, None, upload)
-    item.attach_to_history = True
-
-    item.perform(formdata)
-
-    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
-    with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
-        zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
-    # check the image has been replaced by the one from the formdata
-    assert zinfo.file_size == len(image_data)
-
-    # check with missing data or wrong kind of data
-    for field_value in (None, 'wrong kind'):
-        formdata = formdef.data_class()()
-        formdata.data = {'3': field_value}
-        formdata.just_created()
-        formdata.store()
-        pub.substitutions.feed(formdata)
-
-        item.perform(formdata)
-
-        with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
-            zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
-        # check the original image has been left
-        assert zinfo.file_size == 580
-
-    item.filename = 'formulaire-{{form_number}}/2.odt'
-    item.perform(formdata)
-    assert formdata.evolution[-1].parts[-1].base_filename == 'formulaire-%s-%s-2.odt' % (
-        formdef.id,
-        formdata.id,
-    )
-
-
-def test_export_to_model_qrcode(pub):
-    FormDef.wipe()
-
-    formdef = FormDef()
-    formdef.name = 'baz'
-    formdef.fields = []
-    formdef.store()
-
-    formdef.data_class().wipe()
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-
-    item = ExportToModel()
-    item.convert_to_pdf = False
-    item.method = 'non-interactive'
-    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-with-qrcode.odt')
-    with open(template_filename, 'rb') as fd:
-        template = fd.read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
-    upload.fp = io.BytesIO()
-    upload.fp.write(template)
-    upload.fp.seek(0)
-    item.model_file = UploadedFile(pub.app_dir, None, upload)
-    item.attach_to_history = True
-
-    item.perform(formdata)
-
-    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
-    with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
-        # base template use a jpg images and export_to_model does not rename it
-        # event when content is PNG, but it still works inside LibreOffice
-        # which ignores the filename extension.
-        image_filename = [name for name in zfile.namelist() if name.endswith('.jpg')][0]
-        with zfile.open(image_filename, 'r') as image_fd:
-            img = Image.open(image_fd)
-            assert (
-                zbar_decode_qrcode(img, symbols=[ZBarSymbol.QRCODE])[0].data.decode()
-                == 'http://example.net/backoffice/management/baz/1/'
-            )
-
-
-def test_export_to_model_backoffice_field(pub):
-    wf = Workflow(name='email with attachments')
-    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
-    wf.backoffice_fields_formdef.fields = [
-        FileField(id='bo1', label='bo field 1', varname='backoffice_file1'),
-    ]
-    st1 = wf.add_status('Status1')
-    wf.store()
-    formdef = FormDef()
-    formdef.name = 'foo-export-to-bofile'
-    formdef.fields = [
-        StringField(id='1', label='String', varname='string'),
-    ]
-    formdef.workflow_id = wf.id
-    formdef.store()
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-
-    item = ExportToModel()
-    item.method = 'non-interactive'
-    item.convert_to_pdf = False
-    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
-    with open(template_filename, 'rb') as fd:
-        template = fd.read()
-    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
-    upload.fp = io.BytesIO()
-    upload.fp.write(template)
-    upload.fp.seek(0)
-    item.model_file = UploadedFile(pub.app_dir, None, upload)
-    item.parent = st1
-    item.backoffice_filefield_id = 'bo1'
-    item.perform(formdata)
-
-    assert 'bo1' in formdata.data
-    fbo1 = formdata.data['bo1']
-    assert fbo1.base_filename == 'template.odt'
-    assert fbo1.content_type == 'application/octet-stream'
-    with zipfile.ZipFile(fbo1.get_file()) as zfile:
-        assert b'foo-export-to-bofile' in zfile.read('content.xml')
-
-    # no more 'bo1' backoffice field: do nothing
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-    # id is not bo1:
-    wf.backoffice_fields_formdef.fields = [
-        FileField(id='bo2', label='bo field 2'),
-    ]
-    item.perform(formdata)
-    assert formdata.data == {}
-    # field is not a field file:
-    wf.backoffice_fields_formdef.fields = [
-        StringField(id='bo1', label='bo field 1'),
-    ]
-    item.perform(formdata)
-    assert formdata.data == {}
-    # no field at all:
-    wf.backoffice_fields_formdef.fields = []
-    item.perform(formdata)
-    assert formdata.data == {}
-
-
-def test_export_to_model_django_template(pub):
-    formdef = FormDef()
-    formdef.name = 'foo-export-to-template-with-django'
-    formdef.fields = [
-        StringField(id='1', label='String', varname='string'),
-    ]
-    formdef.store()
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-
-    item = ExportToModel()
-    item.method = 'non-interactive'
-    item.attach_to_history = True
-    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-django.odt')
-    with open(template_filename, 'rb') as fd:
-        template = fd.read()
-    upload = QuixoteUpload('/foo/template-django.odt', content_type='application/octet-stream')
-    upload.fp = io.BytesIO()
-    upload.fp.write(template)
-    upload.fp.seek(0)
-    item.model_file = UploadedFile(pub.app_dir, None, upload)
-    item.convert_to_pdf = False
-    item.perform(formdata)
-
-    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
-        with zipfile.ZipFile(fd) as zout:
-            new_content = zout.read('content.xml')
-    assert b'>foo-export-to-template-with-django<' in new_content
-
-    formdef.name = 'Name with a \' simple quote'
-    formdef.store()
-    item.perform(formdata)
-
-    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
-        with zipfile.ZipFile(fd) as zout:
-            new_content = zout.read('content.xml')
-    assert b'>Name with a \' simple quote<' in new_content
-
-    formdef.name = 'A <> name'
-    formdef.store()
-    item.perform(formdata)
-
-    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
-        with zipfile.ZipFile(fd) as zout:
-            new_content = zout.read('content.xml')
-    assert b'>A &lt;&gt; name<' in new_content
-
-
-def test_export_to_model_xml(pub):
-    LoggedError = pub.loggederror_class
-    if LoggedError:
-        LoggedError.wipe()
-
-    formdef = FormDef()
-    formdef.name = 'foo-export-to-template-with-django'
-    formdef.fields = [
-        StringField(id='1', label='String', varname='string'),
-    ]
-    formdef.store()
-    formdata = formdef.data_class()()
-    formdata.data = {'1': 'écho'}
-    formdata.just_created()
-    formdata.store()
-
-    # good XML
-    item = ExportToModel()
-    item.method = 'non-interactive'
-    item.attach_to_history = True
-
-    def run(template, filename='/foo/template.xml', content_type='application/xml'):
-        upload = QuixoteUpload(filename, content_type=content_type)
-        upload.fp = io.BytesIO()
-        upload.fp.write(template.encode())
-        upload.fp.seek(0)
-        item.model_file = UploadedFile(pub.app_dir, None, upload)
-        item.convert_to_pdf = False
-        pub.substitutions.reset()
-        pub.substitutions.feed(formdata)
-        item.perform(formdata)
-        with open(formdata.evolution[0].parts[-1].get_file_path()) as fd:
-            return fd.read()
-
-    # good XML
-    assert run(template='<a>{{ form_var_string }}</a>') == '<a>écho</a>'
-    assert (
-        run(template='<a>{{ form_var_string }}</a>', content_type='application/octet-stream') == '<a>écho</a>'
-    )
-    assert run(template='<a>{{ form_var_string }}</a>', filename='/foo/template.svg') == '<a>écho</a>'
-
-    # unknown file format
-    with pytest.raises(UploadValidationError):
-        run(
-            template='<a>{{ form_var_string }}</a>',
-            filename='/foo/template.txt',
-            content_type='application/octet-stream',
-        )
-
-    # malformed XML
-    assert not LoggedError or LoggedError.count() == 0
-    assert run(template='<a>{{ form_var_string }}<a>') == '<a>écho<a>'
-    # on error in the XML correctness no exception is raised but an error is logged
-    assert not LoggedError or LoggedError.count() == 1
-
-
-@pytest.mark.parametrize('filename', ['template-form-details.odt', 'template-form-details-no-styles.odt'])
-def test_export_to_model_form_details_section(pub, filename):
-    BlockDef.wipe()
-    FormDef.wipe()
-
-    block = BlockDef()
-    block.name = 'foobar'
-    block.digest_template = 'X{{foobar_var_foo}}Y'
-    block.fields = [
-        StringField(id='123', required=True, label='Test', varname='foo'),
-        StringField(id='234', required=True, label='Test2', varname='bar'),
-    ]
-    block.store()
-
-    FormDef.wipe()
-    formdef = FormDef()
-    formdef.name = 'foo-export-details'
-    formdef.fields = [
-        PageField(id='1', label='Page 1'),
-        TitleField(id='2', label='Title'),
-        SubtitleField(id='3', label='Subtitle'),
-        StringField(id='4', label='String', varname='string'),
-        EmailField(id='5', label='Email'),
-        TextField(id='6', label='Text'),
-        BoolField(id='8', label='Bool'),
-        FileField(id='9', label='File'),
-        DateField(id='10', label='Date'),
-        ItemField(id='11', label='Item', items=['foo', 'bar']),
-        TableField(id='12', label='Table', columns=['a', 'b'], rows=['c', 'd']),
-        PageField(id='13', label='Empty Page'),
-        TitleField(id='14', label='Empty Title'),
-        StringField(id='15', label='Empty String', varname='invisiblestr'),
-        BlockField(id='16', label='Block Field', block_slug='foobar'),
-        ItemsField(id='17', label='Items', items=['foo', 'bar']),
-    ]
-    formdef.store()
-    formdef.data_class().wipe()
-    upload = PicklableUpload('test.jpeg', 'image/jpeg')
-    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
-        upload.receive([fd.read()])
-    formdata = formdef.data_class()()
-    formdata.data = {
-        '4': 'string',
-        '5': 'foo@localhost',
-        '6': 'para1\npara2',
-        '8': False,
-        '9': upload,
-        '10': time.strptime('2015-05-12', '%Y-%m-%d'),
-        '11': 'foo',
-        '12': [['1', '2'], ['3', '4']],
-        # value from test_block_digest in tests/form_pages/test_block.py
-        '16': {
-            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
-            'schema': {'123': 'string', '234': 'string'},
-        },
-        '16_display': 'XfooY, Xfoo2Y',
-    }
-    formdata.just_created()
-    formdata.store()
-    pub.substitutions.feed(formdata)
-
-    item = ExportToModel()
-    item.method = 'non-interactive'
-    item.attach_to_history = True
-    template_filename = os.path.join(os.path.dirname(__file__), '..', filename)
-    with open(template_filename, 'rb') as fd:
-        template = fd.read()
-    upload = QuixoteUpload(filename, content_type='application/octet-stream')
-    upload.fp = io.BytesIO()
-    upload.fp.write(template)
-    upload.fp.seek(0)
-    item.model_file = UploadedFile(pub.app_dir, None, upload)
-    item.convert_to_pdf = False
-    item.perform(formdata)
-
-    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
-        with zipfile.ZipFile(fd) as zout:
-            new_content = force_str(zout.read('content.xml'))
-    # section content has been removed
-    assert 'Titre de page' not in new_content
-    assert 'Titre' not in new_content
-    assert 'Libell&#233; de champ' not in new_content
-    assert 'Valeur de champ' not in new_content
-    # and new content has been inserted
-    assert '>Page 1<' in new_content
-    assert '>Title<' in new_content
-    assert '>Subtitle<' in new_content
-    assert '<text:span>string</text:span>' in new_content
-    assert '>para1<' in new_content
-    assert '>para2<' in new_content
-    assert '<text:span>No</text:span>' in new_content
-    assert 'xlink:href="http://example.net/foo-export-details/1/download?f=9"' in new_content
-    assert '>test.jpeg</text:a' in new_content
-    assert '>2015-05-12<' in new_content
-    assert 'Invisible' not in new_content
-    assert new_content.count('/table:table-cell') == 8
-    assert 'XfooY, Xfoo2Y' in new_content
-
-    if filename == 'template-form-details-no-styles.odt':
-        with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
-            with zipfile.ZipFile(fd) as zout:
-                new_styles = force_str(zout.read('styles.xml'))
-        assert 'Field_20_Label' in new_styles
-
-
 @pytest.mark.parametrize('formdef_class', [FormDef, CardDef])
 def test_global_timeouts(pub, formdef_class):
     CardDef.wipe()

tests/workflow/test_export_to_model.py

@@ -0,0 +1,462 @@
+import io
+import os
+import time
+import zipfile
+
+import pytest
+from PIL import Image
+from pyzbar.pyzbar import ZBarSymbol
+from pyzbar.pyzbar import decode as zbar_decode_qrcode
+from quixote import cleanup
+from quixote.http_request import Upload as QuixoteUpload
+
+from wcs import sessions
+from wcs.blocks import BlockDef
+from wcs.fields import (
+    BlockField,
+    BoolField,
+    DateField,
+    EmailField,
+    FileField,
+    ItemField,
+    ItemsField,
+    NumericField,
+    PageField,
+    StringField,
+    SubtitleField,
+    TableField,
+    TextField,
+    TitleField,
+)
+from wcs.formdef import FormDef
+from wcs.qommon import force_str
+from wcs.qommon.form import UploadedFile
+from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.upload_storage import PicklableUpload
+from wcs.wf.export_to_model import ExportToModel, UploadValidationError, transform_to_pdf
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
+
+from ..utilities import clean_temporary_pub, create_temporary_pub
+
+
+def setup_module(module):
+    cleanup()
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+@pytest.fixture
+def pub():
+    pub = create_temporary_pub()
+    pub.cfg['language'] = {'language': 'en'}
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.write_cfg()
+    req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
+    req.response.filter = {}
+    req._user = None
+    pub._set_request(req)
+    req.session = sessions.BasicSession(id=1)
+    pub.set_config(req)
+    return pub
+
+
+@pytest.mark.skipif(transform_to_pdf is None, reason='libreoffice not found')
+def test_transform_to_pdf():
+    with open(os.path.join(os.path.dirname(__file__), '..', 'template.odt'), 'rb') as instream:
+        outstream = transform_to_pdf(instream)
+        assert outstream is not False
+        assert outstream.read(10).startswith(b'%PDF-')
+
+
+@pytest.mark.parametrize(
+    'template_name', ['template-with-image.odt', 'template-with-image-django-syntax.odt']
+)
+def test_export_to_model_image(pub, template_name):
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        FileField(id='3', label='File', varname='image'),
+    ]
+    formdef.store()
+
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
+        image_data = fd.read()
+    upload.receive([image_data])
+
+    formdata = formdef.data_class()()
+    formdata.data = {'3': upload}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = ExportToModel()
+    item.convert_to_pdf = False
+    item.method = 'non-interactive'
+    template_filename = os.path.join(os.path.dirname(__file__), '..', template_name)
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    item.model_file = UploadedFile(pub.app_dir, None, upload)
+    item.attach_to_history = True
+
+    item.perform(formdata)
+
+    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
+    with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
+        zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
+    # check the image has been replaced by the one from the formdata
+    assert zinfo.file_size == len(image_data)
+
+    # check with missing data or wrong kind of data
+    for field_value in (None, 'wrong kind'):
+        formdata = formdef.data_class()()
+        formdata.data = {'3': field_value}
+        formdata.just_created()
+        formdata.store()
+        pub.substitutions.feed(formdata)
+
+        item.perform(formdata)
+
+        with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
+            zinfo = zfile.getinfo('Pictures/10000000000000320000003276E9D46581B55C88.jpg')
+        # check the original image has been left
+        assert zinfo.file_size == 580
+
+    item.filename = 'formulaire-{{form_number}}/2.odt'
+    item.perform(formdata)
+    assert formdata.evolution[-1].parts[-1].base_filename == 'formulaire-%s-%s-2.odt' % (
+        formdef.id,
+        formdata.id,
+    )
+
+
+def test_export_to_model_qrcode(pub):
+    FormDef.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = []
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = ExportToModel()
+    item.convert_to_pdf = False
+    item.method = 'non-interactive'
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-with-qrcode.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    item.model_file = UploadedFile(pub.app_dir, None, upload)
+    item.attach_to_history = True
+
+    item.perform(formdata)
+
+    assert formdata.evolution[-1].parts[-1].base_filename == 'template.odt'
+    with zipfile.ZipFile(formdata.evolution[-1].parts[-1].get_file_path(), mode='r') as zfile:
+        # base template use a jpg images and export_to_model does not rename it
+        # event when content is PNG, but it still works inside LibreOffice
+        # which ignores the filename extension.
+        image_filename = [name for name in zfile.namelist() if name.endswith('.jpg')][0]
+        with zfile.open(image_filename, 'r') as image_fd:
+            img = Image.open(image_fd)
+            assert (
+                zbar_decode_qrcode(img, symbols=[ZBarSymbol.QRCODE])[0].data.decode()
+                == 'http://example.net/backoffice/management/baz/1/'
+            )
+
+
+def test_export_to_model_backoffice_field(pub):
+    wf = Workflow(name='email with attachments')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+    formdef = FormDef()
+    formdef.name = 'foo-export-to-bofile'
+    formdef.fields = [
+        StringField(id='1', label='String', varname='string'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = ExportToModel()
+    item.method = 'non-interactive'
+    item.convert_to_pdf = False
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    item.model_file = UploadedFile(pub.app_dir, None, upload)
+    item.parent = st1
+    item.backoffice_filefield_id = 'bo1'
+    item.perform(formdata)
+
+    assert 'bo1' in formdata.data
+    fbo1 = formdata.data['bo1']
+    assert fbo1.base_filename == 'template.odt'
+    assert fbo1.content_type == 'application/octet-stream'
+    with zipfile.ZipFile(fbo1.get_file()) as zfile:
+        assert b'foo-export-to-bofile' in zfile.read('content.xml')
+
+    # no more 'bo1' backoffice field: do nothing
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+    # id is not bo1:
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo2', label='bo field 2'),
+    ]
+    item.perform(formdata)
+    assert formdata.data == {}
+    # field is not a field file:
+    wf.backoffice_fields_formdef.fields = [
+        StringField(id='bo1', label='bo field 1'),
+    ]
+    item.perform(formdata)
+    assert formdata.data == {}
+    # no field at all:
+    wf.backoffice_fields_formdef.fields = []
+    item.perform(formdata)
+    assert formdata.data == {}
+
+
+def test_export_to_model_django_template(pub):
+    formdef = FormDef()
+    formdef.name = 'foo-export-to-template-with-django'
+    formdef.fields = [
+        StringField(id='1', label='String', varname='string'),
+    ]
+    formdef.store()
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.store()
+    pub.substitutions.feed(formdata)
+
+    item = ExportToModel()
+    item.method = 'non-interactive'
+    item.attach_to_history = True
+    template_filename = os.path.join(os.path.dirname(__file__), '..', 'template-django.odt')
+    with open(template_filename, 'rb') as fd:
+        template = fd.read()
+    upload = QuixoteUpload('/foo/template-django.odt', content_type='application/octet-stream')
+    upload.fp = io.BytesIO()
+    upload.fp.write(template)
+    upload.fp.seek(0)
+    item.model_file = UploadedFile(pub.app_dir, None, upload)
+    item.convert_to_pdf = False
+    item.perform(formdata)
+
+    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
+        with zipfile.ZipFile(fd) as zout:
+            new_content = zout.read('content.xml')
+    assert b'>foo-export-to-template-with-django<' in new_content
+
+    formdef.name = 'Name with a \' simple quote'
+    formdef.store()
+    item.perform(formdata)
+
+    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
+        with zipfile.ZipFile(fd) as zout:
+            new_content = zout.read('content.xml')
+    assert b'>Name with a \' simple quote<' in new_content
+
+    formdef.name = 'A <> name'
+    formdef.store()
+    item.perform(formdata)
+
+    with open(formdata.evolution[0].parts[-1].get_file_path(), 'rb') as fd:
+        with zipfile.ZipFile(fd) as zout:
+            new_content = zout.read('content.xml')
+    assert b'>A &lt;&gt; name<' in new_content
+
+
+def test_export_to_model_xml(pub):
+    LoggedError = pub.loggederror_class
+    if LoggedError:
+        LoggedError.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'foo-export-to-template-with-django'
+    formdef.fields = [
+        StringField(id='1', label='String', varname='string'),
+    ]
+    formdef.store()
+    formdata = formdef.data_class()()
+    formdata.data = {'1': 'écho'}
+    formdata.just_created()
+    formdata.store()
+
+    # good XML
+    item = ExportToModel()
+    item.method = 'non-interactive'
+    item.attach_to_history = True
+
+    def run(template, filename='/foo/template.xml', content_type='application/xml'):
+        upload = QuixoteUpload(filename, content_type=content_type)
+        upload.fp = io.BytesIO()