backoffice: formdata filtering by block field (#58451)

tests/backoffice_pages/test_filters.py

@@ -5,6 +5,8 @@ import time
 import pytest
 
 from wcs import fields
+from wcs.blocks import BlockDef
+from wcs.data_sources import NamedDataSource
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
@@ -708,3 +710,179 @@ def test_backoffice_table_varname_filter(pub):
 
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('<tr') == 6
+
+
+def test_backoffice_block_field_filter(pub, local_user):
+    if not pub.is_using_postgresql():
+        pytest.skip('this requires SQL')
+
+    create_superuser(pub)
+    NamedDataSource.wipe()
+    data_source = NamedDataSource(name='foobar')
+    data_source.data_source = {
+        'type': 'formula',
+        'value': repr([{'id': '1', 'text': 'foo', 'more': 'XXX'}, {'id': '2', 'text': 'bar', 'more': 'YYY'}]),
+    }
+    data_source.store()
+
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='1', label='String', type='string', varname='string'),
+        fields.ItemField(id='2', label='Item', type='item', data_source={'type': 'foobar'}, varname='item'),
+        fields.BoolField(id='3', label='Bool', type='bool', varname='bool'),
+        fields.DateField(id='4', label='Date', type='date', varname='date'),
+        fields.EmailField(id='5', label='Email', type='email', varname='email'),
+    ]
+    block.store()
+
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    local_user.roles = [role.id]
+    local_user.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.fields = [
+        fields.BlockField(id='0', label='Block Data', varname='blockdata', type='block:foobar', max_items=3),
+    ]
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(10):
+        formdata = data_class()
+        formdata.data = {
+            '0': {
+                'data': [
+                    {
+                        '1': 'plop%s' % i,
+                        '2': '1' if i % 2 else '2',
+                        '2_display': 'foo' if i % 2 else 'bar',
+                        '2_structured': 'XXX' if i % 2 else 'YYY',
+                        '3': bool(i % 2),
+                        '4': '2021-06-%02d' % (i + 1),
+                        '5': 'a@localhost' if i % 2 else 'b@localhost',
+                    },
+                ],
+                'schema': {},  # not important here
+            },
+            '0_display': 'hello',
+        }
+        if i == 0:
+            formdata.data['0']['data'].append(
+                {
+                    '1': 'plop%s' % i,
+                    '2': '1',
+                    '2_display': 'foo',
+                    '2_structured': 'XXX',
+                    '3': True,
+                    '4': '2021-06-02',
+                    '5': 'a@localhost',
+                },
+            )
+        formdata.user_id = local_user.id
+        formdata.just_created()
+        formdata.jump_status('new')
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    assert '<label><input type="checkbox" name="1"/>Block Data / String</label>' in resp
+    assert '<label><input type="checkbox" name="2"/>Block Data / Item</label>' in resp
+    assert '<label><input type="checkbox" name="3"/>Block Data / Bool</label>' in resp
+    assert '<label><input type="checkbox" name="4"/>Block Data / Date</label>' in resp
+    assert '<label><input type="checkbox" name="5"/>Block Data / Email</label>' in resp
+
+    # string
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-1'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-0-1-value'].value == ''
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop0'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop2'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop10'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 0
+
+    # item
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-2'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-0-2-value'].value == ''
+    resp.forms['listing-settings']['filter-0-2-value'].value = '1'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 6
+    resp.forms['listing-settings']['filter-0-2-value'].value = '2'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 5
+
+    # bool
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-3'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-0-3-value'].value == ''
+    resp.forms['listing-settings']['filter-0-3-value'].value = 'true'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 6
+    resp.forms['listing-settings']['filter-0-3-value'].value = 'false'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 5
+
+    # date
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-4'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-0-4-value'].value == ''
+    resp.forms['listing-settings']['filter-0-4-value'].value = '2021-06-01'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1
+    resp.forms['listing-settings']['filter-0-4-value'].value = '2021-06-02'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 2
+    resp.forms['listing-settings']['filter-0-4-value'].value = '02/06/2021'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 2
+
+    # email
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-5'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.forms['listing-settings']['filter-0-5-value'].value == ''
+    resp.forms['listing-settings']['filter-0-5-value'].value = 'a@localhost'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 6
+    resp.forms['listing-settings']['filter-0-5-value'].value = 'b@localhost'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 5
+    resp.forms['listing-settings']['filter-0-5-value'].value = 'c@localhost'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 0
+
+    # mix
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter-0-1'].checked = True
+    resp.forms['listing-settings']['filter-0-2'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop1'
+    resp.forms['listing-settings']['filter-0-2-value'].value = '1'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop0'
+    resp.forms['listing-settings']['filter-0-2-value'].value = '1'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1
+    resp.forms['listing-settings']['filter-0-1-value'].value = 'plop0'
+    resp.forms['listing-settings']['filter-0-2-value'].value = '2'
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('<tr') == 1 + 1

wcs/backoffice/management.py

@@ -939,10 +939,30 @@ class FormPage(Directory):
             criterias.append(Equal('status', 'wf-%s' % selected_filter))
         from wcs import sql
 
-        criterias.append(NotNull(sql.get_field_id(filter_field)))
-        options = self.formdef.data_class().select_distinct(
-            [sql.get_field_id(filter_field), '%s_display' % sql.get_field_id(filter_field)], clause=criterias
-        )
+        # for item/items fields, get actual option values from database
+        if not getattr(filter_field, 'block_field', None):
+            criterias.append(NotNull(sql.get_field_id(filter_field)))
+            options = self.formdef.data_class().select_distinct(
+                [sql.get_field_id(filter_field), '%s_display' % sql.get_field_id(filter_field)],
+                clause=criterias,
+            )
+        else:
+            # in case of blocks, this requires digging into the jsonb columns,
+            # jsonb_array_elements(BLOCK->'data')->> 'FOOBAR' will return all
+            # values used in repeated blocks, ex:
+            # {"data": [{"FOOBAR": "value1"}, {"FOOBAR": "value2}]}
+            # → ["value1", "value2"}
+            field1 = "jsonb_array_elements(%s->'data')->> '%s'" % (
+                sql.get_field_id(filter_field.block_field),
+                filter_field.id,
+            )
+            field2 = "jsonb_array_elements(%s->'data')->> '%s_display'" % (
+                sql.get_field_id(filter_field.block_field),
+                filter_field.id,
+            )
+            options = self.formdef.data_class().select_distinct(
+                [field1, field2], clause=criterias, first_field_alias='_fid'
+            )
 
         if filter_field.type == 'items':
             # unnest key/values
@@ -1008,32 +1028,46 @@ class FormPage(Directory):
                 continue
             filter_fields.append(field)
 
+            # add contextual_id/contextual_varname attributes
+            # they are id/varname for normal fields
+            # but in case of blocks they are concatenation of block id/varname + field id/varname
+            is_in_block_field = getattr(field, 'block_field', None)
+            field.contextual_id = field.id
+            field.contextual_varname = None
+            if is_in_block_field:
+                field.contextual_id = '%s-%s' % (field.block_field.id, field.id)
+                field.label = '%s / %s' % (field.block_field.label, field.label)
+                if field.varname and field.block_field.varname:
+                    field.contextual_varname = '%s_%s' % (field.block_field.varname, field.varname)
+            else:
+                field.contextual_varname = field.varname
+
             if get_request().form:
-                field.enabled = ('filter-%s' % field.id in get_request().form) or (
-                    'filter-%s' % field.varname in get_request().form
+                field.enabled = ('filter-%s' % field.contextual_id in get_request().form) or (
+                    'filter-%s' % field.contextual_varname in get_request().form
                 )
-                if 'filter-%s' % field.varname in get_request().form and (
-                    'filter-%s-value' % field.varname not in get_request().form
+                if 'filter-%s' % field.contextual_varname in get_request().form and (
+                    'filter-%s-value' % field.contextual_varname not in get_request().form
                 ):
                     # if ?filter-<varname>= is used, take the value and put it
                     # into filter-<field id>-value so it is used to fill the
                     # fields.
-                    get_request().form['filter-%s-value' % field.id] = get_request().form.get(
-                        'filter-%s' % field.varname
+                    get_request().form['filter-%s-value' % field.contextual_id] = get_request().form.get(
+                        'filter-%s' % field.contextual_varname
                     )
                     if (
-                        field.varname in ('start', 'end', 'user', 'submission-agent')
-                        and get_request().form['filter-%s-value' % field.id] == 'on'
+                        field.contextual_varname in ('start', 'end', 'user', 'submission-agent')
+                        and get_request().form['filter-%s-value' % field.contextual_id] == 'on'
                     ):
                         # reset start/end to an empty value when they're just
                         # being enabled
-                        get_request().form['filter-%s-value' % field.id] = ''
+                        get_request().form['filter-%s-value' % field.contextual_id] = ''
                 if not field.enabled and self.view and get_request().form.get('keep-view-filters'):
                     # keep-view-filters=on is used to initialize page with
                     # filters from both the custom view and the query string.
-                    field.enabled = field.id in default_filters
+                    field.enabled = field.contextual_id in default_filters
             else:
-                field.enabled = field.id in default_filters
+                field.enabled = field.contextual_id in default_filters
                 if not self.view and field.type in ('item', 'items'):
                     field.enabled = field.in_filters
 
@@ -1057,7 +1091,7 @@ class FormPage(Directory):
             if not filter_field.enabled:
                 continue
 
-            filter_field_key = 'filter-%s-value' % filter_field.id
+            filter_field_key = 'filter-%s-value' % filter_field.contextual_id
             filter_field_value = filters_dict.get(filter_field_key)
 
             if filter_field.type == 'status':
@@ -1144,16 +1178,16 @@ class FormPage(Directory):
                         options = self.get_item_filter_options(filter_field, selected_filter, criterias)
                         options = [(x[0], x[1], x[0]) for x in options]
                         options.insert(0, (None, '', ''))
-                        attrs = {'data-refresh-options': str(filter_field.id)}
+                        attrs = {'data-refresh-options': str(filter_field.contextual_id)}
                         if self.view and self.view.visibility == 'datasource':
                             options.append(('{}', _('custom value'), '{}'))
                             if filter_field_value and filter_field_value not in [x[0] for x in options]:
                                 options.append((filter_field_value, filter_field_value, filter_field_value))
                             attrs['data-allow-template'] = 'true'
                     else:
-                        current_filter = filters_dict.get('filter-%s-value' % filter_field.id)
+                        current_filter = filters_dict.get('filter-%s-value' % filter_field.contextual_id)
                         options = [(current_filter, '', current_filter or '')]
-                        attrs = {'data-remote-options': str(filter_field.id)}
+                        attrs = {'data-remote-options': str(filter_field.contextual_id)}
                         get_response().add_javascript(
                             ['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js']
                         )
@@ -1221,13 +1255,13 @@ class FormPage(Directory):
         for field in filter_fields:
             addable = getattr(field, 'addable', True)
             r += htmltext('<li %s>') % ('' if addable else 'hidden')
-            r += htmltext('<input type="checkbox" name="filter-%s"') % field.id
+            r += htmltext('<input type="checkbox" name="filter-%s"') % field.contextual_id
             if field.enabled:
                 r += htmltext(' checked="checked"')
-            r += htmltext(' id="fields-filter-%s"') % field.id
+            r += htmltext(' id="fields-filter-%s"') % field.contextual_id
             r += htmltext('/>')
             r += htmltext('<label for="fields-filter-%s">%s</label>') % (
-                field.id,
+                field.contextual_id,
                 misc.ellipsize(field.label, 70),
             )
             r += htmltext('</li>')
@@ -1610,19 +1644,24 @@ class FormPage(Directory):
 
             filter_field_key = None
 
-            field_varname = None
-            is_in_block_field = False
-            if filter_field.varname:
-                if getattr(filter_field, 'block_field', None) and filter_field.block_field.varname:
-                    field_varname = '%s_%s' % (filter_field.block_field.varname, filter_field.varname)
-                    is_in_block_field = True
-                else:
-                    field_varname = filter_field.varname
-            if field_varname:
+            is_in_block_field = getattr(filter_field, 'block_field', None)
+            filter_field.contextual_id = filter_field.id
+            filter_field.contextual_varname = None
+            if is_in_block_field:
+                filter_field.contextual_id = '%s-%s' % (filter_field.block_field.id, filter_field.id)
+                if filter_field.varname and filter_field.block_field.varname:
+                    filter_field.contextual_varname = '%s_%s' % (
+                        filter_field.block_field.varname,
+                        filter_field.varname,
+                    )
+            else:
+                filter_field.contextual_varname = filter_field.varname
+
+            if filter_field.contextual_varname:
                 # if this is a field with a varname and filter-%(varname)s is
                 # present in the query string, enable this filter.
-                if filters_dict.get('filter-%s' % field_varname):
-                    filter_field_key = 'filter-%s' % field_varname
+                if filters_dict.get('filter-%s' % filter_field.contextual_varname):
+                    filter_field_key = 'filter-%s' % filter_field.contextual_varname
 
             if filter_field.type == 'user-id':
                 # convert uuid based filter into local id filter
@@ -1656,10 +1695,10 @@ class FormPage(Directory):
                         filters_dict['filter-submission-agent-value'] = '-1'
                         request_form['filter-submission-agent-value'] = '-1'
 
-            if filters_dict.get('filter-%s' % filter_field.id):
+            if filters_dict.get('filter-%s' % filter_field.contextual_id):
                 # if there's a filter-%(id)s, it is used to enable the actual
                 # filter, and the value will be found in filter-%s-value.
-                filter_field_key = 'filter-%s-value' % filter_field.id
+                filter_field_key = 'filter-%s-value' % filter_field.contextual_id
 
             if not filter_field_key:
                 # if there's not known filter key, skip.

wcs/sql.py

@@ -1749,19 +1749,22 @@ class SqlMixin:
 
     @classmethod
     @guard_postgres
-    def select_distinct(cls, columns, clause=None):
+    def select_distinct(cls, columns, clause=None, first_field_alias=None):
         # do note this method returns unicode strings.
+        column0 = columns[0]
+        if first_field_alias:
+            column0 = '%s as %s' % (column0, first_field_alias)
         conn, cur = get_connection_and_cursor()
         sql_statement = 'SELECT DISTINCT ON (%s) %s FROM %s' % (
             columns[0],
-            ', '.join(columns),
+            ', '.join([column0] + columns[1:]),
             cls._table_name,
         )
         where_clauses, parameters, func_clause = parse_clause(clause)
         assert not func_clause
         if where_clauses:
             sql_statement += ' WHERE ' + ' AND '.join(where_clauses)
-        sql_statement += ' ORDER BY %s' % columns[0]
+        sql_statement += ' ORDER BY %s' % (first_field_alias or columns[0])
         cur.execute(sql_statement, parameters)
         values = [x for x in cur.fetchall()]
         conn.commit()