tests/api/test_statistics.py

@@ -53,6 +53,7 @@ def formdef(pub):
     workflow.add_status(name='End status')
     middle_status1 = workflow.add_status(name='Middle status 1')
     middle_status2 = workflow.add_status(name='Middle status 2')
+    workflow.add_status(name='Just submitted', id='just_sumbitted')
     jump = new_status.add_action('jump', id='_jump')
     jump.status = '2'
     jump.timeout = 86400
@@ -111,19 +112,6 @@ def test_statistics_index(pub):
     assert resp.json['data'][0]['url'] == 'http://example.net/api/statistics/forms/count/'
 
 
-def test_statistics_index_categories(pub):
-    Category(name='Category A').store()
-    Category(name='Category B').store()
-    resp = get_app(pub).get(sign_uri('/api/statistics/'))
-    category_filter = [x for x in resp.json['data'][0]['filters'] if x['id'] == 'category'][0]
-    assert category_filter['options'] == [
-        {'id': '_all', 'label': 'All'},
-        {'id': 'category-a', 'label': 'Category A'},
-        {'id': 'category-b', 'label': 'Category B'},
-    ]
-    assert category_filter['deprecated'] is True
-
-
 def test_statistics_index_forms(pub):
     formdef = FormDef()
     formdef.name = 'test 1'
@@ -388,30 +376,15 @@ def test_statistics_forms_count(pub):
 
     # apply category filter through form parameter
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=category:category-a'))
-    assert resp.json == {
-        'data': {
-            'series': [{'data': [20], 'label': 'Forms Count'}],
-            'x_labels': ['2021-01'],
-            'subfilters': [],
-        },
-        'err': 0,
-    }
-
-    # apply category filter (legacy)
-    new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?category=category-a'))
-    assert new_resp.json == resp.json
-
-    # apply category id filter (legacy)
-    new_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?category=%s' % category_a.id))
-    assert new_resp.json == resp.json
+    assert resp.json['data']['series'] == [{'data': [20], 'label': 'Forms Count'}]
+    assert resp.json['data']['x_labels'] == ['2021-01']
 
     # apply form filter
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % formdef.url_name))
     assert resp.json['data']['series'] == [{'data': [20], 'label': 'Forms Count'}]
     assert resp.json['data']['x_labels'] == ['2021-01']
 
-    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % 'invalid'), status=400)
-    assert resp.text == 'invalid form'
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=%s' % 'invalid'), status=404)
 
     # apply period filter
     resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?end=2021-02-01'))
@@ -510,6 +483,7 @@ def test_statistics_forms_count_subfilters(pub, formdef):
             {'id': 'done', 'label': 'Done'},
             {'id': '1', 'label': 'New status'},
             {'id': '2', 'label': 'End status'},
+            {'id': 'just_sumbitted', 'label': 'Just submitted'},
         ],
         'required': True,
     }
@@ -980,8 +954,7 @@ def test_statistics_cards_count(pub):
     assert resp.json['data']['series'] == [{'data': [20], 'label': 'Cards Count'}]
     assert resp.json['data']['x_labels'] == ['2021-01']
 
-    resp = get_app(pub).get(sign_uri('/api/statistics/cards/count/?card=%s' % 'invalid'), status=400)
-    assert resp.text == 'invalid form'
+    resp = get_app(pub).get(sign_uri('/api/statistics/cards/count/?card=%s' % 'invalid'), status=404)
 
 
 def test_statistics_resolution_time(pub, freezer):
@@ -1255,3 +1228,157 @@ def test_statistics_resolution_time_cards(pub, freezer):
         '5 day(s) and 0 hour(s)',
         '5 day(s) and 0 hour(s)',
     ]
+
+
+def test_statistics_multiple_forms_count(pub, formdef):
+    formdef1 = FormDef()
+    formdef1.name = 'xxx'
+    formdef1.fields = [x for x in formdef.fields if x.varname != 'blockdata']
+    formdef1.store()
+
+    formdef2 = FormDef()
+    formdef2.name = 'yyy'
+    formdef2.workflow = formdef.workflow
+    formdef2.fields = formdef.fields
+    formdef2.store()
+
+    for i in range(20):
+        formdata = formdef1.data_class()()
+        formdata.data['2'] = 'foo'
+        formdata.data['2_display'] = 'Foo'
+        formdata.data['3'] = ['foo']
+        formdata.data['3_display'] = 'Foo'
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.store()
+
+    for _i in range(30):
+        formdata = formdef2.data_class()()
+        formdata.data['2'] = 'baz'
+        formdata.data['2_display'] = 'Baz'
+        formdata.data['3'] = ['foo', 'bar', 'baz']
+        formdata.data['3_display'] = 'Bar, Baz'
+        formdata.data['4'] = {'data': [{'1': True}]}
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.jump_status('2')
+        formdata.store()
+
+    resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/'))
+    assert resp.json['data']['series'] == [{'data': [20, 0, 30], 'label': 'Forms Count'}]
+    assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03']
+
+    # filter by all forms explicitely
+    url = '/api/statistics/forms/count/?form=%s&form=%s' % (formdef1.url_name, formdef2.url_name)
+    resp = get_app(pub).get(sign_uri(url))
+    assert resp.json['data']['series'] == [{'data': [20, 0, 30], 'label': 'Forms Count'}]
+    assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03']
+
+    # filter on item fields
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-item=foo'))
+    assert resp.json['data']['series'][0]['data'] == [20]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-item=baz'))
+    assert resp.json['data']['series'][0]['data'] == [30]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-item=bar'))
+    assert resp.json['data']['series'][0]['data'] == []
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-items=foo'))
+    assert resp.json['data']['series'][0]['data'] == [20, 0, 30]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-test-items=bar'))
+    assert resp.json['data']['series'][0]['data'] == [30]
+
+    # group by item field
+    resp = get_app(pub).get(sign_uri(url + '&group-by=test-item'))
+    assert resp.json['data']['x_labels'] == ['2021-01', '2021-02', '2021-03']
+    assert resp.json['data']['series'] == [
+        {'data': [20, None, None], 'label': 'Foo'},
+        {'data': [None, None, 30], 'label': 'Baz'},
+    ]
+
+    # filter on status
+    resp = get_app(pub).get(sign_uri(url + '&filter-status=_all'))
+    assert resp.json['data']['series'][0]['data'] == [20, 0, 30]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-status=just_submitted'))
+    assert resp.json['data']['series'][0]['data'] == [20]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-status=done'))
+    assert resp.json['data']['series'][0]['data'] == [30]
+
+    resp = get_app(pub).get(sign_uri(url + '&filter-status=pending'))
+    assert resp.json['data']['series'][0]['data'] == [20]
+
+    # filter on status exclusive to one formdef is ignored
+    resp = get_app(pub).get(sign_uri(url + '&filter-status=2'))
+    assert resp.json['data']['series'][0]['data'] == [20, 0, 30]
+
+    # filter on block boolean field exclusive to one formdef yields empty results
+    resp = get_app(pub).get(sign_uri(url + '&filter-blockdata_bool=true'))
+    assert resp.json['data']['series'][0]['data'] == []
+
+
+def test_statistics_multiple_forms_count_subfilters(pub, formdef):
+    category_a = Category(name='Category A')
+    category_a.store()
+
+    formdef.category_id = category_a.id
+    formdef.store()
+
+    formdef2 = FormDef()
+    formdef2.name = 'test 2'
+    formdef2.category_id = category_a.id
+    formdef2.workflow = formdef.workflow
+    formdef2.fields = [x for x in formdef.fields if x.varname not in ('blockdata', 'test-items')]
+    formdef2.store()
+
+    for i in range(20):
+        formdata = formdef.data_class()()
+        formdata.data['2'] = 'foo'
+        formdata.data['2_display'] = 'Foo'
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple()
+        formdata.jump_status('2')
+        formdata.store()
+
+    for _i in range(30):
+        formdata = formdef2.data_class()()
+        formdata.data['2'] = 'baz'
+        formdata.data['2_display'] = 'Baz'
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple()
+        formdata.store()
+
+    resp = get_app(pub).get(
+        sign_uri('/api/statistics/forms/count/?form=%s&form=%s' % (formdef.url_name, formdef2.url_name))
+    )
+
+    # group-by subfilter shows all common fields
+    group_by_filter = [x for x in resp.json['data']['subfilters'] if x['id'] == 'group-by'][0]
+    assert group_by_filter['options'] == [
+        {'id': 'channel', 'label': 'Channel'},
+        {'id': 'simple-status', 'label': 'Simplified status'},
+        {'id': 'test-item', 'label': 'Test item'},
+        {'id': 'checkbox', 'label': 'Checkbox'},
+        {'id': 'status', 'label': 'Status'},
+    ]
+
+    # item field subfilter shows all possible values
+    item_filter = [x for x in resp.json['data']['subfilters'] if x['id'] == 'filter-test-item'][0]
+    assert item_filter['options'] == [{'id': 'baz', 'label': 'Baz'}, {'id': 'foo', 'label': 'Foo'}]
+
+    # boolean field subfilter options are not altered
+    boolean_filter = [x for x in resp.json['data']['subfilters'] if x['id'] == 'filter-checkbox'][0]
+    assert boolean_filter['options'] == [{'id': 'true', 'label': 'Yes'}, {'id': 'false', 'label': 'No'}]
+
+    # block boolean and items subfilters are not shown as they are exclusive to one formdef
+    assert not any(
+        x
+        for x in resp.json['data']['subfilters']
+        if x['id'] in ('filter-blockdata_bool', 'filter-test-items')
+    )
+
+    category_resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?form=category:category-a'))
+    assert category_resp.json == resp.json

wcs/statistics/views.py

@@ -30,6 +30,7 @@ from wcs.categories import Category
 from wcs.formdata import FormData
 from wcs.formdef import FormDef
 from wcs.qommon import _, misc, pgettext_lazy
+from wcs.qommon.errors import TraversalError
 from wcs.qommon.storage import Contains, Equal, GreaterOrEqual, Less, Null, Or, StrictNotEqual
 
 
@@ -42,11 +43,6 @@ class RestrictedView(View):
 
 class IndexView(RestrictedView):
     def get(self, request, *args, **kwargs):
-        categories = Category.select()
-        categories.sort(key=lambda x: misc.simplify(x.name))
-        category_options = [{'id': '_all', 'label': pgettext_lazy('categories', 'All')}] + [
-            {'id': x.url_name, 'label': x.name} for x in categories
-        ]
         channel_options = [{'id': '_all', 'label': pgettext_lazy('channel', 'All')}] + [
             {'id': key, 'label': label} for key, label in FormData.get_submission_channels().items()
         ]
@@ -93,17 +89,6 @@ class IndexView(RestrictedView):
                                 'required': True,
                                 'default': '_all',
                             },
-                            {
-                                'id': 'category',
-                                'label': _('Category'),
-                                'options': category_options,
-                                'required': True,
-                                'default': '_all',
-                                'deprecated': True,
-                                'deprecation_hint': _(
-                                    'Category should now be selected using the Form field below.'
-                                ),
-                            },
                             {
                                 'id': 'form',
                                 'label': _('Form'),
@@ -111,6 +96,7 @@ class IndexView(RestrictedView):
                                 'required': True,
                                 'default': '_all',
                                 'has_subfilters': True,
+                                'multiple': True,
                             },
                         ],
                     },
@@ -239,38 +225,29 @@ class FormsCountView(RestrictedView):
         totals_kwargs = {
             'period_start': request.GET.get('start'),
             'period_end': request.GET.get('end'),
-            'criterias': [],
+            'criterias': [StrictNotEqual('status', 'draft')],
         }
-        category_slug = request.GET.get('category', '_all')
-        formdef_slug = request.GET.get('form', '_all' if self.has_global_count_support else '_nothing')
         group_by = request.GET.get('group-by')
         group_labels = {}
         subfilters = []
-        if formdef_slug != '_all' and not formdef_slug.startswith('category:'):
-            try:
-                formdef = self.formdef_class.get_by_urlname(formdef_slug, ignore_migration=True)
-            except KeyError:
-                return HttpResponseBadRequest('invalid form')
-            form_page = self.formpage_class(formdef=formdef, update_breadcrumbs=False)
-            self.set_formdef_parameters(totals_kwargs, formdef)
-            totals_kwargs['criterias'].extend(self.get_filters_criterias(formdef, form_page))
-            self.set_group_by_parameters(group_by, formdef, form_page, totals_kwargs, group_labels)
-            subfilters = self.get_subfilters(form_page, group_by)
-        else:
-            totals_kwargs['criterias'].append(StrictNotEqual('status', 'draft'))
 
-            if formdef_slug.startswith('category:'):
-                category_slug = formdef_slug.split(':', 1)[1]
-            if category_slug != '_all':
-                try:
-                    category = Category.get_by_urlname(category_slug)
-                except KeyError:
-                    if category_slug.isdigit():  # legacy
-                        totals_kwargs['criterias'].append(Equal('category_id', category_slug))
-                    else:
-                        return HttpResponseBadRequest('invalid category')
-                else:
-                    totals_kwargs['criterias'].append(Equal('category_id', category.id))
+        slugs = request.GET.getlist('form', ['_all'] if self.has_global_count_support else ['_nothing'])
+        if slugs != ['_all']:
+            formdef_slugs = [x for x in slugs if not x.startswith('category:')]
+            criterias = [Contains('url_name', formdef_slugs)]
+            self.filter_by_category(slugs, criterias)
+
+            formdefs = self.formdef_class.select([Or(criterias)])
+            if not formdefs:
+                raise TraversalError()
+
+            for formdef in formdefs:
+                formdef.form_page = self.formpage_class(formdef=formdef, update_breadcrumbs=False)
+
+            self.set_formdef_parameters(totals_kwargs, formdefs)
+            totals_kwargs['criterias'].extend(self.get_filters_criterias(formdefs))
+            self.set_group_by_parameters(group_by, formdefs, totals_kwargs, group_labels)
+            subfilters = self.get_subfilters(formdefs, group_by)
 
         channel = request.GET.get('channel', '_all')
         if channel == 'web':
@@ -310,10 +287,16 @@ class FormsCountView(RestrictedView):
             {'data': {'x_labels': x_labels, 'series': series, 'subfilters': subfilters}, 'err': 0}
         )
 
-    def set_formdef_parameters(self, totals_kwargs, formdef):
+    def filter_by_category(self, slugs, criterias):
+        category_slugs = [x.split(':', 1)[1] for x in slugs if x.startswith('category:')]
+        categories = Category.select([Contains('url_name', category_slugs)], ignore_errors=True)
+        category_ids = [x.id for x in categories]
+        criterias.append(Contains('category_id', category_ids))
+
+    def set_formdef_parameters(self, totals_kwargs, formdefs):
         # set formdef_klass to None to deactivate switching to formdef specific table
         totals_kwargs['criterias'].append(Equal('formdef_klass', None))
-        totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
+        totals_kwargs['criterias'].append(Contains('formdef_id', [x.id for x in formdefs]))
 
     def transform_criteria(self, criteria):
         if not hasattr(criteria, 'field'):
@@ -328,20 +311,24 @@ class FormsCountView(RestrictedView):
 
         return sql.ArrayContains(attribute, value)
 
-    def get_filters_criterias(self, formdef, form_page):
-        criterias = form_page.get_criterias_from_query(statistics_fields_only=True)
+    def get_filters_criterias(self, formdefs):
+        criterias = formdefs[0].form_page.get_criterias_from_query(statistics_fields_only=True)
         criterias = [self.transform_criteria(criteria) for criteria in criterias]
 
         selected_status = self.request.GET.get('filter-status')
         applied_filters = None
         if selected_status and selected_status != '_all':
             if selected_status == 'pending':
-                applied_filters = ['wf-%s' % x.id for x in formdef.workflow.get_not_endpoint_status()]
+                applied_filters = [
+                    'wf-%s' % x.id for formdef in formdefs for x in formdef.workflow.get_not_endpoint_status()
+                ]
             elif selected_status == 'done':
-                applied_filters = ['wf-%s' % x.id for x in formdef.workflow.get_endpoint_status()]
+                applied_filters = [
+                    'wf-%s' % x.id for formdef in formdefs for x in formdef.workflow.get_endpoint_status()
+                ]
             else:
                 try:
-                    formdef.workflow.get_status(selected_status)
+                    formdefs[0].workflow.get_status(selected_status)
                     applied_filters = ['wf-%s' % selected_status]
                 except KeyError:
                     pass
@@ -353,7 +340,44 @@ class FormsCountView(RestrictedView):
 
         return criterias
 
-    def get_subfilters(self, form_page, group_by):
+    def get_subfilters(self, formdefs, group_by):
+        subfilters = None
+        for formdef in formdefs:
+            new_subfilters = self.get_form_subfilters(formdef.form_page, group_by)
+
+            if not subfilters:
+                subfilters = new_subfilters
+                continue
+
+            # keep only common subfilters
+            subfilters = {k: v for k, v in subfilters.items() if k in new_subfilters}
+
+            for filter_id, subfilter in subfilters.copy().items():
+                if subfilter['options'] != new_subfilters[filter_id]['options']:
+                    if filter_id in ('filter-status', 'group-by'):
+                        # keep only common options
+                        subfilter['options'] = {
+                            k: v
+                            for k, v in subfilter['options'].items()
+                            if k in new_subfilters[filter_id]['options']
+                        }
+                    else:
+                        # merge all options for standard filter
+                        subfilter['options'].update(new_subfilters[filter_id]['options'])
+                        subfilter['options']['needs_sorting'] = True
+
+        subfilters = list(subfilters.values())
+        for subfilter in subfilters:
+            needs_sorting = subfilter['options'].pop('needs_sorting', False)
+            subfilter['options'] = [
+                {'id': option, 'label': label} for option, label in subfilter['options'].items()
+            ]
+            if needs_sorting:
+                subfilter['options'].sort(key=lambda x: x['label'])
+
+        return subfilters
+
+    def get_form_subfilters(self, form_page, group_by):
         subfilters = []
         field_choices = []
         for field in form_page.get_formdef_fields():
@@ -389,7 +413,7 @@ class FormsCountView(RestrictedView):
             filter_description = {
                 'id': field_key,
                 'label': field.label,
-                'options': [{'id': x[0], 'label': x[1]} for x in options],
+                'options': {x[0]: x[1] for x in options},
                 'required': field.required,
             }
             if hasattr(field, 'default_filter_value'):
@@ -404,11 +428,11 @@ class FormsCountView(RestrictedView):
                 {
                     'id': 'group-by',
                     'label': _('Group by'),
-                    'options': [
-                        {'id': 'channel', 'label': _('Channel')},
-                        {'id': 'simple-status', 'label': _('Simplified status')},
-                    ]
-                    + [{'id': x[0], 'label': x[1]} for x in field_choices],
+                    'options': {
+                        'channel': _('Channel'),
+                        'simple-status': _('Simplified status'),
+                        **{x[0]: x[1] for x in field_choices},
+                    },
                     'has_subfilters': True,
                 }
             ]
@@ -420,7 +444,7 @@ class FormsCountView(RestrictedView):
                         {
                             'id': 'hide_none_label',
                             'label': _('Ignore forms where "%s" is empty.') % group_by_field.label,
-                            'options': [{'id': 'true', 'label': _('Yes')}, {'id': 'false', 'label': _('No')}],
+                            'options': {'true': _('Yes'), 'false': _('No')},
                             'required': True,
                             'default': 'false',
                         }
@@ -428,6 +452,7 @@ class FormsCountView(RestrictedView):
 
             subfilters = additionnal_filters + subfilters
 
+        subfilters = {x['id']: x for x in subfilters}
         return subfilters
 
     def get_group_by_field(self, form_page, group_by):
@@ -441,7 +466,7 @@ class FormsCountView(RestrictedView):
             if not hasattr(fields[0], 'block_field'):  # block fields are not supported
                 return fields[0]
 
-    def get_group_labels(self, group_by_field, formdef, form_page, group_by):
+    def get_group_labels(self, group_by_field, formdef, group_by):
         group_labels = {}
         if group_by == 'status':
             group_labels = {'wf-%s' % status.id: status.name for status in formdef.workflow.possible_status}
@@ -455,7 +480,7 @@ class FormsCountView(RestrictedView):
         elif group_by_field.type == 'bool':
             group_labels = {True: _('Yes'), False: _('No')}
         elif group_by_field.type in ('item', 'items'):
-            options = form_page.get_item_filter_options(
+            options = formdef.form_page.get_item_filter_options(
                 group_by_field, selected_filter='all', anonymised=True
             )
             group_labels = {option[0]: option[1] for option in options}
@@ -463,7 +488,7 @@ class FormsCountView(RestrictedView):
         group_labels[None] = _('None')
         return group_labels
 
-    def set_group_by_parameters(self, group_by, formdef, form_page, totals_kwargs, group_labels):
+    def set_group_by_parameters(self, group_by, formdefs, totals_kwargs, group_labels):
         if not group_by:
             return
 
@@ -476,9 +501,9 @@ class FormsCountView(RestrictedView):
             group_labels[''] = _('Web')
             return
         elif group_by == 'simple-status':
-            group_by_field = self.get_group_by_field(form_page, 'status')
+            group_by_field = self.get_group_by_field(formdefs[0].form_page, 'status')
         else:
-            group_by_field = self.get_group_by_field(form_page, group_by)
+            group_by_field = self.get_group_by_field(formdefs[0].form_page, group_by)
 
         if not group_by_field:
             return
@@ -491,7 +516,8 @@ class FormsCountView(RestrictedView):
         if self.request.GET.get('hide_none_label') == 'true':
             totals_kwargs['criterias'].append(StrictNotEqual(totals_kwargs['group_by'], '[]'))
 
-        group_labels.update(self.get_group_labels(group_by_field, formdef, form_page, group_by))
+        for formdef in formdefs:
+            group_labels.update(self.get_group_labels(group_by_field, formdef, group_by))
 
     def get_grouped_time_data(self, totals, group_labels):
         totals_by_time = collections.OrderedDict(
@@ -581,11 +607,14 @@ class CardsCountView(FormsCountView):
     has_global_count_support = False
     label = _('Cards Count')
 
-    def set_formdef_parameters(self, totals_kwargs, formdef):
+    def set_formdef_parameters(self, totals_kwargs, formdefs):
         # formdef_klass is a fake criteria, it will be used in time interval functions
         # to switch to appropriate class, it must appear before formdef_id.
         totals_kwargs['criterias'].append(Equal('formdef_klass', CardDef))
-        totals_kwargs['criterias'].append(Equal('formdef_id', formdef.id))
+        totals_kwargs['criterias'].append(Equal('formdef_id', formdefs[0].id))
+
+    def filter_by_category(self, slugs, criterias):
+        pass  # unsupported
 
 
 class ResolutionTimeView(RestrictedView):