From 54fc8a9ec3d6750e5f8b34c3f59307ab2f7f6827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sun, 4 Apr 2021 15:11:08 +0200 Subject: [PATCH] api: add statistics API for direct combo usage (#52731) --- tests/api/test_statistics.py | 155 +++++++++++++++++++++++++++++++++++ wcs/backoffice/management.py | 18 +--- wcs/sql.py | 16 +++- wcs/statistics/__init__.py | 0 wcs/statistics/views.py | 128 +++++++++++++++++++++++++++++ wcs/urls.py | 7 ++ 6 files changed, 306 insertions(+), 18 deletions(-) create mode 100644 tests/api/test_statistics.py create mode 100644 wcs/statistics/__init__.py create mode 100644 wcs/statistics/views.py diff --git a/tests/api/test_statistics.py b/tests/api/test_statistics.py new file mode 100644 index 000000000..ff10472c9 --- /dev/null +++ b/tests/api/test_statistics.py @@ -0,0 +1,155 @@ +import datetime +import os + +import pytest + +from wcs.categories import Category +from wcs.formdef import FormDef +from wcs.qommon.http_request import HTTPRequest + +from ..utilities import clean_temporary_pub, create_temporary_pub, get_app +from .utils import sign_uri + + +@pytest.fixture +def pub(): + pub = create_temporary_pub(sql_mode=True) + Category.wipe() + FormDef.wipe() + + req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'}) + pub.set_app_dir(req) + pub.cfg['identification'] = {'methods': ['password']} + pub.cfg['language'] = {'language': 'en'} + pub.write_cfg() + + with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd: + fd.write( + '''\ +[api-secrets] +coucou = 1234 +''' + ) + + return pub + + +def teardown_module(module): + clean_temporary_pub() + + +def test_statistics_index(pub): + get_app(pub).get('/api/statistics/', status=403) + resp = get_app(pub).get(sign_uri('/api/statistics/')) + assert resp.json['data'][0]['name'] == 'Forms Count' + assert resp.json['data'][0]['url'] == 'http://example.net/api/statistics/forms/count/' + + +def test_statistics_index_no_sql(pub): + pub.is_using_postgresql = lambda: False + assert get_app(pub).get(sign_uri('/api/statistics/')).json == {'data': [], 'err': 0} + + +def test_statistics_index_categories(pub): + Category(name='Category A').store() + Category(name='Category B').store() + resp = get_app(pub).get(sign_uri('/api/statistics/')) + category_filter = [x for x in resp.json['data'][0]['filters'] if x['id'] == 'category'][0] + assert len(category_filter['options']) == 3 + + +def test_statistics_forms_count(pub): + category_a = Category(name='Category A') + category_a.store() + category_b = Category(name='Category B') + category_b.store() + + formdef = FormDef() + formdef.name = 'test 1' + formdef.category_id = category_a.id + formdef.fields = [] + formdef.store() + formdef.data_class().wipe() + + formdef2 = FormDef() + formdef2.name = 'test 2' + formdef2.category_id = category_b.id + formdef2.fields = [] + formdef2.store() + formdef2.data_class().wipe() + + for _i in range(20): + formdata = formdef.data_class()() + formdata.just_created() + formdata.receipt_time = datetime.datetime(2021, 1, 1, 0, 0).timetuple() + formdata.store() + + for _i in range(30): + formdata = formdef2.data_class()() + formdata.just_created() + formdata.receipt_time = datetime.datetime(2021, 3, 1, 2, 0).timetuple() + formdata.store() + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/')) + assert resp.json == { + 'data': { + 'series': [{'data': [20, 0, 30], 'label': 'Forms Count'}], + 'x_labels': ['2021-01', '2021-02', '2021-03'], + }, + 'err': 0, + } + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=year')) + assert resp.json == { + 'data': { + 'series': [{'data': [50], 'label': 'Forms Count'}], + 'x_labels': ['2021'], + }, + 'err': 0, + } + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=weekday')) + assert resp.json == { + 'data': { + 'series': [{'data': [30, 0, 0, 0, 20, 0, 0], 'label': 'Forms Count'}], + 'x_labels': ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'], + }, + 'err': 0, + } + + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=hour')) + assert resp.json == { + 'data': { + 'series': [ + { + 'label': 'Forms Count', + 'data': [20, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + } + ], + 'x_labels': list(range(24)), + }, + 'err': 0, + } + + # time_interval=day is not supported + get_app(pub).get(sign_uri('/api/statistics/forms/count/?time_interval=day'), status=400) + + # apply category filter + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?category=%s' % category_a.id)) + assert resp.json == { + 'data': { + 'series': [{'data': [20], 'label': 'Forms Count'}], + 'x_labels': ['2021-01'], + }, + 'err': 0, + } + + # apply period filter + resp = get_app(pub).get(sign_uri('/api/statistics/forms/count/?end=2021-02-01')) + assert resp.json == { + 'data': { + 'series': [{'data': [20], 'label': 'Forms Count'}], + 'x_labels': ['2021-01'], + }, + 'err': 0, + } diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py index 3f52e536a..e4d2084cf 100644 --- a/wcs/backoffice/management.py +++ b/wcs/backoffice/management.py @@ -3461,22 +3461,6 @@ def do_graphs_section(period_start=None, period_end=None, criterias=None): yearly_totals = [(datetime.date.today().year, 0)] weekday_totals = sql.get_weekday_totals(period_start, period_end, criterias) - weekday_line = [] - weekday_names = [ - _('Sunday'), - _('Monday'), - _('Tuesday'), - _('Wednesday'), - _('Thursday'), - _('Friday'), - _('Saturday'), - ] - for weekday, total in weekday_totals: - label = weekday_names[weekday] - weekday_line.append((label, total)) - # move Sunday to the last place - weekday_line = weekday_line[1:] + [weekday_line[0]] - hour_totals = sql.get_hour_totals(period_start, period_end, criterias) r += htmltext( @@ -3487,7 +3471,7 @@ var month_line = %(month_line)s; var year_line = %(year_line)s; ''' % { - 'weekday_line': json.dumps(weekday_line), + 'weekday_line': json.dumps(weekday_totals), 'hour_line': json.dumps(hour_totals), 'month_line': json.dumps(monthly_totals), 'year_line': json.dumps(yearly_totals), diff --git a/wcs/sql.py b/wcs/sql.py index 83f66b6f8..dc0c98abd 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -46,7 +46,7 @@ from wcs.qommon import PICKLE_KWARGS, force_str from . import qommon from .publisher import UnpicklerClass -from .qommon import get_cfg +from .qommon import _, get_cfg from .qommon.misc import strftime from .qommon.storage import _take, deep_bytes2str from .qommon.storage import parse_clause as parse_storage_clause @@ -3178,6 +3178,20 @@ def get_weekday_totals(period_start=None, period_end=None, criterias=None): result.append((weekday, 0)) result.sort() + # add labels, + weekday_names = [ + _('Sunday'), + _('Monday'), + _('Tuesday'), + _('Wednesday'), + _('Thursday'), + _('Friday'), + _('Saturday'), + ] + result = [(weekday_names[x], y) for (x, y) in result] + # and move Sunday last + result = result[1:] + [result[0]] + conn.commit() cur.close() diff --git a/wcs/statistics/__init__.py b/wcs/statistics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/wcs/statistics/views.py b/wcs/statistics/views.py new file mode 100644 index 000000000..a0fa5e22e --- /dev/null +++ b/wcs/statistics/views.py @@ -0,0 +1,128 @@ +# w.c.s. - web application for online forms +# Copyright (C) 2005-2021 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +from django.http import HttpResponseBadRequest, HttpResponseForbidden, JsonResponse +from django.urls import reverse +from django.views.generic import View +from quixote import get_publisher + +from wcs.api_utils import is_url_signed +from wcs.categories import Category +from wcs.qommon import _, misc +from wcs.qommon.misc import C_ +from wcs.qommon.storage import Equal + + +class RestrictedView(View): + def dispatch(self, *args, **kwargs): + if not is_url_signed(): + return HttpResponseForbidden() + return super().dispatch(*args, **kwargs) + + +class IndexView(RestrictedView): + def get(self, request, *args, **kwargs): + if not get_publisher().is_using_postgresql(): + return JsonResponse({'data': [], 'err': 0}) + categories = Category.select() + categories.sort(key=lambda x: misc.simplify(x.name)) + category_options = [{'id': '_all', 'label': C_('categories|All')}] + [ + {'id': x.id, 'label': x.name} for x in categories + ] + return JsonResponse( + { + 'data': [ + { + 'name': _('Forms Count'), + 'url': request.build_absolute_uri(reverse('api-statistics-forms-count')), + 'id': 'forms_counts', + 'filters': [ + { + 'id': 'time_interval', + 'label': _('Interval'), + 'options': [ + { + 'id': 'month', + 'label': _('Month'), + }, + { + 'id': 'year', + 'label': _('Year'), + }, + { + 'id': 'weekday', + 'label': _('Week day'), + }, + { + 'id': 'hour', + 'label': _('Hour'), + }, + ], + 'required': True, + 'default': 'month', + }, + { + 'id': 'category', + 'label': _('Category'), + 'options': category_options, + 'required': False, + 'default': '_all', + }, + ], + } + ] + } + ) + + +class FormsCountView(RestrictedView): + def get(self, request, *args, **kwargs): + from wcs import sql + + time_interval = request.GET.get('time_interval', 'month') + totals_kwargs = { + 'period_start': request.GET.get('start'), + 'period_end': request.GET.get('end'), + 'criterias': [], + } + category_id = request.GET.get('category') + if category_id and category_id != '_all': + totals_kwargs['criterias'].append(Equal('category_id', category_id)) + time_interval_methods = { + 'month': sql.get_monthly_totals, + 'year': sql.get_yearly_totals, + 'weekday': sql.get_weekday_totals, + 'hour': sql.get_hour_totals, + } + if time_interval in time_interval_methods: + totals = time_interval_methods[time_interval](**totals_kwargs) + else: + return HttpResponseBadRequest('invalid time_interval parameter') + + return JsonResponse( + { + 'data': { + 'x_labels': [x[0] for x in totals], + 'series': [ + { + 'label': _('Forms Count'), + 'data': [x[1] for x in totals], + } + ], + }, + 'err': 0, + } + ) diff --git a/wcs/urls.py b/wcs/urls.py index 7e7528d24..7bb10c78b 100644 --- a/wcs/urls.py +++ b/wcs/urls.py @@ -17,6 +17,7 @@ from django.conf.urls import url from . import api, compat, views +from .statistics import views as statistics_views urlpatterns = [ url(r'^robots.txt$', views.robots_txt), @@ -26,6 +27,12 @@ urlpatterns = [ url(r'^api/validate-expression$', api.validate_expression, name='api-validate-expression'), url(r'^api/reverse-geocoding$', api.reverse_geocoding, name='api-reverse-geocoding'), url(r'^api/geocoding$', api.geocoding, name='api-geocoding'), + url(r'^api/statistics/$', statistics_views.IndexView.as_view()), + url( + r'^api/statistics/forms/count/$', + statistics_views.FormsCountView.as_view(), + name='api-statistics-forms-count', + ), # provide django.contrib.auth view names for compatibility with # templates created for classic django applications. url(r'^login/$', compat.quixote, name='auth_login'),