entrouvert
/
authentic
16
0
Fork 0
Code Issues Pull Requests 33 Releases Activity

api: fix aggregation by service in statistics (#64853) 

Reference to service can be any kind of model deriving from Service, and
grouping by the array column reference_ids cannot work as the column
could contain other references than to the service.
This commit is contained in:
Benjamin Dauvergne 2022-05-04 17:37:43 +02:00
parent f3c52bc279
commit c1b80a6408
5 changed files with 104 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/authentic2/apps/journal/models.py
View File

@ -119,7 +119,6 @@ class EventTypeDefinition(metaclass=EventTypeDefinitionMeta):
cls,
group_by_time,
group_by_field=None,
group_by_references=False,
which_references=None,
users_ou=None,
start=None,
@ -153,9 +152,6 @@ class EventTypeDefinition(metaclass=EventTypeDefinitionMeta):
if users_ou:
qs = qs.filter(user__ou=users_ou)
if group_by_references:
values.append('reference_ids')
qs = qs.values(*values)
qs = qs.annotate(count=Count('id'))
return qs.order_by(group_by_time)

20
src/authentic2/apps/journal/utils.py
View File

@ -17,6 +17,7 @@
from datetime import date, timedelta
from django.db.models import Max, Min
from django.utils.translation import gettext_lazy as _
def _json_value(value):
@ -45,12 +46,17 @@ class Statistics:
'month': '%Y-%m',
'day': '%Y-%m-%d',
}
default_y_label = _('None')
def __init__(self, qs, time_interval):
self.time_interval = time_interval
self.x_labels = self.build_x_labels(qs)
self._x_labels_indexes = {label: i for i, label in enumerate(self.x_labels)}
self.series = {}
self.y_labels = []
def set_y_labels(self, y_labels):
self.y_labels[:] = y_labels
def build_x_labels(self, qs):
if self.time_interval == 'timestamp':
@ -75,6 +81,8 @@ class Statistics:
return x_labels
def add(self, x_label, y_label, value):
if y_label not in self.y_labels:
self.y_labels.append(y_label)
serie = self.get_serie(y_label)
index = self.x_index(x_label)
serie[index] = (serie[index] or 0) + value
@ -86,7 +94,17 @@ class Statistics:
return self._x_labels_indexes[x_label]
def to_json(self, get_y_label=lambda x: x):
series = [{'label': get_y_label(label), 'data': data} for label, data in self.series.items()]
series = []
if None in self.series:
series.append({'label': self.default_y_label, 'data': self.series[None]})
y_labels = [
(get_y_label(serie_y_label), serie_y_label)
for serie_y_label in self.y_labels
if serie_y_label is not None
]
y_labels.sort()
for y_label, serie_y_label in y_labels:
series.append({'label': y_label, 'data': self.get_serie(serie_y_label)})
return {
'x_labels': [self.format_x_label(label) for label in self.x_labels],
'series': series,

40
src/authentic2/journal_event_types.py
View File

@ -14,11 +14,11 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext_noop as N_
from authentic2.apps.journal.models import EventTypeDefinition, n_2_pairing_rev
from authentic2.a2_rbac.models import OrganizationalUnit as OU
from authentic2.apps.journal.models import EventTypeDefinition
from authentic2.apps.journal.utils import Statistics, form_to_old_new
from authentic2.custom_user.models import User, get_attributes_map
@ -71,42 +71,42 @@ class EventTypeWithHow(EventTypeWithService):
for stat in qs:
stats.add(x_label=stat[group_by_time], y_label=stat['how'], value=stat['count'])
return stats.to_json(get_y_label=lambda x: _(login_method_label(x or '')))
@classmethod
def _get_method_statistics_by_reference(cls, group_by_time, reference, **kwargs):
qs = cls.get_statistics(group_by_time, group_by_references=True, **kwargs)
def _get_method_statistics_by_service_or_ou(cls, group_by_time, reference, **kwargs):
qs = cls.get_statistics(group_by_time, group_by_field='service_name', **kwargs)
stats = Statistics(qs, time_interval=group_by_time)
if reference == 'service':
reference_labels = {service.pk: str(service) for service in Service.objects.all()}
if reference == 'ou':
services = Service.objects.all()
reference_labels = {str(service): str(service) for service in services}
stats.set_y_labels(service.name for service in services)
elif reference == 'ou':
reference_labels = {
service.pk: str(service.ou) for service in Service.objects.all().select_related('ou')
str(service): str(service.ou) for service in Service.objects.all().select_related('ou')
}
stats.set_y_labels(OU.objects.values_list('name', flat=True))
else:
raise NotImplementedError
service_ct_id = ContentType.objects.get_for_model(Service).pk
for stat in qs:
for reference_id in stat['reference_ids'] or []:
content_type_id, instance_pk = n_2_pairing_rev(reference_id)
if content_type_id == service_ct_id:
reference_label = reference_labels.get(instance_pk)
break
else:
reference_label = _('None')
if reference_label:
stats.add(x_label=stat[group_by_time], y_label=reference_label, value=stat['count'])
y_label = (
reference_labels.get(stat['service_name'], stat['service_name'])
if stat['service_name'] is not None
else None
)
stats.add(x_label=stat[group_by_time], y_label=y_label, value=stat['count'])
return stats.to_json()
@classmethod
def get_service_statistics(cls, group_by_time, start=None, end=None):
return cls._get_method_statistics_by_reference(group_by_time, 'service', start=start, end=end)
return cls._get_method_statistics_by_service_or_ou(group_by_time, 'service', start=start, end=end)
@classmethod
def get_service_ou_statistics(cls, group_by_time, start=None, end=None):
return cls._get_method_statistics_by_reference(group_by_time, 'ou', start=start, end=end)
return cls._get_method_statistics_by_service_or_ou(group_by_time, 'ou', start=start, end=end)
def login_method_label(how):

32
tests/test_api.py
View File

@ -2495,33 +2495,35 @@ def test_api_statistics(app, admin, freezer, event_type_name, event_name):
event_type = EventType.objects.get_for_name(event_type_name)
freezer.move_to('2020-02-03 12:00')
Event.objects.create(type=event_type, references=[portal], data=method)
Event.objects.create(type=event_type, references=[agendas, user], user=user, data=method)
Event.objects.create(type=event_type, references=[portal], data=dict(method, service_name=str(portal)))
Event.objects.create(
type=event_type, references=[agendas, user], user=user, data=dict(method, service_name=str(agendas))
)
freezer.move_to('2020-03-04 13:00')
Event.objects.create(type=event_type, references=[agendas], data=method)
Event.objects.create(type=event_type, references=[portal], data=method2)
Event.objects.create(type=event_type, references=[agendas], data=dict(method, service_name=str(agendas)))
Event.objects.create(type=event_type, references=[portal], data=dict(method2, service_name=str(portal)))
resp = app.get('/api/statistics/%s/?time_interval=month' % event_name, headers=headers)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020-02', '2020-03'],
'series': [{'label': 'FranceConnect', 'data': [None, 1]}, {'label': 'password', 'data': [2, 1]}],
'series': [
{'label': 'FranceConnect', 'data': [None, 1]},
{'label': 'password', 'data': [2, 1]},
],
}
# default time interval is 'month'
month_data = data
resp = app.get('/api/statistics/%s/' % event_name, headers=headers)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert month_data == data
resp = app.get(
'/api/statistics/%s/?time_interval=month&services_ou=default' % event_name, headers=headers
)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020-02', '2020-03'],
'series': [{'label': 'password', 'data': [1, 1]}],
@ -2531,7 +2533,6 @@ def test_api_statistics(app, admin, freezer, event_type_name, event_name):
services_ou_data = data
resp = app.get('/api/statistics/%s/?time_interval=month&ou=default' % event_name, headers=headers)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert services_ou_data == data
resp = app.get(
@ -2558,7 +2559,6 @@ def test_api_statistics(app, admin, freezer, event_type_name, event_name):
'/api/statistics/%s/?time_interval=month&start=2020-03-01T01:01' % event_name, headers=headers
)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020-03'],
'series': [{'label': 'FranceConnect', 'data': [1]}, {'label': 'password', 'data': [1]}],
@ -2578,7 +2578,6 @@ def test_api_statistics(app, admin, freezer, event_type_name, event_name):
'/api/statistics/%s/?time_interval=year&service=portal second' % event_name, headers=headers
)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020'],
'series': [{'label': 'FranceConnect', 'data': [1]}, {'label': 'password', 'data': [1]}],
@ -2586,20 +2585,21 @@ def test_api_statistics(app, admin, freezer, event_type_name, event_name):
resp = app.get('/api/statistics/service_%s/?time_interval=month' % event_name, headers=headers)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020-02', '2020-03'],
'series': [{'label': 'agendas', 'data': [1, 1]}, {'label': 'portal', 'data': [1, 1]}],
'series': [
{'data': [1, 1], 'label': 'agendas'},
{'data': [1, 1], 'label': 'portal'},
],
}
resp = app.get('/api/statistics/service_ou_%s/?time_interval=month' % event_name, headers=headers)
data = resp.json['data']
data['series'].sort(key=lambda x: x['label'])
assert data == {
'x_labels': ['2020-02', '2020-03'],
'series': [
{'label': 'Default organizational unit', 'data': [1, 1]},
{'label': 'Second OU', 'data': [1, 1]},
{'data': [1, 1], 'label': 'Default organizational unit'},
{'data': [1, 1], 'label': 'Second OU'},
],
}

87
tests/test_journal.py
View File

@ -483,27 +483,34 @@ def test_statistics(db, event_type_name, freezer):
stats = event_type_definition.get_method_statistics('month')
assert stats == {'series': [], 'x_labels': []}
def create_event(user, service=None, data=None):
data = (data or {}).copy()
references = [user]
if service:
references.append(service)
data['service_name'] = str(service)
Event.objects.create(type=event_type, references=references, user=user, data=data)
freezer.move_to('2020-02-03 12:00')
Event.objects.create(type=event_type, references=[user, portal], user=user, data=method)
Event.objects.create(type=event_type, references=[user2, portal], user=user2, data=method)
create_event(user, portal, method)
create_event(user2, portal, method)
freezer.move_to('2020-02-03 13:00')
Event.objects.create(type=event_type, references=[user, portal], user=user, data=method2)
Event.objects.create(type=event_type, references=[user2, portal], user=user2, data=method2)
create_event(user, portal, method2)
create_event(user2, portal, method2)
freezer.move_to('2020-03-03 12:00')
Event.objects.create(type=event_type, references=[user, portal], user=user, data=method)
Event.objects.create(type=event_type, references=[user, agendas], user=user, data=method)
Event.objects.create(type=event_type, references=[user, forms], user=user, data=method)
Event.objects.create(type=event_type, user=user)
create_event(user, portal, method)
create_event(user, agendas, method)
create_event(user, forms, method)
create_event(user)
stats = event_type_definition.get_method_statistics('timestamp')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02-03T12:00:00+00:00', '2020-02-03T13:00:00+00:00', '2020-03-03T12:00:00+00:00'],
'series': [
{'label': 'None', 'data': [None, None, 1]},
{'label': 'FranceConnect', 'data': [None, 2, None]},
{'label': 'none', 'data': [None, None, 1]},
{'label': 'password', 'data': [2, None, 3]},
],
}
@ -519,13 +526,12 @@ def test_statistics(db, event_type_name, freezer):
}
stats = event_type_definition.get_method_statistics('month')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02', '2020-03'],
'series': [
{'label': 'FranceConnect', 'data': [2, None]},
{'label': 'none', 'data': [None, 1]},
{'label': 'password', 'data': [2, 3]},
{'data': [None, 1], 'label': 'None'},
{'data': [2, None], 'label': 'FranceConnect'},
{'data': [2, 3], 'label': 'password'},
],
}
@ -538,64 +544,66 @@ def test_statistics(db, event_type_name, freezer):
}
stats = event_type_definition.get_method_statistics('month', services_ou=ou)
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02', '2020-03'],
'series': [{'label': 'FranceConnect', 'data': [2, None]}, {'label': 'password', 'data': [2, 1]}],
'series': [
{'label': 'FranceConnect', 'data': [2, None]},
{'label': 'password', 'data': [2, 1]},
],
}
stats = event_type_definition.get_method_statistics('month', users_ou=ou)
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02'],
'series': [{'label': 'FranceConnect', 'data': [1]}, {'label': 'password', 'data': [1]}],
'series': [
{'data': [1], 'label': 'FranceConnect'},
{'data': [1], 'label': 'password'},
],
}
stats = event_type_definition.get_method_statistics('month', service=portal)
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02', '2020-03'],
'series': [{'label': 'FranceConnect', 'data': [2, None]}, {'label': 'password', 'data': [2, 1]}],
'series': [
{'label': 'FranceConnect', 'data': [2, None]},
{'label': 'password', 'data': [2, 1]},
],
}
stats = event_type_definition.get_method_statistics('month', service=agendas, users_ou=get_default_ou())
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-03'],
'series': [{'label': 'password', 'data': [1]}],
}
stats = event_type_definition.get_method_statistics('year')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020'],
'series': [
{'label': 'FranceConnect', 'data': [2]},
{'label': 'none', 'data': [1]},
{'label': 'password', 'data': [5]},
{'data': [1], 'label': 'None'},
{'data': [2], 'label': 'FranceConnect'},
{'data': [5], 'label': 'password'},
],
}
stats = event_type_definition.get_service_statistics('month')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02', '2020-03'],
'series': [
{'label': 'None', 'data': [None, 1]},
{'label': 'agendas', 'data': [None, 1]},
{'label': 'forms', 'data': [None, 1]},
{'label': 'portal', 'data': [4, 1]},
{'data': [None, 1], 'label': 'None'},
{'data': [None, 1], 'label': 'agendas'},
{'data': [None, 1], 'label': 'forms'},
{'data': [4, 1], 'label': 'portal'},
],
}
stats = event_type_definition.get_service_ou_statistics('month')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02', '2020-03'],
'series': [
{'label': 'Default organizational unit', 'data': [None, 2]},
{'label': 'None', 'data': [None, 1]},
{'label': 'Second OU', 'data': [4, 1]},
{'data': [None, 1], 'label': 'None'},
{'data': [None, 2], 'label': 'Default organizational unit'},
{'data': [4, 1], 'label': 'Second OU'},
],
}
@ -651,11 +659,12 @@ def test_statistics_deleted_service(db, freezer):
event_type_definition = event_type.definition
freezer.move_to('2020-02-03 12:00')
Event.objects.create(type=event_type, references=[user, portal], user=user, data=method)
Event.objects.create(
type=event_type, references=[user, portal], user=user, data=dict(method, service_name=str(portal))
)
Event.objects.create(type=event_type, references=[user], user=user, data=method)
stats = event_type_definition.get_service_statistics('month')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {
'x_labels': ['2020-02'],
'series': [{'label': 'None', 'data': [1]}, {'label': 'portal', 'data': [1]}],
@ -663,8 +672,10 @@ def test_statistics_deleted_service(db, freezer):
portal.delete()
stats = event_type_definition.get_service_statistics('month')
stats['series'].sort(key=lambda x: x['label'])
assert stats == {'x_labels': ['2020-02'], 'series': [{'label': 'None', 'data': [1]}]}
assert stats == {
'x_labels': ['2020-02'],
'series': [{'data': [1], 'label': 'None'}, {'data': [1], 'label': 'portal'}],
}
def test_statistics_ou_with_no_service(db, freezer):