journal: permit custom prefetching (#51808)
This commit is contained in:
parent
32cc38c814
commit
e77c98b57b
|
@ -330,13 +330,16 @@ class JournalForm(forms.Form):
|
||||||
first = len(page) <= limit
|
first = len(page) <= limit
|
||||||
last = True
|
last = True
|
||||||
page = page[-limit:]
|
page = page[-limit:]
|
||||||
models.prefetch_events_references(page)
|
models.prefetch_events_references(page, prefetcher=self.prefetcher)
|
||||||
if page:
|
if page:
|
||||||
self.data = self.data.copy()
|
self.data = self.data.copy()
|
||||||
self.cleaned_data['after_cursor'] = self.data['after_cursor'] = page[0].cursor.minus_one()
|
self.cleaned_data['after_cursor'] = self.data['after_cursor'] = page[0].cursor.minus_one()
|
||||||
self.cleaned_data['before_cursor'] = ''
|
self.cleaned_data['before_cursor'] = ''
|
||||||
return Page(self, page, first, last)
|
return Page(self, page, first, last)
|
||||||
|
|
||||||
|
def prefetcher(self, model, pks):
|
||||||
|
return []
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def date_hierarchy(self):
|
def date_hierarchy(self):
|
||||||
self.is_valid()
|
self.is_valid()
|
||||||
|
|
|
@ -20,6 +20,7 @@ from collections import defaultdict
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
|
import django
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
|
@ -458,7 +459,7 @@ class EventCursor(str):
|
||||||
return EventCursor('%s %s' % (self.timestamp.timestamp(), self.event_id - 1))
|
return EventCursor('%s %s' % (self.timestamp.timestamp(), self.event_id - 1))
|
||||||
|
|
||||||
|
|
||||||
def prefetch_events_references(events):
|
def prefetch_events_references(events, prefetcher=None):
|
||||||
'''Prefetch references on an iterable of events, prevent N+1 queries problem.'''
|
'''Prefetch references on an iterable of events, prevent N+1 queries problem.'''
|
||||||
grouped_references = defaultdict(set)
|
grouped_references = defaultdict(set)
|
||||||
references = {}
|
references = {}
|
||||||
|
@ -473,6 +474,25 @@ def prefetch_events_references(events):
|
||||||
content_type = ContentType.objects.get_for_id(content_type_id)
|
content_type = ContentType.objects.get_for_id(content_type_id)
|
||||||
for instance in content_type.get_all_objects_for_this_type(pk__in=instance_pks):
|
for instance in content_type.get_all_objects_for_this_type(pk__in=instance_pks):
|
||||||
references[(content_type_id, instance.pk)] = instance
|
references[(content_type_id, instance.pk)] = instance
|
||||||
|
if prefetcher:
|
||||||
|
deleted_pks = [pk for pk in instance_pks if (content_type_id, pk) not in references]
|
||||||
|
if deleted_pks:
|
||||||
|
for found_pk, instance in prefetcher(content_type.model_class(), deleted_pks):
|
||||||
|
references[(content_type_id, found_pk)] = instance
|
||||||
|
|
||||||
|
# prefetch the user column if absent
|
||||||
|
if prefetcher:
|
||||||
|
user_to_events = {}
|
||||||
|
for event in events:
|
||||||
|
if event.user is None and event.user_id:
|
||||||
|
user_to_events.setdefault(event.user_id, []).append(event)
|
||||||
|
for found_pk, instance in prefetcher(User, user_to_events.keys()):
|
||||||
|
for event in user_to_events[found_pk]:
|
||||||
|
# prevent TypeError in user's field descriptor __set__ method
|
||||||
|
if django.VERSION < (2,):
|
||||||
|
event._user_cache = instance
|
||||||
|
else:
|
||||||
|
event._state.fields_cache['user'] = instance
|
||||||
|
|
||||||
# assign references to events
|
# assign references to events
|
||||||
for event in events:
|
for event in events:
|
||||||
|
|
|
@ -28,7 +28,13 @@ from authentic2.a2_rbac.models import OrganizationalUnit as OU
|
||||||
from authentic2.a2_rbac.utils import get_default_ou
|
from authentic2.a2_rbac.utils import get_default_ou
|
||||||
from authentic2.apps.journal.forms import JournalForm
|
from authentic2.apps.journal.forms import JournalForm
|
||||||
from authentic2.apps.journal.journal import Journal
|
from authentic2.apps.journal.journal import Journal
|
||||||
from authentic2.apps.journal.models import Event, EventType, EventTypeDefinition, clean_registry
|
from authentic2.apps.journal.models import (
|
||||||
|
Event,
|
||||||
|
EventType,
|
||||||
|
EventTypeDefinition,
|
||||||
|
clean_registry,
|
||||||
|
prefetch_events_references,
|
||||||
|
)
|
||||||
from authentic2.models import Service
|
from authentic2.models import Service
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
|
@ -146,6 +152,7 @@ def test_references(db):
|
||||||
assert list(event.get_typed_references(User, None)) == [None, None]
|
assert list(event.get_typed_references(User, None)) == [None, None]
|
||||||
event = Event.objects.get()
|
event = Event.objects.get()
|
||||||
assert list(event.get_typed_references(Service, User)) == [None, None]
|
assert list(event.get_typed_references(Service, User)) == [None, None]
|
||||||
|
assert event.user is None
|
||||||
|
|
||||||
|
|
||||||
def test_event_types(clean_event_types_definition_registry):
|
def test_event_types(clean_event_types_definition_registry):
|
||||||
|
@ -669,3 +676,32 @@ def test_statistics_ou_with_no_service(db, freezer):
|
||||||
ou_with_no_service = OU.objects.create(name='Second OU')
|
ou_with_no_service = OU.objects.create(name='Second OU')
|
||||||
stats = event_type_definition.get_method_statistics('month', services_ou=ou_with_no_service)
|
stats = event_type_definition.get_method_statistics('month', services_ou=ou_with_no_service)
|
||||||
assert stats == {'x_labels': [], 'series': []}
|
assert stats == {'x_labels': [], 'series': []}
|
||||||
|
|
||||||
|
|
||||||
|
def test_prefetcher(db):
|
||||||
|
event_type = EventType.objects.get_for_name('user.login')
|
||||||
|
for i in range(10):
|
||||||
|
user = User.objects.create()
|
||||||
|
Event.objects.create(type=event_type, user=user, references=[user])
|
||||||
|
Event.objects.create(type=event_type, user=user, references=[user])
|
||||||
|
|
||||||
|
User.objects.all().delete()
|
||||||
|
|
||||||
|
events = list(Event.objects.all())
|
||||||
|
prefetch_events_references(events)
|
||||||
|
for event in events:
|
||||||
|
assert event.user is None
|
||||||
|
assert list(event.get_typed_references(User)) == [None]
|
||||||
|
|
||||||
|
def prefetcher(model, pks):
|
||||||
|
if not issubclass(model, User):
|
||||||
|
return
|
||||||
|
for pk in pks:
|
||||||
|
yield pk, 'deleted %s' % pk
|
||||||
|
|
||||||
|
events = list(Event.objects.all())
|
||||||
|
prefetch_events_references(events, prefetcher=prefetcher)
|
||||||
|
for event in events:
|
||||||
|
s = 'deleted %s' % event.user_id
|
||||||
|
assert event.user == s
|
||||||
|
assert list(event.get_typed_references((str, User))) == [s]
|
||||||
|
|
Loading…
Reference in New Issue