dataviz: improve queries on statistics list update (#50891)
This commit is contained in:
parent
452e671c7b
commit
38ba201a5f
|
@ -1,3 +1,4 @@
|
|||
import django
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
|
@ -10,6 +11,7 @@ def update_available_statistics():
|
|||
if not settings.KNOWN_SERVICES:
|
||||
return
|
||||
|
||||
results = []
|
||||
start_update = timezone.now()
|
||||
for provider in settings.STATISTICS_PROVIDERS:
|
||||
if isinstance(provider, dict):
|
||||
|
@ -36,16 +38,46 @@ def update_available_statistics():
|
|||
result = result['data'] # detect new api
|
||||
|
||||
for stat in result:
|
||||
Statistic.objects.update_or_create(
|
||||
slug=stat.get('slug') or stat['id'],
|
||||
site_slug=site_key,
|
||||
service_slug=provider,
|
||||
defaults={
|
||||
'label': stat['name'],
|
||||
'url': stat.get('data-url') or stat['url'],
|
||||
'site_title': site_dict.get('title', ''),
|
||||
'filters': stat.get('filters', []),
|
||||
'available': True,
|
||||
},
|
||||
results.append(
|
||||
Statistic(
|
||||
slug=stat.get('slug') or stat['id'],
|
||||
site_slug=site_key,
|
||||
service_slug=provider,
|
||||
label=stat['name'],
|
||||
url=stat.get('data-url') or stat['url'],
|
||||
site_title=site_dict.get('title', ''),
|
||||
filters=stat.get('filters', []),
|
||||
available=True,
|
||||
)
|
||||
)
|
||||
Statistic.objects.filter(last_update__lt=start_update).update(available=False)
|
||||
|
||||
update_fields = ('label', 'url', 'site_title', 'filters', 'available')
|
||||
all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
|
||||
statistics_to_create = []
|
||||
statistics_to_update = {}
|
||||
for stat in results:
|
||||
existing_stat = all_statistics.get(stat.natural_key())
|
||||
if existing_stat:
|
||||
for field in update_fields:
|
||||
new_value = getattr(stat, field)
|
||||
if getattr(existing_stat, field) != new_value:
|
||||
setattr(existing_stat, field, new_value)
|
||||
statistics_to_update[existing_stat.pk] = existing_stat
|
||||
else:
|
||||
statistics_to_create.append(stat)
|
||||
|
||||
Statistic.objects.bulk_create(statistics_to_create)
|
||||
if django.VERSION < (2, 2, 0):
|
||||
for statistic in statistics_to_update.values():
|
||||
Statistic.objects.filter(pk=statistic.pk).update(
|
||||
**{field: getattr(statistic, field) for field in update_fields}
|
||||
)
|
||||
else:
|
||||
Statistic.objects.bulk_update(statistics_to_update.values(), update_fields)
|
||||
|
||||
available_stats = Statistic.objects.filter(available=True)
|
||||
for stat in results:
|
||||
available_stats = available_stats.exclude(
|
||||
slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug
|
||||
)
|
||||
available_stats.update(available=False)
|
||||
|
|
Loading…
Reference in New Issue