dataviz: improve queries on statistics list update (#50891)

combo/apps/dataviz/utils.py

@@ -1,3 +1,4 @@
+import django
 from django.conf import settings
 from django.utils import timezone
 
@@ -10,6 +11,7 @@ def update_available_statistics():
     if not settings.KNOWN_SERVICES:
         return
 
+    results = []
     start_update = timezone.now()
     for provider in settings.STATISTICS_PROVIDERS:
         if isinstance(provider, dict):
@@ -36,16 +38,46 @@ def update_available_statistics():
                 result = result['data']  # detect new api
 
             for stat in result:
-                Statistic.objects.update_or_create(
+                results.append(
-                    slug=stat.get('slug') or stat['id'],
+                    Statistic(
-                    site_slug=site_key,
+                        slug=stat.get('slug') or stat['id'],
-                    service_slug=provider,
+                        site_slug=site_key,
-                    defaults={
+                        service_slug=provider,
-                        'label': stat['name'],
+                        label=stat['name'],
-                        'url': stat.get('data-url') or stat['url'],
+                        url=stat.get('data-url') or stat['url'],
-                        'site_title': site_dict.get('title', ''),
+                        site_title=site_dict.get('title', ''),
-                        'filters': stat.get('filters', []),
+                        filters=stat.get('filters', []),
-                        'available': True,
+                        available=True,
-                    },
+                    )
                 )
-    Statistic.objects.filter(last_update__lt=start_update).update(available=False)
+
+    update_fields = ('label', 'url', 'site_title', 'filters', 'available')
+    all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()}
+    statistics_to_create = []
+    statistics_to_update = {}
+    for stat in results:
+        existing_stat = all_statistics.get(stat.natural_key())
+        if existing_stat:
+            for field in update_fields:
+                new_value = getattr(stat, field)
+                if getattr(existing_stat, field) != new_value:
+                    setattr(existing_stat, field, new_value)
+                    statistics_to_update[existing_stat.pk] = existing_stat
+        else:
+            statistics_to_create.append(stat)
+
+    Statistic.objects.bulk_create(statistics_to_create)
+    if django.VERSION < (2, 2, 0):
+        for statistic in statistics_to_update.values():
+            Statistic.objects.filter(pk=statistic.pk).update(
+                **{field: getattr(statistic, field) for field in update_fields}
+            )
+    else:
+        Statistic.objects.bulk_update(statistics_to_update.values(), update_fields)
+
+    available_stats = Statistic.objects.filter(available=True)
+    for stat in results:
+        available_stats = available_stats.exclude(
+            slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug
+        )
+    available_stats.update(available=False)