diff --git a/combo/apps/dataviz/utils.py b/combo/apps/dataviz/utils.py index c8f6981f..90f3e7fa 100644 --- a/combo/apps/dataviz/utils.py +++ b/combo/apps/dataviz/utils.py @@ -1,3 +1,4 @@ +import django from django.conf import settings from django.utils import timezone @@ -10,6 +11,7 @@ def update_available_statistics(): if not settings.KNOWN_SERVICES: return + results = [] start_update = timezone.now() for provider in settings.STATISTICS_PROVIDERS: if isinstance(provider, dict): @@ -36,16 +38,46 @@ def update_available_statistics(): result = result['data'] # detect new api for stat in result: - Statistic.objects.update_or_create( - slug=stat.get('slug') or stat['id'], - site_slug=site_key, - service_slug=provider, - defaults={ - 'label': stat['name'], - 'url': stat.get('data-url') or stat['url'], - 'site_title': site_dict.get('title', ''), - 'filters': stat.get('filters', []), - 'available': True, - }, + results.append( + Statistic( + slug=stat.get('slug') or stat['id'], + site_slug=site_key, + service_slug=provider, + label=stat['name'], + url=stat.get('data-url') or stat['url'], + site_title=site_dict.get('title', ''), + filters=stat.get('filters', []), + available=True, + ) ) - Statistic.objects.filter(last_update__lt=start_update).update(available=False) + + update_fields = ('label', 'url', 'site_title', 'filters', 'available') + all_statistics = {stat.natural_key(): stat for stat in Statistic.objects.all()} + statistics_to_create = [] + statistics_to_update = {} + for stat in results: + existing_stat = all_statistics.get(stat.natural_key()) + if existing_stat: + for field in update_fields: + new_value = getattr(stat, field) + if getattr(existing_stat, field) != new_value: + setattr(existing_stat, field, new_value) + statistics_to_update[existing_stat.pk] = existing_stat + else: + statistics_to_create.append(stat) + + Statistic.objects.bulk_create(statistics_to_create) + if django.VERSION < (2, 2, 0): + for statistic in statistics_to_update.values(): + Statistic.objects.filter(pk=statistic.pk).update( + **{field: getattr(statistic, field) for field in update_fields} + ) + else: + Statistic.objects.bulk_update(statistics_to_update.values(), update_fields) + + available_stats = Statistic.objects.filter(available=True) + for stat in results: + available_stats = available_stats.exclude( + slug=stat.slug, site_slug=stat.site_slug, service_slug=stat.service_slug + ) + available_stats.update(available=False)