From 452e671c7b4577c2020e6fd9930a72b0d5464564 Mon Sep 17 00:00:00 2001
From: Valentin Deniaud <vdeniaud@entrouvert.com>
Date: Wed, 17 Feb 2021 10:39:02 +0100
Subject: [PATCH] dataviz: move statistic list update code (#50891)

---
 combo/apps/dataviz/__init__.py | 52 ++--------------------------------
 combo/apps/dataviz/utils.py    | 51 +++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 49 deletions(-)
 create mode 100644 combo/apps/dataviz/utils.py

diff --git a/combo/apps/dataviz/__init__.py b/combo/apps/dataviz/__init__.py
index ef39f642..eaebe438 100644
--- a/combo/apps/dataviz/__init__.py
+++ b/combo/apps/dataviz/__init__.py
@@ -15,13 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import django.apps
-from django.core import checks
-from django.conf import settings
-from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
-from combo.utils import requests
-
 
 class AppConfig(django.apps.AppConfig):
     name = 'combo.apps.dataviz'
@@ -33,51 +28,10 @@ class AppConfig(django.apps.AppConfig):
         return urls.urlpatterns
 
     def hourly(self):
-        self.update_available_statistics()
-
-    def update_available_statistics(self):
-        from .models import Statistic, ChartNgCell
-
-        if not settings.KNOWN_SERVICES:
-            return
-
-        start_update = timezone.now()
-        for provider in settings.STATISTICS_PROVIDERS:
-            if isinstance(provider, dict):
-                url = provider['url']
-                sites = {provider['id']: {'title': provider['name']}}
-                provider = provider['id']
-            else:
-                sites = settings.KNOWN_SERVICES.get(provider, {})
-                url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
-
-            for site_key, site_dict in sites.items():
-                site_title = site_dict.pop('title', '')
-                response = requests.get(
-                    url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'}
-                )
-                if response.status_code != 200:
-                    continue
-
-                result = response.json()
-                if isinstance(result, dict):
-                    result = result['data']  # detect new api
-
-                for stat in result:
-                    Statistic.objects.update_or_create(
-                        slug=stat.get('slug') or stat['id'],
-                        site_slug=site_key,
-                        service_slug=provider,
-                        defaults={
-                            'label': stat['name'],
-                            'url': stat.get('data-url') or stat['url'],
-                            'site_title': site_title,
-                            'filters': stat.get('filters', []),
-                            'available': True,
-                        },
-                    )
-        Statistic.objects.filter(last_update__lt=start_update).update(available=False)
+        from .models import ChartNgCell
+        from .utils import update_available_statistics
 
+        update_available_statistics()
         for cell in ChartNgCell.objects.all():
             cell.check_validity()
 
diff --git a/combo/apps/dataviz/utils.py b/combo/apps/dataviz/utils.py
new file mode 100644
index 00000000..c8f6981f
--- /dev/null
+++ b/combo/apps/dataviz/utils.py
@@ -0,0 +1,51 @@
+from django.conf import settings
+from django.utils import timezone
+
+from combo.utils import requests
+
+from .models import Statistic
+
+
+def update_available_statistics():
+    if not settings.KNOWN_SERVICES:
+        return
+
+    start_update = timezone.now()
+    for provider in settings.STATISTICS_PROVIDERS:
+        if isinstance(provider, dict):
+            url = provider['url']
+            sites = {provider['id']: {'title': provider['name']}}
+            provider = provider['id']
+        else:
+            sites = settings.KNOWN_SERVICES.get(provider, {})
+            url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
+
+        for site_key, site_dict in sites.items():
+            response = requests.get(
+                url,
+                timeout=5,
+                remote_service=site_dict if provider in settings.KNOWN_SERVICES else {},
+                without_user=True,
+                headers={'accept': 'application/json'},
+            )
+            if response.status_code != 200:
+                continue
+
+            result = response.json()
+            if isinstance(result, dict):
+                result = result['data']  # detect new api
+
+            for stat in result:
+                Statistic.objects.update_or_create(
+                    slug=stat.get('slug') or stat['id'],
+                    site_slug=site_key,
+                    service_slug=provider,
+                    defaults={
+                        'label': stat['name'],
+                        'url': stat.get('data-url') or stat['url'],
+                        'site_title': site_dict.get('title', ''),
+                        'filters': stat.get('filters', []),
+                        'available': True,
+                    },
+                )
+    Statistic.objects.filter(last_update__lt=start_update).update(available=False)