dataviz: get statistics from arbitrary urls (#49247)

combo/apps/dataviz/__init__.py

@@ -41,27 +41,33 @@ class AppConfig(django.apps.AppConfig):
 
         statistics_providers = settings.STATISTICS_PROVIDERS + ['bijoe']
         start_update = timezone.now()
-        for service in statistics_providers:
-            sites = settings.KNOWN_SERVICES.get(service, {}).items()
-            for site_key, site_dict in sites:
-                if service == 'bijoe':
-                    result = requests.get('/visualization/json/',
-                            remote_service=site_dict, without_user=True,
-                            headers={'accept': 'application/json'}).json()
-                else:
-                    result = requests.get('/api/statistics/',
-                            remote_service=site_dict, without_user=True,
-                            headers={'accept': 'application/json'}).json()['data']
+        for provider in statistics_providers:
+            if isinstance(provider, dict):
+                url = provider['url']
+                sites = {provider['id']: {'title': provider['name']}}
+                provider = provider['id']
+            else:
+                sites = settings.KNOWN_SERVICES.get(provider, {})
+                url = '/visualization/json/' if provider == 'bijoe' else '/api/statistics/'
+
+            for site_key, site_dict in sites.items():
+                site_title = site_dict.pop('title', '')
+                result = requests.get(
+                    url, remote_service=site_dict, without_user=True, headers={'accept': 'application/json'}
+                ).json()
+
+                if isinstance(result, dict):
+                    result = result['data']  # detect new api
 
                 for stat in result:
                     Statistic.objects.update_or_create(
                         slug=stat.get('slug') or stat['id'],
                         site_slug=site_key,
-                        service_slug=service,
+                        service_slug=provider,
                         defaults={
                             'label': stat['name'],
                             'url': stat.get('data-url') or stat['url'],
-                            'site_title': site_dict.get('title', ''),
+                            'site_title': site_title,
                             'filters': stat.get('filters', []),
                             'available': True,
                         }

tests/test_dataviz.py

@@ -3,7 +3,7 @@ import json
 import mock
 import pytest
 from datetime import timedelta
-from httmock import HTTMock, with_httmock, remember_called
+from httmock import HTTMock, with_httmock, remember_called, urlmatch
 from requests.exceptions import HTTPError
 
 from django.apps import apps
@@ -1163,7 +1163,7 @@ def test_dataviz_cell_migration(settings):
 
 
 @with_httmock(new_api_mock)
-def test_dataviz_api_list_statistics(new_api_statistics):
+def test_dataviz_api_list_statistics(new_api_statistics, settings):
     statistic = Statistic.objects.get(slug='one-serie')
     assert statistic.label == 'One serie stat'
     assert statistic.site_slug == 'connection'
@@ -1172,6 +1172,30 @@ def test_dataviz_api_list_statistics(new_api_statistics):
     assert statistic.url == 'https://authentic.example.com/api/statistics/one-serie/'
     assert statistic.available
 
+    # try with external url
+    statistics_count = Statistic.objects.count()
+    settings.STATISTICS_PROVIDERS.append(
+        {'url': 'https://stat.com/stats/', 'id': 'example', 'name': 'Example Provider'}
+    )
+    catalog = {'data': [{'url': 'https://stat.com/stats/1/', 'name': 'Test', 'id': 'test'}]}
+
+    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/')
+    def url_mock(url, request):
+        return {'content': json.dumps(catalog), 'status_code': 200}
+
+    appconfig = apps.get_app_config('dataviz')
+    with HTTMock(url_mock):
+        appconfig.hourly()
+
+    assert Statistic.objects.count() == statistics_count + 1
+    statistic = Statistic.objects.get(slug='test')
+    assert statistic.label == 'Test'
+    assert statistic.site_slug == 'example'
+    assert statistic.service_slug == 'example'
+    assert statistic.site_title == 'Example Provider'
+    assert statistic.url == 'https://stat.com/stats/1/'
+    assert statistic.available
+
 
 @with_httmock(new_api_mock)
 def test_chartng_cell_new_api_filter_params(new_api_statistics, nocache):