dataviz: include ChartNgCell in invalidity report (#49720)

combo/apps/dataviz/__init__.py

@@ -35,7 +35,7 @@ class AppConfig(django.apps.AppConfig):
         self.update_available_statistics()
 
     def update_available_statistics(self):
-        from .models import Statistic
+        from .models import Statistic, ChartNgCell
         if not settings.KNOWN_SERVICES:
             return
 
@@ -74,5 +74,8 @@ class AppConfig(django.apps.AppConfig):
                     )
         Statistic.objects.filter(last_update__lt=start_update).update(available=False)
 
+        for cell in ChartNgCell.objects.all():
+            cell.check_validity()
+
 
 default_app_config = 'combo.apps.dataviz.AppConfig'

combo/apps/dataviz/models.py

@@ -17,6 +17,7 @@
 import copy
 import os
 from datetime import date
+from requests.exceptions import RequestException
 
 from django.urls import reverse
 from django.db import models
@@ -215,6 +216,20 @@ class ChartNgCell(CellBase):
     def is_relevant(self, context):
         return bool(self.statistic)
 
+    def check_validity(self):
+        if not self.statistic:
+            return
+
+        resp = None
+        try:
+            resp = self.get_statistic_data()
+        except RequestException:
+            pass
+
+        self.set_validity_from_url(
+            resp, not_found_code='statistic_data_not_found', invalid_code='statistic_url_invalid'
+        )
+
     def get_cell_extra_context(self, context):
         ctx = super(ChartNgCell, self).get_cell_extra_context(context)
         if self.chart_type == 'table' and self.statistic:
@@ -236,16 +251,19 @@ class ChartNgCell(CellBase):
                     ctx['table'] = ctx['table'].replace('<table>', '<table class="main">')
         return ctx
 
-    def get_chart(self, width=None, height=None, raise_if_not_cached=False):
-        response = requests.get(
-                self.statistic.url,
-                params=self.get_filter_params(),
-                cache_duration=300,
-                remote_service='auto',
-                without_user=True,
-                raise_if_not_cached=raise_if_not_cached,
-                log_errors=False,
+    def get_statistic_data(self, raise_if_not_cached=False):
+        return requests.get(
+            self.statistic.url,
+            params=self.get_filter_params(),
+            cache_duration=300,
+            remote_service='auto',
+            without_user=True,
+            raise_if_not_cached=raise_if_not_cached,
+            log_errors=False,
         )
+
+    def get_chart(self, width=None, height=None, raise_if_not_cached=False):
+        response = self.get_statistic_data(raise_if_not_cached)
         response.raise_for_status()
         response = response.json()
 

tests/test_dataviz.py

@@ -13,7 +13,7 @@ from django.db.migrations.executor import MigrationExecutor
 from django.test import override_settings
 from django.utils import timezone
 
-from combo.data.models import Page
+from combo.data.models import Page, ValidityInfo
 from combo.apps.dataviz.models import Gauge, ChartNgCell, UnsupportedDataSet, Statistic
 
 from .test_public import login, normal_user
@@ -1267,3 +1267,26 @@ def test_chartng_cell_new_api_filter_params(new_api_statistics, nocache, freezer
     chart = cell.get_chart()
     request = new_api_mock.call['requests'][5]
     assert 'start=2020-10-01' in request.url and 'end=2020-11-03' in request.url
+
+
+def test_dataviz_check_validity(nocache):
+    page = Page.objects.create(title='One', slug='index')
+    stat = Statistic.objects.create(url='https://stat.com/stats/1/')
+    cell = ChartNgCell.objects.create(page=page, order=1, placeholder='content', statistic=stat)
+
+    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/1/')
+    def url_mock(url, request):
+        return {'content': json.dumps({'data': [], 'err': 0}), 'status_code': 200}
+
+    with HTTMock(url_mock):
+        cell.check_validity()
+    assert ValidityInfo.objects.exists() is False
+
+    @urlmatch(scheme='https', netloc=r'stat.com', path='/stats/1/')
+    def url_mock(url, request):
+        return {'content': json.dumps({'data': [], 'err': 1}), 'status_code': 404}
+
+    with HTTMock(url_mock):
+        cell.check_validity()
+    validity_info = ValidityInfo.objects.latest('pk')
+    assert validity_info.invalid_reason_code == 'statistic_data_not_found'