misc: add cache duration option to named data sources (#26620)

tests/test_datasource.py

@@ -439,3 +439,37 @@ def test_data_source_signed(no_request_pub):
         assert len(data_sources.get_items({'type': 'foobar'})) == 1
         unsigned_url = urlopen.call_args[0][0]
     assert unsigned_url == 'https://no-secret.example.com/json'
+
+def test_named_datasource_json_cache():
+    NamedDataSource.wipe()
+    datasource = NamedDataSource(name='foobar')
+    datasource.data_source = {'type': 'json', 'value': 'http://whatever/'}
+    datasource.store()
+
+    with mock.patch('qommon.misc.urlopen') as urlopen:
+        urlopen.side_effect = lambda *args: StringIO(
+            json.dumps({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}))
+
+        assert data_sources.get_structured_items({'type': 'foobar'}) == [
+            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
+        assert urlopen.call_count == 1
+
+        get_request().datasources_cache = {}
+        assert data_sources.get_structured_items({'type': 'foobar'}) == [
+            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
+        assert urlopen.call_count == 2
+
+        datasource.cache_duration = '60'
+        datasource.store()
+
+        # will cache
+        get_request().datasources_cache = {}
+        assert data_sources.get_structured_items({'type': 'foobar'}) == [
+            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
+        assert urlopen.call_count == 3
+
+        # will get from cache
+        get_request().datasources_cache = {}
+        assert data_sources.get_structured_items({'type': 'foobar'}) == [
+            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
+        assert urlopen.call_count == 3

wcs/admin/data_sources.py

@@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext
 
 from qommon import _
 from qommon.form import *
+from qommon.humantime import seconds2humanduration
 from qommon.backoffice.menu import html_top
 from wcs.data_sources import (NamedDataSource, DataSourceSelectionWidget,
         get_structured_items)
@@ -43,6 +44,18 @@ class NamedDataSourceUI(object):
                 title=_('Data Source'),
                 allow_named_sources=False,
                 required=True)
+        form.add(DurationWidget, 'cache_duration',
+                value=self.datasource.cache_duration,
+                title=_('Cache Duration'),
+                hint=_('Caching data will improve performances but will keep changes '
+                       'from being visible immediately.  You should keep this duration '
+                       'reasonably short.'),
+                required=False,
+                advanced=False,
+                attrs={
+                    'data-dynamic-display-child-of': 'data_source$type',
+                    'data-dynamic-display-value': _('JSON URL'),
+                })
         if self.datasource.slug:
             form.add(StringWidget, 'slug',
                     value=self.datasource.slug,
@@ -74,6 +87,7 @@ class NamedDataSourceUI(object):
         self.datasource.name = name
         self.datasource.description = form.get_widget('description').parse()
         self.datasource.data_source = form.get_widget('data_source')
+        self.datasource.cache_duration = form.get_widget('cache_duration').parse()
         if self.datasource.slug:
             self.datasource.slug = slug
         self.datasource.store()
@@ -127,6 +141,10 @@ class NamedDataSourcePage(Directory):
                 r += htmltext('<li>%s<tt>%s</tt></li>') % (
                         _('Python Expression: '),
                     self.datasource.data_source.get('value'))
+            if self.datasource.cache_duration:
+                r += htmltext('<li>%s %s</li>') % (
+                        _('Cache Duration:'),
+                        seconds2humanduration(int(self.datasource.cache_duration)))
             r += htmltext('</ul>')
 
             if data_source_type in ('json', 'formula'):

wcs/data_sources.py

@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import collections
+import hashlib
 import urllib
 import urlparse
 import xml.etree.ElementTree as ET
@@ -108,7 +109,14 @@ def get_items(data_source, include_disabled=False):
 
 
 def get_structured_items(data_source):
-    data_source = get_real(data_source)
+    cache_duration = 0
+    if data_source.get('type') not in ('json', 'jsonp', 'formula'):
+        # named data source
+        named_data_source = NamedDataSource.get_by_slug(data_source['type'])
+        if named_data_source.cache_duration:
+            cache_duration = int(named_data_source.cache_duration)
+        data_source = named_data_source.data_source
+
     if data_source.get('type') == 'formula':
         # the result of a python expression, it must be a list.
         # - of strings
@@ -161,6 +169,13 @@ def get_structured_items(data_source):
         if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
             return request.datasources_cache[url]
 
+        if cache_duration:
+            cache_key = 'data-source-%s' % hashlib.md5(url).hexdigest()
+            from django.core.cache import cache
+            items = cache.get(cache_key)
+            if items is not None:
+                return items
+
         try:
             signature_key, orig = get_secret_and_orig(url)
         except MissingSecret:
@@ -188,6 +203,10 @@ def get_structured_items(data_source):
                 items.append(item)
             if hasattr(request, 'datasources_cache'):
                 request.datasources_cache[url] = items
+
+            if cache_duration:
+                cache.set(cache_key, items, cache_duration)
+
             return items
         except qommon.misc.ConnectionError as e:
             get_logger().warn('Error loading JSON data source (%s)' % str(e))
@@ -213,10 +232,13 @@ class NamedDataSource(XmlStorableObject):
     slug = None
     description = None
     data_source = None
+    cache_duration = None
 
     # declarations for serialization
     XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'),
-            ('data_source', 'data_source')]
+            ('cache_duration', 'str'),
+            ('data_source', 'data_source'),
+            ]
 
     def __init__(self, name=None):
         StorableObject.__init__(self)

wcs/qommon/form.py

@@ -70,6 +70,7 @@ from wcs.conditions import Condition, ValidationError
 
 from qommon import _, ngettext
 import misc
+from .humantime import humanduration2seconds, seconds2humanduration, timewords
 from .misc import strftime, C_
 from publisher import get_cfg
 from .template_utils import render_block_to_string
@@ -480,6 +481,23 @@ class StringWidget(quixote.form.StringWidget):
                 self.error = str(e)
 
 
+class DurationWidget(StringWidget):
+    def __init__(self, name, value=None, **kwargs):
+        if value:
+            value = seconds2humanduration(int(value))
+        if 'hint' in kwargs:
+            kwargs['hint'] += htmltext('<br>')
+        else:
+            kwargs['hint'] = ''
+        kwargs['hint'] += htmltext(
+                _('Usable units of time: %s.')) % ', '.join(timewords())
+        super(DurationWidget, self).__init__(name, value=value, **kwargs)
+
+    def parse(self, request=None):
+        value = super(DurationWidget, self).parse(request)
+        return str(humanduration2seconds(self.value)) if value else None
+
+
 class TextWidget(quixote.form.TextWidget):
     def __init__(self, name, *args, **kwargs):
         self.validation_function = kwargs.pop('validation_function', None)