entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 34 Releases Activity

misc: keep data source calls in request cache (#26612)

This commit is contained in:
Frédéric Péters 2018-09-21 12:39:20 +02:00
parent 2670e3bf25
commit 4506208e56
3 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tests/test_datasource.py
View File

@ -117,39 +117,46 @@ def test_python_datasource_with_evalutils():
('foo', 'Foo', 'foo', {'id': 'foo', 'text': 'Foo', 'value': '2017-01-01'})]
def test_json_datasource(http_requests):
get_request().datasources_cache = {}
datasource = {'type': 'json', 'value': ''}
assert data_sources.get_items(datasource) == []
# missing file
get_request().datasources_cache = {}
json_file_path = os.path.join(pub.app_dir, 'test.json')
datasource = {'type': 'json', 'value': 'file://%s' % json_file_path}
assert data_sources.get_items(datasource) == []
# invalid json file
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json_file.write(u'foobar'.encode('zlib_codec'))
json_file.close()
assert data_sources.get_items(datasource) == []
# empty json file
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({}, json_file)
json_file.close()
assert data_sources.get_items(datasource) == []
# unrelated json file
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump('foobar', json_file)
json_file.close()
assert data_sources.get_items(datasource) == []
# another unrelated json file
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': 'foobar'}, json_file)
json_file.close()
assert data_sources.get_items(datasource) == []
# a good json file
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file)
json_file.close()
@ -160,6 +167,7 @@ def test_json_datasource(http_requests):
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
# a json file with additional keys
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': '1', 'text': 'foo', 'more': 'xxx'},
{'id': '2', 'text': 'bar', 'more': 'yyy'}]}, json_file)
@ -172,6 +180,7 @@ def test_json_datasource(http_requests):
{'id': '2', 'text': 'bar', 'more': 'yyy'}]
# json specified with a variadic url
get_request().datasources_cache = {}
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
@ -182,6 +191,7 @@ def test_json_datasource(http_requests):
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# same with django templated url
get_request().datasources_cache = {}
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
@ -192,6 +202,7 @@ def test_json_datasource(http_requests):
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# json specified with a variadic url with an erroneous space
get_request().datasources_cache = {}
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
@ -202,6 +213,7 @@ def test_json_datasource(http_requests):
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# same with django templated url
get_request().datasources_cache = {}
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
@ -212,6 +224,7 @@ def test_json_datasource(http_requests):
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# a json file with integer as 'id'
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]}, json_file)
json_file.close()
@ -222,6 +235,7 @@ def test_json_datasource(http_requests):
{'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]
# a json file with empty or no text values
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': '1', 'text': ''}, {'id': '2'}]}, json_file)
json_file.close()
@ -233,6 +247,7 @@ def test_json_datasource(http_requests):
{'id': '2', 'text': '2'}]
# a json file with empty or no id
get_request().datasources_cache = {}
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': '', 'text': 'foo'}, {'text': 'bar'}, {'id': None}]}, json_file)
json_file.close()

9
wcs/data_sources.py
View File

@ -19,7 +19,7 @@ import urllib
import urlparse
import xml.etree.ElementTree as ET
from quixote import get_publisher
from quixote import get_publisher, get_request
from quixote.html import TemplateIO
from qommon import _
@ -156,6 +156,11 @@ def get_structured_items(data_source):
if Template.is_template_string(url):
vars = get_publisher().substitutions.get_context_variables()
url = get_variadic_url(url, vars)
request = get_request()
if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
return request.datasources_cache[url]
try:
signature_key, orig = get_secret_and_orig(url)
except MissingSecret:
@ -181,6 +186,8 @@ def get_structured_items(data_source):
if 'text' not in item:
item['text'] = item['id']
items.append(item)
if hasattr(request, 'datasources_cache'):
request.datasources_cache[url] = items
return items
except qommon.misc.ConnectionError as e:
get_logger().warn('Error loading JSON data source (%s)' % str(e))

1
wcs/qommon/http_request.py
View File

@ -37,6 +37,7 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
self.is_json_marker = None
self.ignore_session = False
self.wscalls_cache = {}
self.datasources_cache = {}
# keep a copy of environment to make sure it's not reused along
# uwsgi/gunicorn processes.
self.environ = copy.copy(self.environ)