datasources: json datasource attributes (#47218)
This commit is contained in:
parent
f0890d999c
commit
0cb5ec3cbc
|
@ -283,6 +283,46 @@ def test_json_datasource(requests_pub, http_requests):
|
|||
assert data_sources.get_items(datasource) == []
|
||||
assert data_sources.get_structured_items(datasource) == []
|
||||
|
||||
# specify data_attribute
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'results'}
|
||||
get_request().datasources_cache = {}
|
||||
json_file = open(json_file_path, 'w')
|
||||
json.dump({'results': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file)
|
||||
json_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
||||
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'data'}
|
||||
get_request().datasources_cache = {}
|
||||
assert data_sources.get_structured_items(datasource) == []
|
||||
|
||||
# specify id_attribute
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'pk'}
|
||||
get_request().datasources_cache = {}
|
||||
json_file = open(json_file_path, 'w')
|
||||
json.dump({'data': [{'pk': '1', 'text': 'foo'}, {'pk': '2', 'text': 'bar'}]}, json_file)
|
||||
json_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo', 'pk': '1'}, {'id': '2', 'text': 'bar', 'pk': '2'}]
|
||||
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'id'}
|
||||
get_request().datasources_cache = {}
|
||||
assert data_sources.get_structured_items(datasource) == []
|
||||
|
||||
# specify text_attribute
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'label'}
|
||||
get_request().datasources_cache = {}
|
||||
json_file = open(json_file_path, 'w')
|
||||
json.dump({'data': [{'id': '1', 'label': 'foo'}, {'id': '2', 'label': 'bar'}]}, json_file)
|
||||
json_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo', 'label': 'foo'}, {'id': '2', 'text': 'bar', 'label': 'bar'}]
|
||||
|
||||
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'text'}
|
||||
get_request().datasources_cache = {}
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': '1', 'label': 'foo'}, {'id': '2', 'text': '2', 'label': 'bar'}]
|
||||
|
||||
|
||||
def test_json_datasource_bad_url(http_requests, caplog):
|
||||
datasource = {'type': 'json', 'value': 'http://remote.example.net/404'}
|
||||
|
|
|
@ -161,7 +161,24 @@ def test_data_sources_view(pub):
|
|||
assert 'Preview' in resp.text
|
||||
assert 'foo' in resp.text
|
||||
|
||||
# with other attributes
|
||||
json_file = open(json_file_path, 'w')
|
||||
json.dump({'results': [{'pk': '1', 'label': 'foo'}, {'pk': '2'}]}, json_file)
|
||||
json_file.close()
|
||||
|
||||
data_source.data_attribute = 'results'
|
||||
data_source.id_attribute = 'pk'
|
||||
data_source.text_attribute = 'label'
|
||||
data_source.store()
|
||||
with HttpRequestsMocking():
|
||||
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
|
||||
assert 'Preview' in resp.text
|
||||
assert '<tt>1</tt>: foo</li>' in resp.text
|
||||
assert '<tt>2</tt>: 2</li>' in resp.text
|
||||
assert '<p>Additional keys are available: label, pk</p>' in resp.text
|
||||
|
||||
# variadic url
|
||||
data_source.data_attribute = None
|
||||
data_source.data_source = {'type': 'json', 'value': '{{ site_url }}/foo/bar'}
|
||||
data_source.store()
|
||||
with HttpRequestsMocking():
|
||||
|
|
|
@ -104,6 +104,36 @@ class NamedDataSourceUI(object):
|
|||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'geojson',
|
||||
})
|
||||
form.add(StringWidget, 'data_attribute',
|
||||
value=self.datasource.data_attribute,
|
||||
title=_('Data Attribute'),
|
||||
hint=_('Name of the attribute containing the list of results (default: data)'),
|
||||
required=False,
|
||||
advanced=False,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'json',
|
||||
})
|
||||
form.add(StringWidget, 'id_attribute',
|
||||
value=self.datasource.id_attribute,
|
||||
title=_('Id Attribute'),
|
||||
hint=_('Name of the attribute containing the identifier of an entry (default: id)'),
|
||||
required=False,
|
||||
advanced=False,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'json',
|
||||
})
|
||||
form.add(StringWidget, 'text_attribute',
|
||||
value=self.datasource.text_attribute,
|
||||
title=_('Text Attribute'),
|
||||
hint=_('Name of the attribute containing the label of an entry (default: text)'),
|
||||
required=False,
|
||||
advanced=False,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'json',
|
||||
})
|
||||
form.add(StringWidget, 'label_template_property',
|
||||
value=self.datasource.label_template_property,
|
||||
title=_('Label template'),
|
||||
|
@ -148,6 +178,9 @@ class NamedDataSourceUI(object):
|
|||
self.datasource.cache_duration = form.get_widget('cache_duration').parse()
|
||||
self.datasource.query_parameter = form.get_widget('query_parameter').parse()
|
||||
self.datasource.id_parameter = form.get_widget('id_parameter').parse()
|
||||
self.datasource.data_attribute = form.get_widget('data_attribute').parse()
|
||||
self.datasource.id_attribute = form.get_widget('id_attribute').parse()
|
||||
self.datasource.text_attribute = form.get_widget('text_attribute').parse()
|
||||
self.datasource.id_property = form.get_widget('id_property').parse()
|
||||
self.datasource.label_template_property = form.get_widget('label_template_property').parse()
|
||||
if slug_widget:
|
||||
|
|
|
@ -118,54 +118,69 @@ def get_items(data_source, include_disabled=False, mode=None):
|
|||
|
||||
def request_json_items(url, data_source):
|
||||
url = sign_url_auto_orig(url)
|
||||
geojson = data_source.get('type') == 'geojson'
|
||||
data_key = data_source.get('data_attribute') or 'data'
|
||||
try:
|
||||
entries = misc.json_loads(misc.urlopen(url).read())
|
||||
if not isinstance(entries, dict):
|
||||
raise ValueError('not a json dict')
|
||||
if entries.get('err') not in (None, 0, "0"):
|
||||
raise ValueError('err %s' % entries['err'])
|
||||
if not geojson and not isinstance(entries.get('data'), list):
|
||||
raise ValueError('not a json dict with a data list attribute')
|
||||
if geojson and not isinstance(entries.get('features'), list):
|
||||
raise ValueError('bad geojson format')
|
||||
if not isinstance(entries.get(data_key), list):
|
||||
raise ValueError('not a json dict with a %s list attribute' % data_key)
|
||||
except misc.ConnectionError as e:
|
||||
if geojson:
|
||||
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
|
||||
else:
|
||||
get_logger().warning('Error loading JSON data source (%s)' % str(e))
|
||||
get_logger().warning('Error loading JSON data source (%s)' % str(e))
|
||||
return None
|
||||
except (ValueError, TypeError) as e:
|
||||
if geojson:
|
||||
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
|
||||
else:
|
||||
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
|
||||
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
|
||||
return None
|
||||
items = []
|
||||
if geojson:
|
||||
id_property = data_source.get('id_property') or 'id'
|
||||
for item in entries.get('features'):
|
||||
if not item.get('properties', {}).get(id_property):
|
||||
continue
|
||||
item['id'] = item['properties'][id_property]
|
||||
try:
|
||||
item['text'] = Template(
|
||||
data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
|
||||
except (TemplateSyntaxError, VariableDoesNotExist):
|
||||
pass
|
||||
if not item.get('text'):
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
else:
|
||||
for item in entries.get('data'):
|
||||
# skip malformed items
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get('id') is None or item.get('id') == '':
|
||||
continue
|
||||
if 'text' not in item:
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
id_attribute = data_source.get('id_attribute') or 'id'
|
||||
text_attribute = data_source.get('text_attribute') or 'text'
|
||||
for item in entries.get(data_key):
|
||||
# skip malformed items
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get(id_attribute) is None or item.get(id_attribute) == '':
|
||||
continue
|
||||
item['id'] = item[id_attribute]
|
||||
if text_attribute not in item:
|
||||
item['text'] = item['id']
|
||||
else:
|
||||
item['text'] = item[text_attribute]
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def request_geojson_items(url, data_source):
|
||||
url = sign_url_auto_orig(url)
|
||||
try:
|
||||
entries = misc.json_loads(misc.urlopen(url).read())
|
||||
if not isinstance(entries, dict):
|
||||
raise ValueError('not a json dict')
|
||||
if entries.get('err') not in (None, 0, "0"):
|
||||
raise ValueError('err %s' % entries['err'])
|
||||
if not isinstance(entries.get('features'), list):
|
||||
raise ValueError('bad geojson format')
|
||||
except misc.ConnectionError as e:
|
||||
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
|
||||
return None
|
||||
except (ValueError, TypeError) as e:
|
||||
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
|
||||
return None
|
||||
items = []
|
||||
id_property = data_source.get('id_property') or 'id'
|
||||
for item in entries.get('features'):
|
||||
if not item.get('properties', {}).get(id_property):
|
||||
continue
|
||||
item['id'] = item['properties'][id_property]
|
||||
try:
|
||||
item['text'] = Template(
|
||||
data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
|
||||
except (TemplateSyntaxError, VariableDoesNotExist):
|
||||
pass
|
||||
if not item.get('text'):
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
|
@ -247,7 +262,10 @@ def get_structured_items(data_source, mode=None):
|
|||
if items is not None:
|
||||
return items
|
||||
|
||||
items = request_json_items(url, data_source)
|
||||
if geojson:
|
||||
items = request_geojson_items(url, data_source)
|
||||
else:
|
||||
items = request_json_items(url, data_source)
|
||||
if items is None:
|
||||
return []
|
||||
if hasattr(request, 'datasources_cache'):
|
||||
|
@ -298,6 +316,9 @@ class NamedDataSource(XmlStorableObject):
|
|||
cache_duration = None
|
||||
query_parameter = None
|
||||
id_parameter = None
|
||||
data_attribute = None
|
||||
id_attribute = None
|
||||
text_attribute = None
|
||||
id_property = None
|
||||
label_template_property = None
|
||||
|
||||
|
@ -306,6 +327,9 @@ class NamedDataSource(XmlStorableObject):
|
|||
('cache_duration', 'str'),
|
||||
('query_parameter', 'str'),
|
||||
('id_parameter', 'str'),
|
||||
('data_attribute', 'str'),
|
||||
('id_attribute', 'str'),
|
||||
('text_attribute', 'str'),
|
||||
('id_property', 'str'),
|
||||
('label_template_property', 'str'),
|
||||
('data_source', 'data_source'),
|
||||
|
@ -321,14 +345,22 @@ class NamedDataSource(XmlStorableObject):
|
|||
|
||||
@property
|
||||
def extended_data_source(self):
|
||||
if self.type != 'geojson':
|
||||
return self.data_source
|
||||
data_source = self.data_source.copy()
|
||||
data_source.update({
|
||||
'id_property': self.id_property,
|
||||
'label_template_property': self.label_template_property,
|
||||
})
|
||||
return data_source
|
||||
if self.type == 'geojson':
|
||||
data_source = self.data_source.copy()
|
||||
data_source.update({
|
||||
'id_property': self.id_property,
|
||||
'label_template_property': self.label_template_property,
|
||||
})
|
||||
return data_source
|
||||
if self.type == 'json':
|
||||
data_source = self.data_source.copy()
|
||||
data_source.update({
|
||||
'data_attribute': self.data_attribute,
|
||||
'id_attribute': self.id_attribute,
|
||||
'text_attribute': self.text_attribute,
|
||||
})
|
||||
return data_source
|
||||
return self.data_source
|
||||
|
||||
def can_jsonp(self):
|
||||
if self.type == 'jsonp':
|
||||
|
|
Loading…
Reference in New Issue