diff --git a/tests/test_datasource.py b/tests/test_datasource.py index 2abda1acf..2ed835413 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -283,6 +283,46 @@ def test_json_datasource(requests_pub, http_requests): assert data_sources.get_items(datasource) == [] assert data_sources.get_structured_items(datasource) == [] + # specify data_attribute + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'results'} + get_request().datasources_cache = {} + json_file = open(json_file_path, 'w') + json.dump({'results': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file) + json_file.close() + assert data_sources.get_structured_items(datasource) == [ + {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] + + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'data'} + get_request().datasources_cache = {} + assert data_sources.get_structured_items(datasource) == [] + + # specify id_attribute + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'pk'} + get_request().datasources_cache = {} + json_file = open(json_file_path, 'w') + json.dump({'data': [{'pk': '1', 'text': 'foo'}, {'pk': '2', 'text': 'bar'}]}, json_file) + json_file.close() + assert data_sources.get_structured_items(datasource) == [ + {'id': '1', 'text': 'foo', 'pk': '1'}, {'id': '2', 'text': 'bar', 'pk': '2'}] + + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'id'} + get_request().datasources_cache = {} + assert data_sources.get_structured_items(datasource) == [] + + # specify text_attribute + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'label'} + get_request().datasources_cache = {} + json_file = open(json_file_path, 'w') + json.dump({'data': [{'id': '1', 'label': 'foo'}, {'id': '2', 'label': 'bar'}]}, json_file) + json_file.close() + assert data_sources.get_structured_items(datasource) == [ + {'id': '1', 'text': 'foo', 'label': 'foo'}, {'id': '2', 'text': 'bar', 'label': 'bar'}] + + datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'text'} + get_request().datasources_cache = {} + assert data_sources.get_structured_items(datasource) == [ + {'id': '1', 'text': '1', 'label': 'foo'}, {'id': '2', 'text': '2', 'label': 'bar'}] + def test_json_datasource_bad_url(http_requests, caplog): datasource = {'type': 'json', 'value': 'http://remote.example.net/404'} diff --git a/tests/test_datasources_admin_pages.py b/tests/test_datasources_admin_pages.py index 0bfaa80ce..58010a34c 100644 --- a/tests/test_datasources_admin_pages.py +++ b/tests/test_datasources_admin_pages.py @@ -161,7 +161,24 @@ def test_data_sources_view(pub): assert 'Preview' in resp.text assert 'foo' in resp.text + # with other attributes + json_file = open(json_file_path, 'w') + json.dump({'results': [{'pk': '1', 'label': 'foo'}, {'pk': '2'}]}, json_file) + json_file.close() + + data_source.data_attribute = 'results' + data_source.id_attribute = 'pk' + data_source.text_attribute = 'label' + data_source.store() + with HttpRequestsMocking(): + resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id) + assert 'Preview' in resp.text + assert '1: foo' in resp.text + assert '2: 2' in resp.text + assert '

Additional keys are available: label, pk

' in resp.text + # variadic url + data_source.data_attribute = None data_source.data_source = {'type': 'json', 'value': '{{ site_url }}/foo/bar'} data_source.store() with HttpRequestsMocking(): diff --git a/wcs/admin/data_sources.py b/wcs/admin/data_sources.py index f0aefeae4..725630a57 100644 --- a/wcs/admin/data_sources.py +++ b/wcs/admin/data_sources.py @@ -104,6 +104,36 @@ class NamedDataSourceUI(object): 'data-dynamic-display-child-of': 'data_source$type', 'data-dynamic-display-value': 'geojson', }) + form.add(StringWidget, 'data_attribute', + value=self.datasource.data_attribute, + title=_('Data Attribute'), + hint=_('Name of the attribute containing the list of results (default: data)'), + required=False, + advanced=False, + attrs={ + 'data-dynamic-display-child-of': 'data_source$type', + 'data-dynamic-display-value': 'json', + }) + form.add(StringWidget, 'id_attribute', + value=self.datasource.id_attribute, + title=_('Id Attribute'), + hint=_('Name of the attribute containing the identifier of an entry (default: id)'), + required=False, + advanced=False, + attrs={ + 'data-dynamic-display-child-of': 'data_source$type', + 'data-dynamic-display-value': 'json', + }) + form.add(StringWidget, 'text_attribute', + value=self.datasource.text_attribute, + title=_('Text Attribute'), + hint=_('Name of the attribute containing the label of an entry (default: text)'), + required=False, + advanced=False, + attrs={ + 'data-dynamic-display-child-of': 'data_source$type', + 'data-dynamic-display-value': 'json', + }) form.add(StringWidget, 'label_template_property', value=self.datasource.label_template_property, title=_('Label template'), @@ -148,6 +178,9 @@ class NamedDataSourceUI(object): self.datasource.cache_duration = form.get_widget('cache_duration').parse() self.datasource.query_parameter = form.get_widget('query_parameter').parse() self.datasource.id_parameter = form.get_widget('id_parameter').parse() + self.datasource.data_attribute = form.get_widget('data_attribute').parse() + self.datasource.id_attribute = form.get_widget('id_attribute').parse() + self.datasource.text_attribute = form.get_widget('text_attribute').parse() self.datasource.id_property = form.get_widget('id_property').parse() self.datasource.label_template_property = form.get_widget('label_template_property').parse() if slug_widget: diff --git a/wcs/data_sources.py b/wcs/data_sources.py index 3805f908e..03bc2b347 100644 --- a/wcs/data_sources.py +++ b/wcs/data_sources.py @@ -118,54 +118,69 @@ def get_items(data_source, include_disabled=False, mode=None): def request_json_items(url, data_source): url = sign_url_auto_orig(url) - geojson = data_source.get('type') == 'geojson' + data_key = data_source.get('data_attribute') or 'data' try: entries = misc.json_loads(misc.urlopen(url).read()) if not isinstance(entries, dict): raise ValueError('not a json dict') if entries.get('err') not in (None, 0, "0"): raise ValueError('err %s' % entries['err']) - if not geojson and not isinstance(entries.get('data'), list): - raise ValueError('not a json dict with a data list attribute') - if geojson and not isinstance(entries.get('features'), list): - raise ValueError('bad geojson format') + if not isinstance(entries.get(data_key), list): + raise ValueError('not a json dict with a %s list attribute' % data_key) except misc.ConnectionError as e: - if geojson: - get_logger().warning('Error loading GeoJSON data source (%s)' % str(e)) - else: - get_logger().warning('Error loading JSON data source (%s)' % str(e)) + get_logger().warning('Error loading JSON data source (%s)' % str(e)) return None except (ValueError, TypeError) as e: - if geojson: - get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e)) - else: - get_logger().warning('Error reading JSON data source output (%s)' % str(e)) + get_logger().warning('Error reading JSON data source output (%s)' % str(e)) return None items = [] - if geojson: - id_property = data_source.get('id_property') or 'id' - for item in entries.get('features'): - if not item.get('properties', {}).get(id_property): - continue - item['id'] = item['properties'][id_property] - try: - item['text'] = Template( - data_source.get('label_template_property') or '{{ text }}').render(item['properties']) - except (TemplateSyntaxError, VariableDoesNotExist): - pass - if not item.get('text'): - item['text'] = item['id'] - items.append(item) - else: - for item in entries.get('data'): - # skip malformed items - if not isinstance(item, dict): - continue - if item.get('id') is None or item.get('id') == '': - continue - if 'text' not in item: - item['text'] = item['id'] - items.append(item) + id_attribute = data_source.get('id_attribute') or 'id' + text_attribute = data_source.get('text_attribute') or 'text' + for item in entries.get(data_key): + # skip malformed items + if not isinstance(item, dict): + continue + if item.get(id_attribute) is None or item.get(id_attribute) == '': + continue + item['id'] = item[id_attribute] + if text_attribute not in item: + item['text'] = item['id'] + else: + item['text'] = item[text_attribute] + items.append(item) + return items + + +def request_geojson_items(url, data_source): + url = sign_url_auto_orig(url) + try: + entries = misc.json_loads(misc.urlopen(url).read()) + if not isinstance(entries, dict): + raise ValueError('not a json dict') + if entries.get('err') not in (None, 0, "0"): + raise ValueError('err %s' % entries['err']) + if not isinstance(entries.get('features'), list): + raise ValueError('bad geojson format') + except misc.ConnectionError as e: + get_logger().warning('Error loading GeoJSON data source (%s)' % str(e)) + return None + except (ValueError, TypeError) as e: + get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e)) + return None + items = [] + id_property = data_source.get('id_property') or 'id' + for item in entries.get('features'): + if not item.get('properties', {}).get(id_property): + continue + item['id'] = item['properties'][id_property] + try: + item['text'] = Template( + data_source.get('label_template_property') or '{{ text }}').render(item['properties']) + except (TemplateSyntaxError, VariableDoesNotExist): + pass + if not item.get('text'): + item['text'] = item['id'] + items.append(item) return items @@ -247,7 +262,10 @@ def get_structured_items(data_source, mode=None): if items is not None: return items - items = request_json_items(url, data_source) + if geojson: + items = request_geojson_items(url, data_source) + else: + items = request_json_items(url, data_source) if items is None: return [] if hasattr(request, 'datasources_cache'): @@ -298,6 +316,9 @@ class NamedDataSource(XmlStorableObject): cache_duration = None query_parameter = None id_parameter = None + data_attribute = None + id_attribute = None + text_attribute = None id_property = None label_template_property = None @@ -306,6 +327,9 @@ class NamedDataSource(XmlStorableObject): ('cache_duration', 'str'), ('query_parameter', 'str'), ('id_parameter', 'str'), + ('data_attribute', 'str'), + ('id_attribute', 'str'), + ('text_attribute', 'str'), ('id_property', 'str'), ('label_template_property', 'str'), ('data_source', 'data_source'), @@ -321,14 +345,22 @@ class NamedDataSource(XmlStorableObject): @property def extended_data_source(self): - if self.type != 'geojson': - return self.data_source - data_source = self.data_source.copy() - data_source.update({ - 'id_property': self.id_property, - 'label_template_property': self.label_template_property, - }) - return data_source + if self.type == 'geojson': + data_source = self.data_source.copy() + data_source.update({ + 'id_property': self.id_property, + 'label_template_property': self.label_template_property, + }) + return data_source + if self.type == 'json': + data_source = self.data_source.copy() + data_source.update({ + 'data_attribute': self.data_attribute, + 'id_attribute': self.id_attribute, + 'text_attribute': self.text_attribute, + }) + return data_source + return self.data_source def can_jsonp(self): if self.type == 'jsonp':