datasource: data attribute can be "data/foo/bar/results" (#53911)

tests/test_datasource.py

@@ -338,6 +338,18 @@ def test_json_datasource(pub, requests_pub, http_requests):
     get_request().datasources_cache = {}
     assert data_sources.get_structured_items(datasource) == []
 
+    get_request().datasources_cache = {}
+    with open(json_file_path, 'w') as json_file:
+        json.dump({'data': {'results': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}}, json_file)
+    assert data_sources.get_structured_items(datasource) == []
+
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'data.results'}
+    get_request().datasources_cache = {}
+    assert data_sources.get_structured_items(datasource) == [
+        {'id': '1', 'text': 'foo'},
+        {'id': '2', 'text': 'bar'},
+    ]
+
     # specify id_attribute
     datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'pk'}
     get_request().datasources_cache = {}

wcs/admin/data_sources.py

@@ -181,7 +181,10 @@ class NamedDataSourceUI:
                 'data_attribute',
                 value=self.datasource.data_attribute,
                 title=_('Data Attribute'),
-                hint=_('Name of the attribute containing the list of results (default: data)'),
+                hint=_(
+                    'Name of the attribute containing the list of results (default: data). '
+                    'Possibility to chain attributes with a dot separator (example: data.results)'
+                ),
                 required=False,
                 advanced=True,
                 attrs={

wcs/data_sources.py

@@ -171,6 +171,7 @@ def get_json_from_url(url, data_source=None, log_message_part='JSON data source'
     geojson = data_source.get('type') == 'geojson'
     error_summary = None
     exc = None
+
     try:
         entries = misc.json_loads(misc.urlopen(url).read())
         if not isinstance(entries, dict):
@@ -181,7 +182,14 @@ def get_json_from_url(url, data_source=None, log_message_part='JSON data source'
             if not isinstance(entries.get('features'), list):
                 raise ValueError('bad geojson format')
         else:
-            if not isinstance(entries.get(data_key), list):
+            # data_key can be "data.foo.bar.results"
+            keys = data_key.split('.')
+            data = entries
+            for key in keys[:-1]:
+                if not isinstance(data.get(key), dict):
+                    raise ValueError('not a json dict with a %s list attribute' % data_key)
+                data = data[key]
+            if not isinstance(data.get(keys[-1]), list):
                 raise ValueError('not a json dict with a %s list attribute' % data_key)
         return entries
     except misc.ConnectionError as e:
@@ -213,8 +221,12 @@ def request_json_items(url, data_source):
     data_key = data_source.get('data_attribute') or 'data'
     id_attribute = data_source.get('id_attribute') or 'id'
     text_attribute = data_source.get('text_attribute') or 'text'
+    # data_key can be "data.foo.bar.results"
+    keys = data_key.split('.')
+    for key in keys:
+        entries = entries[key]
     items = []
-    for item in entries.get(data_key):
+    for item in entries:
         # skip malformed items
         if not isinstance(item, dict):
             continue