entrouvert/wcs
entrouvert
/
wcs
14
1
Fork 0
Code Issues Pull Requests 46 Releases Activity

api: add endpoint for geojson data sources (#47066)

This commit is contained in:
Frédéric Péters 2020-11-09 09:41:01 +01:00
parent 65a54ff5da
commit 1daf66e50c
3 changed files with 90 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/test_datasource.py
View File

@ -588,22 +588,22 @@ def test_geojson_datasource(requests_pub, http_requests):
def test_geojson_datasource_bad_url(http_requests, caplog):
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404'}
assert data_sources.get_items(datasource) == []
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
assert 'Error loading JSON data source' in caplog.records[-1].message
assert 'status: 404' in caplog.records[-1].message
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml'}
assert data_sources.get_items(datasource) == []
assert 'Error reading GeoJSON data source output' in caplog.records[-1].message
assert 'Error reading JSON data source output' in caplog.records[-1].message
assert 'Expecting value:' in caplog.records[-1].message
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/connection-error'}
assert data_sources.get_items(datasource) == []
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
assert 'Error loading JSON data source' in caplog.records[-1].message
assert 'error' in caplog.records[-1].message
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/json-list-err1'}
assert data_sources.get_items(datasource) == []
assert 'Error reading GeoJSON data source output (err 1)' in caplog.records[-1].message
assert 'Error reading JSON data source output (err 1)' in caplog.records[-1].message
def test_geojson_datasource_bad_url_scheme(caplog):
@ -613,12 +613,12 @@ def test_geojson_datasource_bad_url_scheme(caplog):
datasource = {'type': 'geojson', 'value': 'foo://bar'}
assert data_sources.get_items(datasource) == []
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
assert 'Error loading JSON data source' in caplog.records[-1].message
assert 'invalid scheme in URL' in caplog.records[-1].message
datasource = {'type': 'geojson', 'value': '/bla/blo'}
assert data_sources.get_items(datasource) == []
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
assert 'Error loading JSON data source' in caplog.records[-1].message
assert 'invalid scheme in URL' in caplog.records[-1].message

14
wcs/api.py
View File

@ -37,6 +37,7 @@ from wcs.categories import Category
from wcs.conditions import Condition, ValidationError
from wcs.carddef import CardDef
from wcs.formdef import FormDef
from wcs.data_sources import get_object as get_data_source_object
from wcs.roles import Role, logged_users_role
from wcs.forms.common import FormStatusPage
import wcs.qommon.storage as st
@ -936,10 +937,20 @@ class AutocompleteDirectory(Directory):
return json.dumps({'data': [{'id': x['id'], 'text': x['text']} for x in values]})
class GeoJsonDirectory(Directory):
def _q_lookup(self, component):
try:
data_source = get_data_source_object({'type': component}, ignore_errors=False)
except KeyError:
raise TraversalError()
get_response().set_content_type('application/json')
return json.dumps(data_source.get_geojson_data())
class ApiDirectory(Directory):
_q_exports = ['forms', 'roles', ('reverse-geocoding', 'reverse_geocoding'),
'formdefs', 'categories', 'user', 'users', 'code', 'autocomplete',
'cards']
'cards', 'geojson']
cards = ApiCardsDirectory()
forms = ApiFormsDirectory()
@ -949,6 +960,7 @@ class ApiDirectory(Directory):
users = ApiUsersDirectory()
code = ApiTrackingCodeDirectory()
autocomplete = AutocompleteDirectory()
geojson = GeoJsonDirectory()
def roles(self):
get_response().set_content_type('application/json')

93
wcs/data_sources.py
View File

@ -126,26 +126,39 @@ def get_items(data_source, include_disabled=False, mode=None):
return tupled_items
def request_json_items(url, data_source):
def get_json_from_url(url, data_source):
url = sign_url_auto_orig(url)
data_key = data_source.get('data_attribute') or 'data'
geojson = data_source.get('type') == 'geojson'
try:
entries = misc.json_loads(misc.urlopen(url).read())
if not isinstance(entries, dict):
raise ValueError('not a json dict')
if entries.get('err') not in (None, 0, "0"):
raise ValueError('err %s' % entries['err'])
if not isinstance(entries.get(data_key), list):
raise ValueError('not a json dict with a %s list attribute' % data_key)
if geojson:
if not isinstance(entries.get('features'), list):
raise ValueError('bad geojson format')
else:
if not isinstance(entries.get(data_key), list):
raise ValueError('not a json dict with a %s list attribute' % data_key)
except misc.ConnectionError as e:
get_logger().warning('Error loading JSON data source (%s)' % str(e))
return None
except (ValueError, TypeError) as e:
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
return None
items = []
return entries
def request_json_items(url, data_source):
entries = get_json_from_url(url, data_source)
if entries is None:
return None
data_key = data_source.get('data_attribute') or 'data'
id_attribute = data_source.get('id_attribute') or 'id'
text_attribute = data_source.get('text_attribute') or 'text'
items = []
for item in entries.get(data_key):
# skip malformed items
if not isinstance(item, dict):
@ -162,20 +175,8 @@ def request_json_items(url, data_source):
def request_geojson_items(url, data_source):
url = sign_url_auto_orig(url)
try:
entries = misc.json_loads(misc.urlopen(url).read())
if not isinstance(entries, dict):
raise ValueError('not a json dict')
if entries.get('err') not in (None, 0, "0"):
raise ValueError('err %s' % entries['err'])
if not isinstance(entries.get('features'), list):
raise ValueError('bad geojson format')
except misc.ConnectionError as e:
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
return None
except (ValueError, TypeError) as e:
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
entries = get_json_from_url(url, data_source)
if entries is None:
return None
items = []
id_property = data_source.get('id_property') or 'id'
@ -306,7 +307,7 @@ def get_real(data_source):
return NamedDataSource.get_by_slug(ds_type).data_source
def get_object(data_source):
def get_object(data_source, ignore_errors=True):
if not data_source:
return None
ds_type = data_source.get('type')
@ -320,7 +321,7 @@ def get_object(data_source):
named_data_source = NamedDataSource()
named_data_source.data_source = data_source
return named_data_source
return NamedDataSource.get_by_slug(ds_type)
return NamedDataSource.get_by_slug(ds_type, ignore_errors=ignore_errors)
class NamedDataSource(XmlStorableObject):
@ -444,10 +445,12 @@ class NamedDataSource(XmlStorableObject):
}
@classmethod
def get_by_slug(cls, slug):
def get_by_slug(cls, slug, ignore_errors=True):
objects = [x for x in cls.select() if x.slug == slug]
if objects:
return objects[0]
if not ignore_errors:
raise KeyError(slug)
get_logger().warning("data source '%s' does not exist" % slug)
return StubNamedDataSource(name=slug)
@ -499,6 +502,52 @@ class NamedDataSource(XmlStorableObject):
}))
return None
def get_geojson_url(self):
assert self.type == 'geojson'
return '/api/geojson/%s' % self.slug
def get_geojson_data(self):
url = self.data_source.get('value').strip()
if Template.is_template_string(url):
vars = get_publisher().substitutions.get_context_variables(mode='lazy')
url = get_variadic_url(url, vars)
request = get_request()
if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
return request.datasources_cache[url]
cache_duration = 0
if self.cache_duration:
cache_duration = int(self.cache_duration)
if cache_duration:
cache_key = 'geojson-data-source-%s' % force_str(hashlib.md5(force_bytes(url)).hexdigest())
from django.core.cache import cache
data = cache.get(cache_key)
if data is not None:
return data
data = get_json_from_url(url, self.data_source)
id_property = self.id_property or 'id'
label_template_property = self.label_template_property or '{{ text }}'
for feature in data['features']:
feature['properties']['_id'] = feature['properties'][id_property]
try:
feature['properties']['_text'] = Template(
label_template_property).render(feature['properties'])
except (TemplateSyntaxError, VariableDoesNotExist):
pass
if not feature['properties'].get('_text'):
feature['properties']['_text'] = feature['properties']['_id']
if hasattr(request, 'datasources_cache'):
request.datasources_cache[url] = data
if cache_duration:
cache.set(cache_key, data, cache_duration)
return data
def get_value_by_id(self, param_name, param_value):
url = self.data_source.get('value').strip()
if Template.is_template_string(url):
@ -561,7 +610,7 @@ class NamedDataSource(XmlStorableObject):
elif self.type == 'json' and self.id_parameter:
value = self.get_value_by_id(self.id_parameter, option_id)
else:
structured_items = get_structured_items(self.data_source, mode='lazy')
structured_items = get_structured_items(self.extended_data_source, mode='lazy')
for item in structured_items:
if str(item['id']) == str(option_id):
value = item