datasources: add geojson support (#42010)
This commit is contained in:
Lauréline Guérin
parent
2cc3d86bb0
commit
337cf7f8c8
|
|
@ -1570,6 +1570,7 @@ def test_form_edit_item_field_data_source(pub):
|
|||
(u'None', True, u'None'),
|
||||
(u'json', False, u'JSON URL'),
|
||||
(u'jsonp', False, u'JSONP URL'),
|
||||
(u'geojson', False, u'GeoJSON URL'),
|
||||
(u'python', False, u'Python Expression')
|
||||
]
|
||||
resp = resp.form.submit('submit').follow()
|
||||
|
|
@ -1584,6 +1585,7 @@ def test_form_edit_item_field_data_source(pub):
|
|||
(u'foobar', False, u'Foobar'),
|
||||
(u'json', False, u'JSON URL'),
|
||||
(u'jsonp', False, u'JSONP URL'),
|
||||
(u'geojson', False, u'GeoJSON URL'),
|
||||
(u'python', False, u'Python Expression')
|
||||
]
|
||||
resp.form['data_source$type'].value = 'foobar'
|
||||
|
|
@ -1600,6 +1602,7 @@ def test_form_edit_item_field_data_source(pub):
|
|||
(u'foobar', True, u'Foobar'),
|
||||
(u'json', False, u'JSON URL'),
|
||||
(u'jsonp', False, u'JSONP URL'),
|
||||
(u'geojson', False, u'GeoJSON URL'),
|
||||
(u'python', False, u'Python Expression')
|
||||
]
|
||||
|
||||
|
|
@ -1613,6 +1616,7 @@ def test_form_edit_item_field_data_source(pub):
|
|||
(u'foobar', True, u'Foobar'),
|
||||
(u'json', False, u'JSON URL'),
|
||||
(u'jsonp', False, u'JSONP URL'),
|
||||
(u'geojson', False, u'GeoJSON URL'),
|
||||
(u'python', False, u'Python Expression')
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -4,17 +4,14 @@ import codecs
|
|||
import pytest
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
from django.utils import six
|
||||
from django.utils.six import StringIO
|
||||
from django.utils.six.moves.urllib import parse as urlparse
|
||||
|
||||
from quixote import cleanup
|
||||
from wcs import publisher
|
||||
from wcs.qommon.http_request import HTTPRequest
|
||||
from wcs.qommon.form import *
|
||||
from wcs.qommon.form import get_request, Form
|
||||
from wcs import fields, data_sources
|
||||
from wcs.data_sources import NamedDataSource, register_data_source_function
|
||||
|
||||
|
|
@ -140,7 +137,7 @@ def test_python_datasource_with_evalutils():
|
|||
('foo', 'Foo', 'foo', {'id': 'foo', 'text': 'Foo', 'value': '2017-01-01'})]
|
||||
|
||||
|
||||
def test_json_datasource(http_requests):
|
||||
def test_json_datasource(requests_pub, http_requests):
|
||||
req = get_request()
|
||||
get_request().datasources_cache = {}
|
||||
datasource = {'type': 'json', 'value': ''}
|
||||
|
|
@ -317,6 +314,255 @@ def test_json_datasource_bad_url_scheme(caplog):
|
|||
assert 'invalid scheme in URL' in caplog.records[-1].message
|
||||
|
||||
|
||||
def test_geojson_datasource(requests_pub, http_requests):
|
||||
get_request()
|
||||
get_request().datasources_cache = {}
|
||||
datasource = {'type': 'geojson', 'value': ''}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# missing file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file_path = os.path.join(pub.app_dir, 'test.geojson')
|
||||
datasource = {'type': 'geojson', 'value': 'file://%s' % geojson_file_path}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# invalid geojson file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'wb')
|
||||
geojson_file.write(codecs.encode(b'foobar', 'zlib_codec'))
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# empty geojson file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# unrelated geojson file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump('foobar', geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# another unrelated geojson file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': 'foobar'}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == []
|
||||
|
||||
# a good geojson file
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar'}})]
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar'}}]
|
||||
|
||||
# a geojson file with additional keys
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}},
|
||||
{'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})]
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}},
|
||||
{'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}}]
|
||||
|
||||
# geojson specified with a variadic url
|
||||
get_request().datasources_cache = {}
|
||||
|
||||
class GeoJSONUrlPath(object):
|
||||
def get_substitution_variables(self):
|
||||
return {'geojson_url': 'file://%s' % geojson_file_path}
|
||||
|
||||
pub.substitutions.feed(GeoJSONUrlPath())
|
||||
datasource = {'type': 'geojson', 'value': '[geojson_url]'}
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})]
|
||||
|
||||
# same with django templated url
|
||||
get_request().datasources_cache = {}
|
||||
|
||||
class GeoJSONUrlPath(object):
|
||||
def get_substitution_variables(self):
|
||||
return {'geojson_url': 'file://%s' % geojson_file_path}
|
||||
|
||||
pub.substitutions.feed(GeoJSONUrlPath())
|
||||
datasource = {'type': 'geojson', 'value': '{{ geojson_url }}'}
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})]
|
||||
|
||||
# geojson specified with a variadic url with an erroneous space
|
||||
get_request().datasources_cache = {}
|
||||
|
||||
class GeoJSONUrlPath(object):
|
||||
def get_substitution_variables(self):
|
||||
return {'geojson_url': 'file://%s' % geojson_file_path}
|
||||
|
||||
pub.substitutions.feed(GeoJSONUrlPath())
|
||||
datasource = {'type': 'geojson', 'value': ' [geojson_url]'}
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})]
|
||||
|
||||
# same with django templated url
|
||||
get_request().datasources_cache = {}
|
||||
|
||||
class GeoJSONUrlPath(object):
|
||||
def get_substitution_variables(self):
|
||||
return {'geojson_url': 'file://%s' % geojson_file_path}
|
||||
|
||||
pub.substitutions.feed(GeoJSONUrlPath())
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}'}
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}),
|
||||
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})]
|
||||
|
||||
# a geojson file with integer as 'id'
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': 1, 'text': 'foo'}},
|
||||
{'properties': {'id': 2, 'text': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', 'foo', '1', {'id': 1, 'text': 'foo', 'properties': {'id': 1, 'text': 'foo'}}),
|
||||
('2', 'bar', '2', {'id': 2, 'text': 'bar', 'properties': {'id': 2, 'text': 'bar'}})]
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': 1, 'text': 'foo', 'properties': {'id': 1, 'text': 'foo'}},
|
||||
{'id': 2, 'text': 'bar', 'properties': {'id': 2, 'text': 'bar'}}]
|
||||
|
||||
# a geojson file with empty or no text values
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': ''}},
|
||||
{'properties': {'id': '2'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == [
|
||||
('1', '1', '1', {'id': '1', 'text': '1', 'properties': {'id': '1', 'text': ''}}),
|
||||
('2', '2', '2', {'id': '2', 'text': '2', 'properties': {'id': '2'}})]
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': '1', 'properties': {'id': '1', 'text': ''}},
|
||||
{'id': '2', 'text': '2', 'properties': {'id': '2'}}]
|
||||
|
||||
# a geojson file with empty or no id
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '', 'text': 'foo'}},
|
||||
{'properties': {'text': 'bar'}},
|
||||
{'properties': {'id': None}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert data_sources.get_structured_items(datasource) == []
|
||||
|
||||
# specify id_property
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'id_property': 'gid'}
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'gid': '1', 'text': 'foo'}},
|
||||
{'properties': {'gid': '2', 'text': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'foo', 'properties': {'gid': '1', 'text': 'foo'}},
|
||||
{'id': '2', 'text': 'bar', 'properties': {'gid': '2', 'text': 'bar'}}]
|
||||
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'id_property': 'id'}
|
||||
get_request().datasources_cache = {}
|
||||
assert data_sources.get_structured_items(datasource) == []
|
||||
|
||||
# specify label_template_property
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ id }}: {{ text }}'}
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': '1: foo', 'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'id': '2', 'text': '2: bar', 'properties': {'id': '2', 'text': 'bar'}}]
|
||||
|
||||
# wrong template
|
||||
get_request().datasources_cache = {}
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ text }'}
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': '{{ text }', 'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'id': '2', 'text': '{{ text }', 'properties': {'id': '2', 'text': 'bar'}}]
|
||||
get_request().datasources_cache = {}
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': 'text'}
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': 'text', 'properties': {'id': '1', 'text': 'foo'}},
|
||||
{'id': '2', 'text': 'text', 'properties': {'id': '2', 'text': 'bar'}}]
|
||||
|
||||
# unknown property or empty value
|
||||
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ label }}'}
|
||||
get_request().datasources_cache = {}
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': 'foo', 'label': ''}},
|
||||
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
assert data_sources.get_structured_items(datasource) == [
|
||||
{'id': '1', 'text': '1', 'properties': {'id': '1', 'text': 'foo', 'label': ''}},
|
||||
{'id': '2', 'text': '2', 'properties': {'id': '2', 'text': 'bar'}}]
|
||||
|
||||
|
||||
def test_geojson_datasource_bad_url(http_requests, caplog):
|
||||
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
|
||||
assert 'status: 404' in caplog.records[-1].message
|
||||
|
||||
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error reading GeoJSON data source output' in caplog.records[-1].message
|
||||
assert 'Expecting value:' in caplog.records[-1].message
|
||||
|
||||
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/connection-error'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
|
||||
assert 'error' in caplog.records[-1].message
|
||||
|
||||
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/json-list-err1'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error reading GeoJSON data source output (err 1)' in caplog.records[-1].message
|
||||
|
||||
|
||||
def test_geojson_datasource_bad_url_scheme(caplog):
|
||||
datasource = {'type': 'geojson', 'value': ''}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert caplog.records[-1].message == 'Empty URL in GeoJSON data source'
|
||||
|
||||
datasource = {'type': 'geojson', 'value': 'foo://bar'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
|
||||
assert 'invalid scheme in URL' in caplog.records[-1].message
|
||||
|
||||
datasource = {'type': 'geojson', 'value': '/bla/blo'}
|
||||
assert data_sources.get_items(datasource) == []
|
||||
assert 'Error loading GeoJSON data source' in caplog.records[-1].message
|
||||
assert 'invalid scheme in URL' in caplog.records[-1].message
|
||||
|
||||
|
||||
def test_item_field_named_python_datasource():
|
||||
NamedDataSource.wipe()
|
||||
data_source = NamedDataSource(name='foobar')
|
||||
|
|
|
|||
|
|
@ -160,6 +160,22 @@ def test_data_sources_view(pub):
|
|||
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
|
||||
assert '<a href="http://example.net/foo/bar"' in resp.text
|
||||
|
||||
# check geojson
|
||||
geojson_file_path = os.path.join(pub.app_dir, 'test.geojson')
|
||||
geojson_file = open(geojson_file_path, 'w')
|
||||
json.dump({'features': [
|
||||
{'properties': {'id': '1', 'text': 'foo', 'label': 'foo'}},
|
||||
{'properties': {'id': '2', 'text': 'bar', 'label': 'bar'}}]}, geojson_file)
|
||||
geojson_file.close()
|
||||
data_source.data_source = {'type': 'geojson', 'value': 'file://%s' % geojson_file_path}
|
||||
data_source.store()
|
||||
with HttpRequestsMocking():
|
||||
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
|
||||
assert 'Preview' in resp.text
|
||||
assert 'foo' in resp.text
|
||||
assert 'bar' in resp.text
|
||||
assert 'Additional keys are available: label' in resp.text
|
||||
|
||||
data_source.data_source = {'type': 'formula', 'value': '[str(x) for x in range(100)]'}
|
||||
data_source.store()
|
||||
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class NamedDataSourceUI(object):
|
|||
advanced=False,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'json',
|
||||
'data-dynamic-display-value-in': 'json|geojson',
|
||||
})
|
||||
form.add(StringWidget, 'query_parameter',
|
||||
value=self.datasource.query_parameter,
|
||||
|
|
@ -92,6 +92,27 @@ class NamedDataSourceUI(object):
|
|||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'json',
|
||||
})
|
||||
form.add(StringWidget, 'id_property',
|
||||
value=self.datasource.id_property,
|
||||
title=_('Id Property'),
|
||||
hint=_('Name of the property to use to get a given entry from data source (default: id)'),
|
||||
required=False,
|
||||
advanced=False,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'geojson',
|
||||
})
|
||||
form.add(StringWidget, 'label_template_property',
|
||||
value=self.datasource.label_template_property,
|
||||
title=_('Label template'),
|
||||
hint=_('Django expression to build label of each value (default: {{ text }})'),
|
||||
required=False,
|
||||
advanced=False,
|
||||
size=80,
|
||||
attrs={
|
||||
'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value': 'geojson',
|
||||
})
|
||||
if self.datasource.slug and not self.is_used():
|
||||
form.add(StringWidget, 'slug',
|
||||
value=self.datasource.slug,
|
||||
|
|
@ -124,6 +145,8 @@ class NamedDataSourceUI(object):
|
|||
self.datasource.cache_duration = form.get_widget('cache_duration').parse()
|
||||
self.datasource.query_parameter = form.get_widget('query_parameter').parse()
|
||||
self.datasource.id_parameter = form.get_widget('id_parameter').parse()
|
||||
self.datasource.id_property = form.get_widget('id_property').parse()
|
||||
self.datasource.label_template_property = form.get_widget('label_template_property').parse()
|
||||
if slug_widget:
|
||||
self.datasource.slug = slug
|
||||
self.datasource.store()
|
||||
|
|
@ -156,9 +179,10 @@ class NamedDataSourcePage(Directory):
|
|||
return formdefs
|
||||
|
||||
def preview_block(self):
|
||||
if self.datasource.data_source.get('type') not in ('json', 'formula'):
|
||||
data_source = self.datasource.extended_data_source
|
||||
if data_source.get('type') not in ('json', 'geojson', 'formula'):
|
||||
return ''
|
||||
items = get_structured_items(self.datasource.data_source)
|
||||
items = get_structured_items(data_source)
|
||||
if not items:
|
||||
return ''
|
||||
r = TemplateIO(html=True)
|
||||
|
|
@ -175,7 +199,10 @@ class NamedDataSourcePage(Directory):
|
|||
else:
|
||||
r += htmltext('<li><tt>%s</tt>: %s</li>') % (
|
||||
item.get('id'), item.get('text'))
|
||||
additional_keys |= set(item.keys())
|
||||
if data_source.get('type') == 'geojson':
|
||||
additional_keys |= set(item.get('properties', {}).keys())
|
||||
else:
|
||||
additional_keys |= set(item.keys())
|
||||
if len(items) > 10:
|
||||
r += htmltext('<li>...</li>')
|
||||
r += htmltext('</ul>')
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import collections
|
|||
import hashlib
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from django.template import TemplateSyntaxError, VariableDoesNotExist
|
||||
from django.utils import six
|
||||
from django.utils.encoding import force_text, force_bytes
|
||||
from django.utils.six.moves.urllib import parse as urllib
|
||||
|
|
@ -71,6 +72,7 @@ class DataSourceSelectionWidget(CompositeWidget):
|
|||
options.append(('json', _('JSON URL'), 'json'))
|
||||
if allow_jsonp:
|
||||
options.append(('jsonp', _('JSONP URL'), 'jsonp'))
|
||||
options.append(('geojson', _('GeoJSON URL'), 'geojson'))
|
||||
options.append(('formula', _('Python Expression'), 'python'))
|
||||
|
||||
self.add(SingleSelectWidget, 'type', options=options, value=value.get('type'),
|
||||
|
|
@ -82,7 +84,7 @@ class DataSourceSelectionWidget(CompositeWidget):
|
|||
|
||||
self.add(StringWidget, 'value', value=value.get('value'), size=80,
|
||||
attrs={'data-dynamic-display-child-of': 'data_source$type',
|
||||
'data-dynamic-display-value-in': 'json|jsonp|python'})
|
||||
'data-dynamic-display-value-in': 'json|jsonp|geojson|python'})
|
||||
|
||||
self._parsed = False
|
||||
|
||||
|
|
@ -116,30 +118,54 @@ def get_items(data_source, include_disabled=False, mode=None):
|
|||
return tupled_items
|
||||
|
||||
|
||||
def request_json_items(url):
|
||||
def request_json_items(url, data_source):
|
||||
url = sign_url_auto_orig(url)
|
||||
geojson = data_source.get('type') == 'geojson'
|
||||
try:
|
||||
entries = misc.json_loads(misc.urlopen(url).read())
|
||||
if not isinstance(entries, dict):
|
||||
raise ValueError('not a json dict')
|
||||
if entries.get('err') not in (None, 0, "0"):
|
||||
raise ValueError('err %s' % entries['err'])
|
||||
if not isinstance(entries.get('data'), list):
|
||||
if not geojson and not isinstance(entries.get('data'), list):
|
||||
raise ValueError('not a json dict with a data list attribute')
|
||||
if geojson and not isinstance(entries.get('features'), list):
|
||||
raise ValueError('bad geojson format')
|
||||
except misc.ConnectionError as e:
|
||||
get_logger().warn('Error loading JSON data source (%s)' % str(e))
|
||||
if geojson:
|
||||
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
|
||||
else:
|
||||
get_logger().warning('Error loading JSON data source (%s)' % str(e))
|
||||
return None
|
||||
except (ValueError, TypeError) as e:
|
||||
get_logger().warn('Error reading JSON data source output (%s)' % str(e))
|
||||
if geojson:
|
||||
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
|
||||
else:
|
||||
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
|
||||
return None
|
||||
items = []
|
||||
for item in entries.get('data'):
|
||||
# skip malformed items
|
||||
if item.get('id') is None or item.get('id') == '':
|
||||
continue
|
||||
if 'text' not in item:
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
if geojson:
|
||||
id_property = data_source.get('id_property') or 'id'
|
||||
for item in entries.get('features'):
|
||||
if not item.get('properties', {}).get(id_property):
|
||||
continue
|
||||
item['id'] = item['properties'][id_property]
|
||||
try:
|
||||
item['text'] = Template(
|
||||
data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
|
||||
except (TemplateSyntaxError, VariableDoesNotExist):
|
||||
pass
|
||||
if not item.get('text'):
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
else:
|
||||
for item in entries.get('data'):
|
||||
# skip malformed items
|
||||
if item.get('id') is None or item.get('id') == '':
|
||||
continue
|
||||
if 'text' not in item:
|
||||
item['text'] = item['id']
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
|
|
@ -159,12 +185,12 @@ def get_structured_items(data_source, mode=None):
|
|||
items.sort(key=lambda x: misc.simplify(x['text']))
|
||||
return items
|
||||
|
||||
if data_source.get('type') not in ('json', 'jsonp', 'formula'):
|
||||
if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula'):
|
||||
# named data source
|
||||
named_data_source = NamedDataSource.get_by_slug(data_source['type'])
|
||||
if named_data_source.cache_duration:
|
||||
cache_duration = int(named_data_source.cache_duration)
|
||||
data_source = named_data_source.data_source
|
||||
data_source = named_data_source.extended_data_source
|
||||
|
||||
if data_source.get('type') == 'formula':
|
||||
# the result of a python expression, it must be a list.
|
||||
|
|
@ -181,8 +207,8 @@ def get_structured_items(data_source, mode=None):
|
|||
try:
|
||||
value = eval(data_source.get('value'), global_eval_dict, variables)
|
||||
if not isinstance(value, collections.Iterable):
|
||||
get_logger().warn('Python data source (%r) gave a non-iterable result' % \
|
||||
data_source.get('value'))
|
||||
get_logger().warn('Python data source (%r) gave a non-iterable result' %
|
||||
data_source.get('value'))
|
||||
return []
|
||||
if len(value) == 0:
|
||||
return []
|
||||
|
|
@ -201,13 +227,17 @@ def get_structured_items(data_source, mode=None):
|
|||
except:
|
||||
get_logger().warn('Failed to eval() Python data source (%r)' % data_source.get('value'))
|
||||
return []
|
||||
elif data_source.get('type') == 'json':
|
||||
elif data_source.get('type') in ['json', 'geojson']:
|
||||
# the content available at a json URL, it must answer with a dict with
|
||||
# a 'data' key holding the list of items, each of them being a dict
|
||||
# with at least both an "id" and a "text" key.
|
||||
geojson = data_source.get('type') == 'geojson'
|
||||
url = data_source.get('value')
|
||||
if not url:
|
||||
get_logger().warn('Empty URL in JSON data source')
|
||||
if geojson:
|
||||
get_logger().warning('Empty URL in GeoJSON data source')
|
||||
else:
|
||||
get_logger().warning('Empty URL in JSON data source')
|
||||
return []
|
||||
url = url.strip()
|
||||
if Template.is_template_string(url):
|
||||
|
|
@ -225,7 +255,7 @@ def get_structured_items(data_source, mode=None):
|
|||
if items is not None:
|
||||
return items
|
||||
|
||||
items = request_json_items(url)
|
||||
items = request_json_items(url, data_source)
|
||||
if items is None:
|
||||
return []
|
||||
if hasattr(request, 'datasources_cache'):
|
||||
|
|
@ -240,7 +270,7 @@ def get_real(data_source):
|
|||
if not data_source:
|
||||
return None
|
||||
ds_type = data_source.get('type')
|
||||
if ds_type in ('json', 'jsonp', 'formula'):
|
||||
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
||||
return data_source
|
||||
if ds_type and ds_type.startswith('carddef:'):
|
||||
return data_source
|
||||
|
|
@ -251,7 +281,7 @@ def get_object(data_source):
|
|||
if not data_source:
|
||||
return None
|
||||
ds_type = data_source.get('type')
|
||||
if ds_type in ('json', 'jsonp', 'formula'):
|
||||
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
||||
named_data_source = NamedDataSource()
|
||||
named_data_source.data_source = data_source
|
||||
return named_data_source
|
||||
|
|
@ -274,12 +304,16 @@ class NamedDataSource(XmlStorableObject):
|
|||
cache_duration = None
|
||||
query_parameter = None
|
||||
id_parameter = None
|
||||
id_property = None
|
||||
label_template_property = None
|
||||
|
||||
# declarations for serialization
|
||||
XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'),
|
||||
('cache_duration', 'str'),
|
||||
('query_parameter', 'str'),
|
||||
('id_parameter', 'str'),
|
||||
('id_property', 'str'),
|
||||
('label_template_property', 'str'),
|
||||
('data_source', 'data_source'),
|
||||
]
|
||||
|
||||
|
|
@ -291,6 +325,17 @@ class NamedDataSource(XmlStorableObject):
|
|||
def type(self):
|
||||
return self.data_source.get('type')
|
||||
|
||||
@property
|
||||
def extended_data_source(self):
|
||||
if self.type != 'geojson':
|
||||
return self.data_source
|
||||
data_source = self.data_source.copy()
|
||||
data_source.update({
|
||||
'id_property': self.id_property,
|
||||
'label_template_property': self.label_template_property,
|
||||
})
|
||||
return data_source
|
||||
|
||||
def can_jsonp(self):
|
||||
if self.type == 'jsonp':
|
||||
return True
|
||||
|
|
@ -387,7 +432,7 @@ class NamedDataSource(XmlStorableObject):
|
|||
return None
|
||||
return items[0]
|
||||
|
||||
items = request_json_items(url)
|
||||
items = request_json_items(url, self.data_source)
|
||||
if not items: # None or empty list are not valid
|
||||
return None
|
||||
if hasattr(request, 'datasources_cache'):
|
||||
|
|
@ -428,6 +473,7 @@ class NamedDataSource(XmlStorableObject):
|
|||
data_source_labels = {
|
||||
'json': _('JSON'),
|
||||
'jsonp': _('JSONP'),
|
||||
'geojson': _('GeoJSON'),
|
||||
'formula': _('Python Expression'),
|
||||
}
|
||||
data_source_type = self.data_source.get('type')
|
||||
|
|
|
|||
Loading…
Reference in New Issue