406 lines
15 KiB
Python
406 lines
15 KiB
Python
# passerelle - uniform access to multiple data sources and services
|
|
# Copyright (C) 2018 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import string
|
|
|
|
from django.core.exceptions import ValidationError
|
|
from django.urls import reverse
|
|
from django.db import models
|
|
from django.http import HttpResponse
|
|
from django.shortcuts import get_object_or_404
|
|
from django.utils.six.moves.urllib import parse as urlparse
|
|
from django.utils import six
|
|
from django.utils.text import slugify
|
|
from django.utils.translation import ugettext_lazy as _
|
|
|
|
from passerelle.utils import mark_safe_lazy
|
|
from passerelle.utils.jsonresponse import APIError
|
|
from passerelle.utils.api import endpoint
|
|
from passerelle.utils.conversion import num2deg
|
|
from passerelle.utils.templates import render_to_string, validate_template
|
|
from passerelle.base.models import BaseResource, HTTPResource, BaseQuery
|
|
|
|
|
|
class ArcGISError(APIError):
|
|
pass
|
|
|
|
|
|
class ArcGIS(BaseResource, HTTPResource):
|
|
category = _('Geographic information system')
|
|
|
|
base_url = models.URLField(_('Webservice Base URL'))
|
|
|
|
class Meta:
|
|
verbose_name = _('ArcGIS REST API')
|
|
|
|
@endpoint(
|
|
name='mapservice-query',
|
|
description=_('Map Service Query'),
|
|
perm='can_access',
|
|
parameters={
|
|
'folder': {
|
|
'description': _('Folder name'),
|
|
'example_value': 'Specialty',
|
|
},
|
|
'service': {
|
|
'description': _('Service name'),
|
|
'example_value': 'ESRI_StateCityHighway_USA',
|
|
},
|
|
'layer': {
|
|
'description': _('Layer or table name'),
|
|
'example_value': '1',
|
|
},
|
|
'lat': {'description': _('Latitude')},
|
|
'lon': {'description': _('Longitude')},
|
|
'latmin': {'description': _('Minimal latitude (envelope)')},
|
|
'lonmin': {'description': _('Minimal longitude (envelope)')},
|
|
'latmax': {'description': _('Maximal latitude (envelope)')},
|
|
'lonmax': {'description': _('Maximal longitude (envelope)')},
|
|
'q': {'description': _('Search text in display field')},
|
|
'template': {
|
|
'description': _('Django template for text attribute'),
|
|
'example_value': '{{ attributes.STATE_NAME }} ({{ attributes.STATE_ABBR }})',
|
|
},
|
|
'id_template': {
|
|
'description': _('Django template for id attribute'),
|
|
},
|
|
'full': {
|
|
'description': _('Returns all ArcGIS informations (geometry, metadata)'),
|
|
'type': 'bool',
|
|
},
|
|
},
|
|
)
|
|
def mapservice_query(
|
|
self,
|
|
request,
|
|
service,
|
|
layer='0',
|
|
folder='',
|
|
lat=None,
|
|
lon=None,
|
|
latmin=None,
|
|
lonmin=None,
|
|
latmax=None,
|
|
lonmax=None,
|
|
q=None,
|
|
template=None,
|
|
id_template=None,
|
|
full=False,
|
|
**kwargs,
|
|
):
|
|
url = urlparse.urljoin(self.base_url, 'services/')
|
|
if folder:
|
|
url = urlparse.urljoin(url, folder + '/')
|
|
url = urlparse.urljoin(url, service + '/MapServer/' + layer + '/query')
|
|
|
|
# build query params
|
|
# cf https://developers.arcgis.com/rest/services-reference/query-map-service-layer-.htm
|
|
params = {
|
|
'f': 'json',
|
|
'inSR': '4326',
|
|
'outSR': '4326',
|
|
'outFields': '*',
|
|
}
|
|
if lat and lon:
|
|
try:
|
|
lon, lat = float(lon), float(lat)
|
|
except (ValueError,):
|
|
raise APIError('<lon> and <lat> must be floats', http_status=400)
|
|
params['geometry'] = '{},{}'.format(lon, lat)
|
|
params['geometryType'] = 'esriGeometryPoint'
|
|
elif latmin and lonmin and latmax and lonmax:
|
|
try:
|
|
lonmin, latmin = float(lonmin), float(latmin)
|
|
lonmax, latmax = float(lonmax), float(latmax)
|
|
except (ValueError,):
|
|
raise APIError('<lonmin> <latmin> <lonmax> and <latmax> must be floats', http_status=400)
|
|
params['geometry'] = '{},{},{},{}'.format(lonmin, latmin, lonmax, latmax)
|
|
params['geometryType'] = 'esriGeometryEnvelope'
|
|
if q is not None:
|
|
params['text'] = q
|
|
# consider all remaining parameters as ArcGIS ones
|
|
params.update(kwargs)
|
|
if 'where' not in params and 'text' not in params:
|
|
params['where'] = '1=1'
|
|
if 'distance' in params and 'units' not in params:
|
|
params['units'] = 'esriSRUnit_Meter'
|
|
|
|
response = self.requests.get(url, params=params)
|
|
|
|
# errors
|
|
if response.status_code // 100 != 2:
|
|
raise ArcGISError('ArcGIS returned status code %s' % response.status_code)
|
|
try:
|
|
infos = response.json()
|
|
except (ValueError,):
|
|
raise ArcGISError('ArcGIS returned invalid JSON content: %r' % response.content)
|
|
if 'error' in infos:
|
|
err_desc = infos['error'].get('message') or 'unknown ArcGIS error'
|
|
raise ArcGISError(err_desc, data=infos)
|
|
|
|
features = infos.pop('features', [])
|
|
id_fieldname = infos.get('objectIdFieldName') or 'OBJECTID'
|
|
text_fieldname = infos.get('displayFieldName')
|
|
if infos.get('fieldAliases'):
|
|
aliases = {v: k for k, v in infos['fieldAliases'].items()}
|
|
else:
|
|
aliases = {}
|
|
|
|
# data is the features list, with 'id' and 'text' entries
|
|
data = []
|
|
|
|
def get_feature_attribute(feature, attribute):
|
|
if attribute in feature['attributes']:
|
|
return feature['attributes'][attribute]
|
|
return feature['attributes'].get(aliases.get(attribute))
|
|
|
|
for n, feature in enumerate(features):
|
|
if 'attributes' in feature:
|
|
feature['id'] = '%s' % get_feature_attribute(feature, id_fieldname)
|
|
feature['text'] = '%s' % get_feature_attribute(feature, text_fieldname)
|
|
else:
|
|
feature['id'] = feature['text'] = '%d' % (n + 1)
|
|
if template:
|
|
feature['text'] = render_to_string(template, feature)
|
|
if id_template:
|
|
feature['id'] = render_to_string(id_template, feature)
|
|
if not full and 'geometry' in feature:
|
|
del feature['geometry']
|
|
data.append(feature)
|
|
|
|
if full:
|
|
return {'data': data, 'metadata': infos}
|
|
return {'data': data}
|
|
|
|
@endpoint(
|
|
name='district',
|
|
description=_('Districts in Nancy Town'),
|
|
parameters={
|
|
'lat': {'description': _('Latitude')},
|
|
'lon': {'description': _('Longitude')},
|
|
},
|
|
show=False,
|
|
)
|
|
def district(self, request, lon=None, lat=None):
|
|
# deprecated endpoint
|
|
if 'NANCY_Grc' in self.base_url:
|
|
# Nancy URL used to contains folder, service and layer, remove them
|
|
self.base_url = 'https://geoservices.grand-nancy.org/arcgis/rest/'
|
|
features = self.mapservice_query(
|
|
request,
|
|
folder='public',
|
|
service='NANCY_Grc',
|
|
layer='0',
|
|
template='{{ attributes.NOM }}',
|
|
id_template='{{ attributes.NUMERO }}',
|
|
lon=lon,
|
|
lat=lat,
|
|
)['data']
|
|
if not features:
|
|
raise APIError('No features found.')
|
|
for feature in features:
|
|
del feature['attributes']
|
|
feature['id'] = int(feature['id'])
|
|
if len(features) == 1:
|
|
return {'data': features[0]}
|
|
return {'data': features}
|
|
|
|
@endpoint(
|
|
name='tile',
|
|
description=_('Tiles layer'),
|
|
pattern=r'^(?P<layer>[\w/]+)/(?P<zoom>\d+)/(?P<tile_x>\d+)/(?P<tile_y>\d+)\.png$',
|
|
)
|
|
def tile(self, request, layer, zoom, tile_x, tile_y):
|
|
zoom = int(zoom)
|
|
tile_x = int(tile_x)
|
|
tile_y = int(tile_y)
|
|
|
|
bbox = '%.6f,%.6f,%.6f,%.6f' % (num2deg(tile_x, tile_y, zoom) + num2deg(tile_x + 1, tile_y + 1, zoom))
|
|
|
|
# imageSR=3857: default projection for leaflet
|
|
base_url = self.base_url
|
|
if not base_url.endswith('/'):
|
|
base_url += '/'
|
|
return HttpResponse(
|
|
self.requests.get(
|
|
base_url
|
|
+ '%s/MapServer/export' % layer
|
|
+ '?dpi=96&format=png24&bboxSR=4326&imageSR=3857&'
|
|
+ 'transparent=true&size=256,256&f=image&'
|
|
+ 'bbox=%s' % bbox
|
|
).content,
|
|
content_type='image/png',
|
|
)
|
|
|
|
@endpoint(
|
|
name='q',
|
|
description=_('Query'),
|
|
pattern=r'^(?P<query_slug>[\w:_-]+)/$',
|
|
perm='can_access',
|
|
show=False,
|
|
)
|
|
def q(self, request, query_slug, q=None, full=False, **kwargs):
|
|
query = get_object_or_404(Query, resource=self, slug=query_slug)
|
|
refs = [ref for ref, _ in query.where_references]
|
|
refs += ['q', 'full']
|
|
kwargs = {}
|
|
for key in request.GET:
|
|
if key not in refs:
|
|
kwargs[key] = request.GET[key]
|
|
return query.q(request, q=None, full=full, **kwargs)
|
|
|
|
def export_json(self):
|
|
d = super(ArcGIS, self).export_json()
|
|
d['queries'] = [query.export_json() for query in self.queries.all()]
|
|
return d
|
|
|
|
@classmethod
|
|
def import_json_real(cls, overwrite, instance, d, **kwargs):
|
|
queries = d.pop('queries', [])
|
|
instance = super(ArcGIS, cls).import_json_real(overwrite, instance, d, **kwargs)
|
|
new = []
|
|
if instance and overwrite:
|
|
Query.objects.filter(resource=instance).delete()
|
|
for query in queries:
|
|
q = Query.import_json(query)
|
|
q.resource = instance
|
|
new.append(q)
|
|
Query.objects.bulk_create(new)
|
|
return instance
|
|
|
|
def create_query_url(self):
|
|
return reverse('arcgis-query-new', kwargs={'slug': self.slug})
|
|
|
|
|
|
class SqlFormatter(string.Formatter):
|
|
def format_field(self, value, format_spec):
|
|
if format_spec and format_spec[-1].isalpha() and format_spec[-1] == 'd':
|
|
value = int(value)
|
|
formatted = super(SqlFormatter, self).format_field(value, format_spec)
|
|
if not format_spec or not format_spec[-1].isalpha() or format_spec[-1] == 's':
|
|
formatted = "'%s'" % formatted.replace("'", "''")
|
|
return formatted
|
|
|
|
|
|
def validate_where(format_string):
|
|
formatter = SqlFormatter()
|
|
for prefix, ref, format_spec, conversion in formatter.parse(format_string):
|
|
if ref is None:
|
|
pass
|
|
elif ref == '':
|
|
raise ValidationError(_('missing reference'))
|
|
elif ref != slugify(ref):
|
|
raise ValidationError(_('invalid reference'))
|
|
|
|
|
|
class Query(BaseQuery):
|
|
resource = models.ForeignKey(
|
|
to=ArcGIS, related_name='queries', verbose_name=_('Resource'), on_delete=models.CASCADE
|
|
)
|
|
|
|
folder = models.CharField(verbose_name=_('ArcGis Folder'), max_length=64, blank=True)
|
|
service = models.CharField(verbose_name=_('ArcGis Service'), max_length=64)
|
|
layer = models.CharField(verbose_name=_('ArcGis Layer'), max_length=8, blank=True)
|
|
|
|
where = models.TextField(
|
|
verbose_name=_('ArcGis Where Clause'),
|
|
blank=True,
|
|
validators=[validate_where],
|
|
help_text=mark_safe_lazy(
|
|
_(
|
|
'<span>Use syntax <tt>{name}</tt> to introduce a string '
|
|
'parameter and <tt>{name:d}</tt> for a decimal parameter. ex.:<br/>'
|
|
'<tt>adress LIKE (\'%\' || UPPER({adress}) || \'%\')</tt><br/>'
|
|
'<tt>population < {population:d}</tt></span>'
|
|
)
|
|
),
|
|
)
|
|
|
|
id_template = models.TextField(
|
|
verbose_name=_('Id template'),
|
|
validators=[validate_template],
|
|
help_text=_(
|
|
'Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}'
|
|
),
|
|
blank=True,
|
|
)
|
|
|
|
text_template = models.TextField(
|
|
verbose_name=_('Text template'),
|
|
help_text=_(
|
|
'Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}'
|
|
),
|
|
validators=[validate_template],
|
|
blank=True,
|
|
)
|
|
|
|
delete_view = 'arcgis-query-delete'
|
|
edit_view = 'arcgis-query-edit'
|
|
|
|
@property
|
|
def where_references(self):
|
|
if self.where:
|
|
return [
|
|
(ref, int if spec and spec[-1] == 'd' else str)
|
|
for _, ref, spec, _ in SqlFormatter().parse(self.where)
|
|
if ref is not None
|
|
]
|
|
else:
|
|
return []
|
|
|
|
def q(self, request, q=None, full=False, **kwargs):
|
|
kwargs.update(
|
|
{
|
|
'service': self.service,
|
|
}
|
|
)
|
|
if self.id_template:
|
|
kwargs['id_template'] = self.id_template
|
|
if self.text_template:
|
|
kwargs['template'] = self.text_template
|
|
if self.folder:
|
|
kwargs['folder'] = self.folder
|
|
if self.layer:
|
|
kwargs['layer'] = self.layer
|
|
if self.where:
|
|
format_kwargs = {key: request.GET.get(key, '') for key, klass in self.where_references}
|
|
formatter = SqlFormatter()
|
|
kwargs['where'] = formatter.format(self.where, **format_kwargs)
|
|
return self.resource.mapservice_query(request, q=q, full=full, **kwargs)
|
|
|
|
def as_endpoint(self):
|
|
endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)
|
|
|
|
mapservice_endpoint = self.resource.mapservice_query.endpoint_info
|
|
endpoint.func = mapservice_endpoint.func
|
|
endpoint.show_undocumented_params = False
|
|
|
|
# Copy generic params descriptions from mapservice_query if they
|
|
# are not overloaded by the query
|
|
for param in mapservice_endpoint.parameters:
|
|
if param in ('folder', 'service', 'layer', 'id_template') and getattr(self, param):
|
|
continue
|
|
if param == 'template' and self.text_template:
|
|
continue
|
|
endpoint.parameters[param] = mapservice_endpoint.parameters[param]
|
|
|
|
for ref, klass in self.where_references:
|
|
endpoint.parameters[ref] = {
|
|
'type': 'integer' if klass is int else 'string',
|
|
'example_value': '',
|
|
}
|
|
return endpoint
|