511 lines
19 KiB
Python
511 lines
19 KiB
Python
# passerelle - uniform access to multiple data sources and services
|
|
# Copyright (C) 2018 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import string
|
|
from urllib import parse as urlparse
|
|
|
|
from django.core.exceptions import ValidationError
|
|
from django.db import models
|
|
from django.http import HttpResponse
|
|
from django.shortcuts import get_object_or_404
|
|
from django.urls import reverse
|
|
from django.utils.text import slugify
|
|
from django.utils.translation import gettext_lazy as _
|
|
|
|
from passerelle.base.models import BaseQuery, BaseResource, HTTPResource
|
|
from passerelle.utils import mark_safe_lazy
|
|
from passerelle.utils.api import endpoint
|
|
from passerelle.utils.conversion import num2deg
|
|
from passerelle.utils.jsonresponse import APIError
|
|
from passerelle.utils.templates import render_to_string, validate_template
|
|
|
|
|
|
class ArcGISError(APIError):
|
|
pass
|
|
|
|
|
|
class ArcGIS(BaseResource, HTTPResource):
|
|
category = _('Geographic information system')
|
|
|
|
base_url = models.URLField(_('Webservice Base URL'))
|
|
|
|
token_username = models.CharField(
|
|
max_length=128, verbose_name=_('User name of the user who wants to get a token'), blank=True
|
|
)
|
|
token_password = models.CharField(
|
|
max_length=128, verbose_name=_('Password of the user who wants to get a token'), blank=True
|
|
)
|
|
|
|
class Meta:
|
|
verbose_name = _('ArcGIS REST API')
|
|
|
|
def request(self, url, params=None, data=None, query_type='query'):
|
|
if data:
|
|
response = self.requests.post(url, params=params, data=data)
|
|
else:
|
|
response = self.requests.get(url, params=params, data=data)
|
|
if response.status_code // 100 != 2:
|
|
raise ArcGISError('ArcGIS/%s returned status code %s' % (query_type, response.status_code))
|
|
try:
|
|
response = response.json()
|
|
except ValueError:
|
|
raise ArcGISError('ArcGIS/%s returned invalid JSON content: %r' % (query_type, response.content))
|
|
if not isinstance(response, dict):
|
|
raise ArcGISError('ArcGIS/%s not returned a dict: %r' % (query_type, response))
|
|
if 'error' in response:
|
|
if not isinstance(response['error'], dict) or 'message' not in response['error']:
|
|
err_desc = 'unknown ArcGIS/%s error' % query_type
|
|
else:
|
|
err_desc = response['error']['message']
|
|
raise ArcGISError(err_desc, data=response)
|
|
return response
|
|
|
|
def generate_token(self):
|
|
if not self.token_username and not self.token_password:
|
|
return None
|
|
url = urlparse.urljoin(self.base_url, 'info')
|
|
info = self.request(url, params={'f': 'json'}, query_type='token')
|
|
token_url = info.get('authInfo', {}).get('tokenServicesUrl')
|
|
if not token_url:
|
|
raise ArcGISError('ArcGIS/token responded no authInfo/tokenServicesUrl in info: %r' % info)
|
|
response = self.request(
|
|
token_url,
|
|
data={
|
|
'username': self.token_username,
|
|
'password': self.token_password,
|
|
'client': 'referer',
|
|
'referer': urlparse.urljoin(self.base_url, 'services'),
|
|
'f': 'json',
|
|
},
|
|
query_type='token',
|
|
)
|
|
if 'token' not in response:
|
|
raise ArcGISError('ArcGIS/token returned no token: %r' % response)
|
|
return response['token']
|
|
|
|
def build_common_params(self, lat, lon, latmin, lonmin, latmax, lonmax, **kwargs):
|
|
# build common query params, see:
|
|
# https://developers.arcgis.com/rest/services-reference/query-map-service-layer-.htm
|
|
# https://developers.arcgis.com/rest/services-reference/enterprise/query-feature-service-layer-.htm
|
|
params = {
|
|
'f': 'json',
|
|
'inSR': '4326',
|
|
'outSR': '4326',
|
|
'outFields': '*',
|
|
}
|
|
if lat and lon:
|
|
try:
|
|
lon, lat = float(lon), float(lat)
|
|
except (ValueError,):
|
|
raise APIError('<lon> and <lat> must be floats', http_status=400)
|
|
params['geometry'] = f'{lon},{lat}'
|
|
params['geometryType'] = 'esriGeometryPoint'
|
|
elif latmin and lonmin and latmax and lonmax:
|
|
try:
|
|
lonmin, latmin = float(lonmin), float(latmin)
|
|
lonmax, latmax = float(lonmax), float(latmax)
|
|
except (ValueError,):
|
|
raise APIError('<lonmin> <latmin> <lonmax> and <latmax> must be floats', http_status=400)
|
|
params['geometry'] = f'{lonmin},{latmin},{lonmax},{latmax}'
|
|
params['geometryType'] = 'esriGeometryEnvelope'
|
|
# consider all remaining parameters as ArcGIS ones
|
|
params.update(kwargs)
|
|
if 'distance' in params and 'units' not in params:
|
|
params['units'] = 'esriSRUnit_Meter'
|
|
# add a token if applicable
|
|
if 'token' not in params:
|
|
token = self.generate_token()
|
|
if token is not None:
|
|
params['token'] = token
|
|
return params
|
|
|
|
def get_query_response(self, uri, params, id_template, text_template, full, text_fieldname=None):
|
|
url = urlparse.urljoin(self.base_url, uri)
|
|
infos = self.request(url, params=params)
|
|
|
|
features = infos.pop('features', [])
|
|
id_fieldname = infos.get('objectIdFieldName') or 'OBJECTID'
|
|
text_fieldname = text_fieldname or infos.get('displayFieldName')
|
|
aliases = {}
|
|
for field in infos.get('fields') or []:
|
|
if field.get('alias') and field.get('name'):
|
|
aliases[field['alias']] = field['name']
|
|
if infos.get('fieldAliases'):
|
|
for name, alias in infos['fieldAliases'].items():
|
|
aliases[alias] = name
|
|
|
|
data = []
|
|
|
|
def get_feature_attribute(feature, attribute):
|
|
if attribute in feature['attributes']:
|
|
return feature['attributes'][attribute]
|
|
return feature['attributes'].get(aliases.get(attribute))
|
|
|
|
for n, feature in enumerate(features):
|
|
if 'attributes' in feature:
|
|
feature['id'] = '%s' % get_feature_attribute(feature, id_fieldname)
|
|
feature['text'] = (
|
|
'%s' % get_feature_attribute(feature, text_fieldname) if text_fieldname else feature['id']
|
|
)
|
|
else:
|
|
feature['id'] = feature['text'] = '%d' % (n + 1)
|
|
if id_template:
|
|
feature['id'] = render_to_string(id_template, feature)
|
|
if text_template:
|
|
feature['text'] = render_to_string(text_template, feature)
|
|
if not full and 'geometry' in feature:
|
|
del feature['geometry']
|
|
data.append(feature)
|
|
|
|
if full:
|
|
return {'data': data, 'metadata': infos}
|
|
return {'data': data}
|
|
|
|
@endpoint(
|
|
name='mapservice-query',
|
|
description=_('Map Service Query'),
|
|
perm='can_access',
|
|
parameters={
|
|
'folder': {
|
|
'description': _('Folder name'),
|
|
'example_value': 'Specialty',
|
|
},
|
|
'service': {
|
|
'description': _('Service name'),
|
|
'example_value': 'ESRI_StateCityHighway_USA',
|
|
},
|
|
'layer': {
|
|
'description': _('Layer or table name'),
|
|
'example_value': '1',
|
|
},
|
|
'lat': {'description': _('Latitude')},
|
|
'lon': {'description': _('Longitude')},
|
|
'latmin': {'description': _('Minimal latitude (envelope)')},
|
|
'lonmin': {'description': _('Minimal longitude (envelope)')},
|
|
'latmax': {'description': _('Maximal latitude (envelope)')},
|
|
'lonmax': {'description': _('Maximal longitude (envelope)')},
|
|
'q': {'description': _('Search text in display field')},
|
|
'template': {
|
|
'description': _('Django template for text attribute'),
|
|
'example_value': '{{ attributes.STATE_NAME }} ({{ attributes.STATE_ABBR }})',
|
|
},
|
|
'id_template': {
|
|
'description': _('Django template for id attribute'),
|
|
},
|
|
'full': {
|
|
'description': _('Returns all ArcGIS informations (geometry, metadata)'),
|
|
'type': 'bool',
|
|
},
|
|
},
|
|
)
|
|
def mapservice_query(
|
|
self,
|
|
request,
|
|
service,
|
|
layer='0',
|
|
folder='',
|
|
lat=None,
|
|
lon=None,
|
|
latmin=None,
|
|
lonmin=None,
|
|
latmax=None,
|
|
lonmax=None,
|
|
q=None,
|
|
template=None,
|
|
id_template=None,
|
|
full=False,
|
|
**kwargs,
|
|
):
|
|
uri = 'services/'
|
|
if folder:
|
|
uri += folder + '/'
|
|
uri = uri + service + '/MapServer/' + layer + '/query'
|
|
|
|
params = self.build_common_params(lat, lon, latmin, lonmin, latmax, lonmax, **kwargs)
|
|
if q is not None and 'text' not in params:
|
|
params['text'] = q
|
|
if 'where' not in params and 'text' not in params:
|
|
params['where'] = '1=1'
|
|
|
|
return self.get_query_response(
|
|
uri, params, id_template=id_template, text_template=template, full=full
|
|
)
|
|
|
|
@endpoint(
|
|
name='featureservice-query',
|
|
description=_('Feature Service Query'),
|
|
perm='can_access',
|
|
parameters={
|
|
'folder': {
|
|
'description': _('Folder name'),
|
|
'example_value': 'Specialty',
|
|
},
|
|
'service': {
|
|
'description': _('Service name'),
|
|
'example_value': 'ESRI_StateCityHighway_USA',
|
|
},
|
|
'layer': {
|
|
'description': _('Layer or table name'),
|
|
'example_value': '1',
|
|
},
|
|
'lat': {'description': _('Latitude')},
|
|
'lon': {'description': _('Longitude')},
|
|
'latmin': {'description': _('Minimal latitude (envelope)')},
|
|
'lonmin': {'description': _('Minimal longitude (envelope)')},
|
|
'latmax': {'description': _('Maximal latitude (envelope)')},
|
|
'lonmax': {'description': _('Maximal longitude (envelope)')},
|
|
'text_fieldname': {
|
|
'description': _('Field name for text attribute'),
|
|
'example_value': 'STATE_NAME',
|
|
},
|
|
'template': {
|
|
'description': _('Django template for text attribute'),
|
|
'example_value': '{{ attributes.STATE_NAME }} ({{ attributes.STATE_ABBR }})',
|
|
},
|
|
'id_template': {
|
|
'description': _('Django template for id attribute'),
|
|
},
|
|
'full': {
|
|
'description': _('Returns all ArcGIS informations (geometry, metadata)'),
|
|
'type': 'bool',
|
|
},
|
|
},
|
|
)
|
|
def featureservice_query(
|
|
self,
|
|
request,
|
|
service,
|
|
layer='0',
|
|
folder='',
|
|
lat=None,
|
|
lon=None,
|
|
latmin=None,
|
|
lonmin=None,
|
|
latmax=None,
|
|
lonmax=None,
|
|
text_fieldname=None,
|
|
template=None,
|
|
id_template=None,
|
|
full=False,
|
|
**kwargs,
|
|
):
|
|
uri = 'services/'
|
|
if folder:
|
|
uri += folder + '/'
|
|
uri = uri + service + '/FeatureServer/' + layer + '/query'
|
|
|
|
params = self.build_common_params(lat, lon, latmin, lonmin, latmax, lonmax, **kwargs)
|
|
|
|
return self.get_query_response(
|
|
uri,
|
|
params,
|
|
id_template=id_template,
|
|
text_template=template,
|
|
full=full,
|
|
text_fieldname=text_fieldname,
|
|
)
|
|
|
|
@endpoint(
|
|
name='tile',
|
|
description=_('Tiles layer'),
|
|
pattern=r'^(?P<layer>[\w/]+)/(?P<zoom>\d+)/(?P<tile_x>\d+)/(?P<tile_y>\d+)\.png$',
|
|
)
|
|
def tile(self, request, layer, zoom, tile_x, tile_y):
|
|
zoom = int(zoom)
|
|
tile_x = int(tile_x)
|
|
tile_y = int(tile_y)
|
|
|
|
bbox = '%.6f,%.6f,%.6f,%.6f' % (num2deg(tile_x, tile_y, zoom) + num2deg(tile_x + 1, tile_y + 1, zoom))
|
|
|
|
# imageSR=3857: default projection for leaflet
|
|
base_url = self.base_url
|
|
if not base_url.endswith('/'):
|
|
base_url += '/'
|
|
return HttpResponse(
|
|
self.requests.get(
|
|
base_url
|
|
+ '%s/MapServer/export' % layer
|
|
+ '?dpi=96&format=png24&bboxSR=4326&imageSR=3857&'
|
|
+ 'transparent=true&size=256,256&f=image&'
|
|
+ 'bbox=%s' % bbox
|
|
).content,
|
|
content_type='image/png',
|
|
)
|
|
|
|
@endpoint(
|
|
name='q',
|
|
description=_('Query'),
|
|
pattern=r'^(?P<query_slug>[\w:_-]+)/$',
|
|
perm='can_access',
|
|
show=False,
|
|
)
|
|
def q(self, request, query_slug, q=None, full=False, **kwargs):
|
|
query = get_object_or_404(Query, resource=self, slug=query_slug)
|
|
refs = [ref for ref, _ in query.where_references]
|
|
refs += ['q', 'full']
|
|
kwargs = {}
|
|
for key in request.GET:
|
|
if key not in refs:
|
|
kwargs[key] = request.GET[key]
|
|
return query.q(request, q=None, full=full, **kwargs)
|
|
|
|
def export_json(self):
|
|
d = super().export_json()
|
|
d['queries'] = [query.export_json() for query in self.queries.all()]
|
|
return d
|
|
|
|
@classmethod
|
|
def import_json_real(cls, overwrite, instance, d, **kwargs):
|
|
queries = d.pop('queries', [])
|
|
instance = super().import_json_real(overwrite, instance, d, **kwargs)
|
|
new = []
|
|
if instance and overwrite:
|
|
Query.objects.filter(resource=instance).delete()
|
|
for query in queries:
|
|
q = Query.import_json(query)
|
|
q.resource = instance
|
|
new.append(q)
|
|
Query.objects.bulk_create(new)
|
|
return instance
|
|
|
|
def create_query_url(self):
|
|
return reverse('arcgis-query-new', kwargs={'slug': self.slug})
|
|
|
|
|
|
class SqlFormatter(string.Formatter):
|
|
def format_field(self, value, format_spec):
|
|
if format_spec and format_spec[-1].isalpha() and format_spec[-1] == 'd':
|
|
value = int(value)
|
|
if format_spec and format_spec[-1] == 'l':
|
|
return '(' + ', '.join(SqlFormatter().format('{x}', x=x) for x in value.split(',')) + ')'
|
|
formatted = super().format_field(value, format_spec)
|
|
if not format_spec or not format_spec[-1].isalpha() or format_spec[-1] == 's':
|
|
formatted = "'%s'" % formatted.replace("'", "''")
|
|
return formatted
|
|
|
|
|
|
def validate_where(format_string):
|
|
formatter = SqlFormatter()
|
|
for dummy, ref, dummy, dummy in formatter.parse(format_string):
|
|
if ref is None:
|
|
pass
|
|
elif ref == '':
|
|
raise ValidationError(_('missing reference'))
|
|
elif ref != slugify(ref):
|
|
raise ValidationError(_('invalid reference'))
|
|
|
|
|
|
class Query(BaseQuery):
|
|
resource = models.ForeignKey(
|
|
to=ArcGIS, related_name='queries', verbose_name=_('Resource'), on_delete=models.CASCADE
|
|
)
|
|
|
|
folder = models.CharField(verbose_name=_('ArcGis Folder'), max_length=64, blank=True)
|
|
service = models.CharField(verbose_name=_('ArcGis Service'), max_length=64)
|
|
layer = models.CharField(verbose_name=_('ArcGis Layer'), max_length=8, blank=True)
|
|
|
|
where = models.TextField(
|
|
verbose_name=_('ArcGis Where Clause'),
|
|
blank=True,
|
|
validators=[validate_where],
|
|
help_text=mark_safe_lazy(
|
|
_(
|
|
'<div>Use syntax <tt>{name}</tt> to introduce a string '
|
|
'parameter and <tt>{name:d}</tt> for a decimal parameter. '
|
|
'Use syntax <tt>{name:l}</tt> for a parameter that should be '
|
|
'interpreted a a list of strings (comma-separated). ex.:</div>'
|
|
'<tt>adress LIKE (\'%\' || UPPER({adress}) || \'%\')</tt><br>'
|
|
'<tt>population < {population:d}</tt><br>'
|
|
'<tt>ID IN {ids:l}</tt> (with ids being "11,13,17")'
|
|
)
|
|
),
|
|
)
|
|
|
|
id_template = models.TextField(
|
|
verbose_name=_('Id template'),
|
|
validators=[validate_template],
|
|
help_text=_(
|
|
'Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}'
|
|
),
|
|
blank=True,
|
|
)
|
|
|
|
text_template = models.TextField(
|
|
verbose_name=_('Text template'),
|
|
help_text=_(
|
|
'Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}'
|
|
),
|
|
validators=[validate_template],
|
|
blank=True,
|
|
)
|
|
|
|
delete_view = 'arcgis-query-delete'
|
|
edit_view = 'arcgis-query-edit'
|
|
|
|
@property
|
|
def where_references(self):
|
|
if self.where:
|
|
return [
|
|
(ref, int if spec and spec[-1] == 'd' else str)
|
|
for _, ref, spec, _ in SqlFormatter().parse(self.where)
|
|
if ref is not None
|
|
]
|
|
else:
|
|
return []
|
|
|
|
def q(self, request, q=None, full=False, **kwargs):
|
|
kwargs.update(
|
|
{
|
|
'service': self.service,
|
|
}
|
|
)
|
|
if self.id_template:
|
|
kwargs['id_template'] = self.id_template
|
|
if self.text_template:
|
|
kwargs['template'] = self.text_template
|
|
if self.folder:
|
|
kwargs['folder'] = self.folder
|
|
if self.layer:
|
|
kwargs['layer'] = self.layer
|
|
if self.where:
|
|
format_kwargs = {key: request.GET.get(key, '') for key, klass in self.where_references}
|
|
formatter = SqlFormatter()
|
|
kwargs['where'] = formatter.format(self.where, **format_kwargs)
|
|
return self.resource.mapservice_query(request, q=q, full=full, **kwargs)
|
|
|
|
def as_endpoint(self):
|
|
endpoint = super().as_endpoint(path=self.resource.q.endpoint_info.name)
|
|
|
|
mapservice_endpoint = self.resource.mapservice_query.endpoint_info
|
|
endpoint.func = mapservice_endpoint.func
|
|
endpoint.show_undocumented_params = False
|
|
|
|
# Copy generic params descriptions from mapservice_query if they
|
|
# are not overloaded by the query
|
|
for param in mapservice_endpoint.parameters:
|
|
if param in ('folder', 'service', 'layer', 'id_template') and getattr(self, param):
|
|
continue
|
|
if param == 'template' and self.text_template:
|
|
continue
|
|
endpoint.parameters[param] = mapservice_endpoint.parameters[param]
|
|
|
|
for ref, klass in self.where_references:
|
|
endpoint.parameters[ref] = {
|
|
'type': 'integer' if klass is int else 'string',
|
|
'example_value': '',
|
|
}
|
|
return endpoint
|