# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2017 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import html
import math
import xml.etree.ElementTree as ET

import pyproj
from django.contrib.postgres.fields import JSONField
from django.core.cache import cache
from django.db import models, transaction
from django.db.models import Q
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
from django.template import Context, Template, TemplateSyntaxError
from django.urls import reverse
from django.utils.text import slugify
from django.utils.translation import gettext_lazy as _

from passerelle.base.models import BaseQuery, BaseResource
from passerelle.utils.api import endpoint
from passerelle.utils.conversion import num2deg, simplify
from passerelle.utils.jsonresponse import APIError
from passerelle.utils.templates import validate_template


def build_dict_from_xml(elem):
    d = {}
    for child in elem.find('.'):
        if child.tag.startswith('{'):
            continue
        attribute_name = slugify(child.tag).replace('-', '_')
        if child.text and child.text.strip():
            d[attribute_name] = child.text.strip()
        else:
            d[attribute_name] = build_dict_from_xml(child)
    return d


PROJECTIONS = (
    ('EPSG:2154', _('EPSG:2154 (Lambert-93)')),
    ('EPSG:3857', _('EPSG:3857 (WGS 84 / Pseudo-Mercator)')),
    ('EPSG:3945', _('EPSG:3945 (CC45)')),
    ('EPSG:4326', _('EPSG:4326 (WGS 84)')),
)


class OpenGIS(BaseResource):
    category = _('Geographic information system')
    wms_service_url = models.URLField(_('Web Map Service (WMS) URL'), max_length=256, blank=True)
    wfs_service_url = models.URLField(_('Web Feature Service (WFS) URL'), max_length=256, blank=True)
    query_layer = models.CharField(
        _('Query Layer'),
        max_length=256,
        help_text=_('Corresponds to a WMS layer name and/or a WFS feature type.'),
        blank=True,
    )
    projection = models.CharField(
        _('GIS projection'), choices=PROJECTIONS, default='EPSG:3857', max_length=16
    )
    search_radius = models.IntegerField(_('Radius for point search'), default=5)
    attributes_mapping = (
        ('road', ('road', 'road_name', 'street', 'street_name', 'voie', 'nom_voie', 'rue')),
        ('city', ('city', 'city_name', 'town', 'town_name', 'commune', 'nom_commune', 'ville', 'nom_ville')),
        ('house_number', ('house_number', 'number', 'numero', 'numero_voie', 'numero_rue')),
        ('postcode', ('postcode', 'postalCode', 'zipcode', 'codepostal', 'cp', 'code_postal', 'code_post')),
        ('country', ('country', 'country_name', 'pays', 'nom_pays')),
    )

    class Meta:
        verbose_name = _('OpenGIS')

    def get_capabilities(self, service_type, service_url):
        if not service_url:
            raise APIError('no %s URL declared' % service_type)
        return self.requests.get(
            service_url,
            params={'request': 'GetCapabilities', 'service': service_type.upper()},
        )

    def get_service_version(self, service_type, service_url, renew=False):
        cache_key = 'opengis-%s-%s-version' % (service_type, self.id)
        if not renew:
            service_version = cache.get(cache_key)
            if service_version:
                return service_version
        response = self.get_capabilities(service_type, service_url)
        element = ET.fromstring(response.content)
        service_version = element.attrib.get('version')
        # cache version number for an hour
        cache.set(cache_key, service_version, 3600)
        return service_version

    def get_wms_service_version(self, renew=False):
        return self.get_service_version('wms', self.wms_service_url, renew=renew)

    def get_wfs_service_version(self, renew=False):
        return self.get_service_version('wfs', self.wfs_service_url, renew=renew)

    def get_output_format(self):
        cache_key = 'opengis-%s-output-format' % self.id
        output_format = cache.get(cache_key)
        if output_format:
            return output_format
        response = self.get_capabilities('wfs', self.wfs_service_url)
        element = ET.fromstring(response.content)
        ns = {'ows': 'http://www.opengis.net/ows/1.1'}
        formats = element.findall(
            './/ows:Operation[@name="GetFeature"]/'
            'ows:Parameter[@name="outputFormat"]/'
            'ows:AllowedValues/ows:Value',
            ns,
        )
        for output_format in formats:
            if 'json' in output_format.text.lower():
                cache.set(cache_key, output_format.text, 3600)
                return output_format.text
        raise APIError('WFS server doesn\'t support json output format for GetFeature request')

    def get_typename_label(self):
        version_str = self.get_wfs_service_version()
        version_tuple = tuple(int(x) for x in version_str.split('.'))
        if version_tuple <= (1, 1, 0):
            return 'typename'
        else:
            return 'typenames'

    def check_status(self):
        if self.wms_service_url:
            response = self.requests.get(
                self.wms_service_url, params={'service': 'WMS', 'request': 'GetCapabilities'}
            )
            response.raise_for_status()
        if self.wfs_service_url:
            response = self.requests.get(
                self.wfs_service_url, params={'service': 'WFS', 'request': 'GetCapabilities'}
            )
            response.raise_for_status()

    def build_get_features_params(self, typename=None, property_name=None, cql_filter=None, xml_filter=None):
        params = {
            'version': self.get_wfs_service_version(),
            'service': 'WFS',
            'request': 'GetFeature',
            self.get_typename_label(): typename or self.query_layer,
            'outputFormat': self.get_output_format(),
        }
        if property_name:
            params['propertyName'] = property_name
        if cql_filter:
            params['cql_filter'] = cql_filter
        if xml_filter:
            params['filter'] = xml_filter
        return params

    @endpoint(
        perm='can_access',
        description=_('Get features'),
        parameters={
            'type_names': {
                'description': _('Type of feature to query. Defaults to globally defined query layer'),
                'example_value': 'feature',
            },
            'property_name': {'description': _('Property to list'), 'example_value': 'nom_commune'},
            'cql_filter': {
                'description': _('CQL filter applied to the query'),
                'example_value': 'commune=\'Paris\'',
            },
            'filter_property_name': {
                'description': _('Property by which to filter'),
                'example_value': 'voie',
            },
            'q': {'description': _('Filter value'), 'example_value': 'rue du chateau'},
            'case_insensitive': {
                'description': _('Enables case-insensitive search'),
            },
            'xml_filter': {
                'description': _('Filter applied to the query'),
                'example_value': '<Filter><PropertyIsEqualTo><PropertyName>typeparking'
                '</PropertyName></PropertyIsEqualTo></Filter>',
            },
        },
    )
    def features(
        self,
        request,
        property_name,
        type_names=None,
        cql_filter=None,
        filter_property_name=None,
        q=None,
        case_insensitive=False,
        xml_filter=None,
        **kwargs,
    ):
        if cql_filter:
            if filter_property_name and q:
                if 'case-insensitive' in kwargs or case_insensitive:
                    operator = 'ILIKE'
                else:
                    operator = 'LIKE'
                cql_filter += ' AND %s %s \'%%%s%%\'' % (filter_property_name, operator, q)
        params = self.build_get_features_params(type_names, property_name, cql_filter, xml_filter)
        response = self.requests.get(self.wfs_service_url, params=params)
        data = []
        try:
            response = response.json()
        except ValueError:
            self.handle_opengis_error(response)
            # if handle_opengis_error did not raise an error, we raise a generic one
            raise APIError('OpenGIS Error: unparsable error', data={'content': repr(response.content[:1024])})
        for feature in response['features']:
            feature['text'] = feature['properties'].get(property_name)
            data.append(feature)
        return {'data': data}

    def handle_opengis_error(self, response):
        try:
            root = ET.fromstring(response.content)
        except ET.ParseError:
            return None
        if root.tag != '{http://www.opengis.net/ows/1.1}ExceptionReport':
            return None
        exception = root.find('{http://www.opengis.net/ows/1.1}Exception')
        exception_code = exception.attrib.get('exceptionCode')
        if exception is None:
            return None
        exception_text = exception.find('{http://www.opengis.net/ows/1.1}ExceptionText')
        if exception_text is None:
            return None
        content = exception_text.text
        content = html.unescape(content)
        raise APIError('OpenGIS Error: %s' % exception_code or 'unknown code', data={'text': content})

    def convert_coordinates(self, lon, lat, reverse=False):
        lon, lat = float(lon), float(lat)
        if self.projection != 'EPSG:4326':
            wgs84 = pyproj.Proj(init='EPSG:4326')
            target_projection = pyproj.Proj(init=self.projection)
            if reverse:
                # pylint: disable=unpacking-non-sequence
                lon, lat = pyproj.transform(  # pylint: disable=unpacking-non-sequence
                    target_projection, wgs84, lon, lat
                )
            else:
                lon, lat = pyproj.transform(  # pylint: disable=unpacking-non-sequence
                    wgs84, target_projection, lon, lat
                )
        return lon, lat

    def get_bbox(self, lon1, lat1, lon2, lat2):
        if self.projection == 'EPSG:4326':
            # send as is but invert coordinates
            return '%.6f,%.6f,%.6f,%.6f' % (lat1, lon1, lat2, lon2)
        wgs84 = pyproj.Proj(init='EPSG:4326')
        target_projection = pyproj.Proj(init=self.projection)
        # pylint: disable=unpacking-non-sequence
        x1, y1 = pyproj.transform(wgs84, target_projection, lon1, lat1)
        # pylint: disable=unpacking-non-sequence
        x2, y2 = pyproj.transform(wgs84, target_projection, lon2, lat2)
        return '%.6f,%.6f,%.6f,%.6f' % (x1, y1, x2, y2)

    @endpoint(
        perm='can_access',
        description=_('Get feature info'),
        parameters={
            'lat': {'description': _('Latitude'), 'example_value': '45.79689'},
            'lon': {'description': _('Longitude'), 'example_value': '4.78414'},
            'query_layer': {'description': _('Defaults to globally defined query layer')},
        },
    )
    def feature_info(self, request, lat, lon, query_layer=None):
        try:
            lat, lon = float(lat), float(lon)
        except ValueError:
            raise APIError('Bad coordinates format')
        bbox = self.get_bbox(lon - 0.0001, lat - 0.0001, lon + 0.0001, lat + 0.0001)
        params = {
            'version': '1.3.0',
            'service': 'WMS',
            'request': 'GetFeatureInfo',
            'info_format': 'application/vnd.ogc.gml',
            'styles': '',
            'i': '24',
            'J': '24',  # pixel in the middle of
            'height': '50',
            'WIDTH': '50',  # a 50x50 square
            'crs': self.projection,
            'layers': query_layer or self.query_layer,
            'query_layers': query_layer or self.query_layer,
            'bbox': bbox,
        }
        response = self.requests.get(self.wms_service_url, params=params)
        element = ET.fromstring(response.content)
        return {'err': 0, 'data': build_dict_from_xml(element)}

    # https://carton.entrouvert.org/hydda-tiles/16/33650/23378.pn
    @endpoint(
        name='tile',
        pattern=r'^(?P<zoom>\d+)/(?P<tile_x>\d+)/(?P<tile_y>\d+).png',
        description=_('Get Map Tile'),
        example_pattern='{zoom}/{tile_x}/{tile_y}.png',
        parameters={
            'zoom': {'description': _('Zoom Level'), 'example_value': '16'},
            'tile_x': {'description': _('X Coordinate'), 'example_value': '33650'},
            'tile_y': {'description': _('Y Coordinate'), 'example_value': '23378'},
            'query_layer': {'description': _('Defaults to globally defined query layer')},
        },
    )
    def tile(self, request, zoom, tile_x, tile_y, query_layer=None):
        zoom = int(zoom)

        tile_x = int(tile_x)
        tile_y = int(tile_y)

        # lower left
        ll_lon, ll_lat = num2deg(tile_x, tile_y + 1, zoom)
        # upper right
        ur_lon, ur_lat = num2deg(tile_x + 1, tile_y, zoom)

        bbox = self.get_bbox(ll_lon, ll_lat, ur_lon, ur_lat)

        params = {
            'version': '1.3.0',
            'service': 'wMS',
            'request': 'getMap',
            'layers': query_layer or self.query_layer,
            'styles': '',
            'format': 'image/png',
            'transparent': 'false',
            'height': '256',
            'width': '256',
            'crs': self.projection,
            'bbox': bbox,
        }
        response = self.requests.get(self.wms_service_url, params=params, cache_duration=300)
        return HttpResponse(response.content, content_type='image/png')

    @endpoint(
        perm='can_access',
        description=_('Get feature info'),
        parameters={
            'lat': {'description': _('Latitude'), 'example_value': '45.79689'},
            'lon': {'description': _('Longitude'), 'example_value': '4.78414'},
            'type_names': {
                'description': _('Type of feature to query. Defaults to globally defined query layer'),
                'example_value': 'feature',
            },
        },
    )
    def reverse(self, request, lat, lon, type_names=None, **kwargs):
        lon, lat = self.convert_coordinates(lon, lat)

        cql_filter = 'DWITHIN(the_geom,Point(%.6f %.6f),%s,meters)' % (lon, lat, self.search_radius)
        params = self.build_get_features_params(typename=type_names, cql_filter=cql_filter)
        response = self.requests.get(self.wfs_service_url, params=params)
        if not response.ok:
            raise APIError('Webservice returned status code %s' % response.status_code)
        closest_feature = {}
        min_delta = None
        for feature in response.json().get('features'):
            if not feature['geometry']['type'] == 'Point':
                continue  # skip unknown
            lon_diff = abs(float(lon) - float(feature['geometry']['coordinates'][0]))
            lat_diff = abs(float(lat) - float(feature['geometry']['coordinates'][1]))
            # compute hypotenuse til the point
            delta = math.sqrt(lon_diff * lon_diff + lat_diff * lat_diff)
            if min_delta is None:
                min_delta = delta
            # choose the shortest
            if delta <= min_delta:
                closest_feature = feature

        if closest_feature:
            result = {}
            point_lon = closest_feature['geometry']['coordinates'][0]
            point_lat = closest_feature['geometry']['coordinates'][1]
            point_lon, point_lat = self.convert_coordinates(point_lon, point_lat, reverse=True)
            result['lon'] = "%.6f" % point_lon
            result['lat'] = "%.6f" % point_lat
            result['address'] = {}

            for attribute, properties in self.attributes_mapping:
                for field in properties:
                    if closest_feature['properties'].get(field):
                        result['address'][attribute] = str(closest_feature['properties'][field])
                        break
            return result
        raise APIError('Unable to geocode')

    @endpoint(
        name='query',
        description=_('Query'),
        pattern=r'^(?P<query_slug>[\w:_-]+)/$',
        perm='can_access',
        parameters={
            'bbox': {
                'description': _(
                    'Only include results inside bounding box. Expected coordinates '
                    'format is lonmin,latmin,lonmax,latmax'
                ),
            },
            'circle': {
                'description': _(
                    'Only include results inside circle. Expected coordinates '
                    'format is center_longitude,center_latitude,radius ("radius" '
                    'being a distance in meters)'
                ),
            },
            'q': {
                'description': _('Text search for specified properties'),
            },
            'property': {
                'description': _(
                    'Filter on any GeoJSON property value. If a feature has an "id" property, '
                    'format will be property:id=value.'
                ),
                'optional': True,
            },
        },
        show=False,
    )
    def query(self, request, query_slug, bbox=None, circle=None, q=None, **kwargs):
        if bbox and circle:
            raise APIError('bbox and circle parameters are mutually exclusive')
        for param in kwargs:
            if not param.startswith('property'):
                raise APIError('extra parameter: %s' % param)
        query = get_object_or_404(Query, resource=self, slug=query_slug)
        return query.q(request, bbox, q, circle, **kwargs)

    def export_json(self):
        d = super().export_json()
        d['queries'] = [query.export_json() for query in self.queries.all()]
        return d

    @classmethod
    def import_json_real(cls, overwrite, instance, d, **kwargs):
        queries = d.pop('queries', [])
        instance = super().import_json_real(overwrite, instance, d, **kwargs)
        new = []
        if instance and overwrite:
            Query.objects.filter(resource=instance).delete()
        for query in queries:
            q = Query.import_json(query)
            q.resource = instance
            new.append(q)
        Query.objects.bulk_create(new)
        return instance

    def daily(self):
        super().daily()
        self.update_queries()

    def update_queries(self, query_id=None):
        queries = self.queries.all()
        if query_id:
            queries = queries.filter(pk=query_id)
        for query in queries:
            try:
                query.update_cache()
            except APIError:
                self.logger.exception('failed to cache results for query %s', query)

    def create_query_url(self):
        return reverse('opengis-query-new', kwargs={'slug': self.slug})

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        if self.queries.exists():
            self.add_job('update_queries')


class Query(BaseQuery):
    resource = models.ForeignKey(
        to=OpenGIS, on_delete=models.CASCADE, related_name='queries', verbose_name=_('Resource')
    )

    typename = models.CharField(_('Feature type'), max_length=256)
    filter_expression = models.TextField(_('XML filter'), blank=True)
    indexing_template = models.TextField(
        verbose_name=_('Indexing template'), blank=True, validators=[validate_template]
    )
    computed_properties = JSONField(blank=True, default=dict)

    delete_view = 'opengis-query-delete'
    edit_view = 'opengis-query-edit'

    def as_endpoint(self):
        resource_endpoint = self.resource.query.endpoint_info
        endpoint = super().as_endpoint(path=resource_endpoint.name)
        # use parameters added by resource endpoint
        endpoint.func = resource_endpoint.func
        endpoint.show_undocumented_params = False
        endpoint.parameters = resource_endpoint.parameters
        return endpoint

    def q(self, request, bbox, q, circle, **kwargs):
        features = self.features.all()
        if not features.exists():
            raise APIError(
                'No data. (maybe not synchronized yet? retry in a few minutes.)', extra_dict={'features': []}
            )

        filters = {}
        for lookup, value in kwargs.items():
            lookup = lookup.replace('property', 'data__properties').replace(':', '__')
            converted_value = self.cast_str(value)
            if converted_value:
                lookup += '__in'
                filters[lookup] = [value, converted_value]
            else:
                filters[lookup] = value
        features = features.filter(**filters)

        lonmin, latmin, lonmax, latmax = None, None, None, None
        if bbox:
            try:
                lonmin, latmin, lonmax, latmax = (float(x) for x in bbox.split(','))
            except (ValueError, AttributeError):
                raise APIError(
                    'Invalid bbox parameter, it must be a comma separated list of floating point numbers.'
                )

        if circle:
            try:
                center_lon, center_lat, radius = (float(x) for x in circle.split(','))
            except (ValueError, AttributeError):
                raise APIError(
                    'Invalid circle parameter, it must be a comma separated list of '
                    'floating point numbers.'
                )
            coords = self.get_bbox_containing_circle(center_lon, center_lat, radius)
            lonmin, latmin, lonmax, latmax = coords

        if lonmin is not None:
            # adjust lonmin, latmin, lonmax, latmax to make sure min are min and max are max
            lonmin, lonmax = min(lonmin, lonmax), max(lonmin, lonmax)
            latmin, latmax = min(latmin, latmax), max(latmin, latmax)

            features = features.filter(
                Q(bbox_lat2__isnull=True, lon__gte=lonmin, lon__lte=lonmax, lat__gte=latmin, lat__lte=latmax)
                | Q(
                    bbox_lat2__isnull=False,  # geometry != Point
                    lon__lte=lonmax,
                    bbox_lon2__gte=lonmin,
                    lat__lte=latmax,
                    bbox_lat2__gte=latmin,
                )
            )

        if q:
            features = features.filter(text__search=simplify(q))

        data = {'type': 'FeatureCollection', 'name': self.typename}
        if circle:
            results = []
            for feature in features:
                if feature.bbox_lat2:  # not a point
                    results.append(feature.data)
                else:
                    distance = self.get_coords_distance(feature.lat, feature.lon, center_lat, center_lon)
                    if distance < radius:
                        results.append(feature.data)

            data['features'] = results
        else:
            data['features'] = list(features.values_list('data', flat=True))
        return data

    @staticmethod
    def get_bbox_containing_circle(lon, lat, r):
        earth_radius = 6378137
        offset_in_radian = 180 / math.pi * (r / earth_radius)
        lonmin = lon - offset_in_radian
        lonmax = lon + offset_in_radian
        latmin = lat - offset_in_radian
        latmax = lat + offset_in_radian
        return lonmin, latmin, lonmax, latmax

    @staticmethod
    def get_coords_distance(lat, lon, lat0, lon0):
        # simplest distance approximation https://www.mkompf.com/gps/distcalc.html
        deglen = 111300
        x = lat - lat0
        lat = (lat + lat0) / 2 * 0.01745
        y = (lon - lon0) * math.cos(lat)
        return deglen * math.sqrt(x * x + y * y)

    @staticmethod
    def cast_str(s):
        if s.lower() == 'true':
            return True
        if s.lower() == 'false':
            return False
        try:
            return int(s)
        except ValueError:
            try:
                return float(s)
            except ValueError:
                return None

    def update_cache(self):
        data = self.resource.features(None, None, type_names=self.typename, xml_filter=self.filter_expression)
        features = []
        templates = {}
        if self.indexing_template:
            templates['text'] = Template(self.indexing_template)
        for key, tplt in (self.computed_properties or {}).items():
            try:
                templates['computed_property_%s' % key] = Template(tplt)
            except TemplateSyntaxError:
                pass

        def add_coordinates(coordinates):
            nonlocal min_lat, min_lon, max_lat, max_lon
            if not coordinates:
                return
            if not isinstance(coordinates[0], (float, int)):
                for child in coordinates:
                    add_coordinates(child)
                return

            # position
            lon, lat = coordinates
            if min_lat is None or lat < min_lat:
                min_lat = lat
            if max_lat is None or lat > max_lat:
                max_lat = lat
            if min_lon is None or lon < min_lon:
                min_lon = lon
            if max_lon is None or lon > max_lon:
                max_lon = lon

        for feature in data['data']:
            geometry = feature.get('geometry') or {}
            if not geometry:
                continue
            if geometry.get('type') == 'Point':
                try:
                    lon, lat = geometry['coordinates']
                except (KeyError, TypeError):
                    self.resource.logger.warning('invalid coordinates in geometry: %s', geometry)
                    continue
                lon2, lat2 = None, None
            else:
                # define bbox, lat/lon as min values, bbox_lat2/bbox_lon2 as max values
                min_lat, min_lon, max_lat, max_lon = None, None, None, None

                add_coordinates(geometry['coordinates'])
                lat, lon, lat2, lon2 = min_lat, min_lon, max_lat, max_lon

            text = ''
            if self.indexing_template:
                context = Context(feature.get('properties', {}))
                text = simplify(templates['text'].render(context))
            context = Context(feature)
            for key in self.computed_properties or {}:
                if not templates.get('computed_property_%s' % key):
                    continue
                if not feature.get('properties'):
                    feature['properties'] = {}
                feature['properties'][key] = templates['computed_property_%s' % key].render(context)
            features.append(
                FeatureCache(
                    query=self,
                    lat=lat,
                    lon=lon,
                    bbox_lat2=lat2,
                    bbox_lon2=lon2,
                    text=text,
                    data=feature,
                )
            )
        with transaction.atomic():
            self.features.all().delete()
            FeatureCache.objects.bulk_create(features)

    def save(self, *args, **kwargs):
        super().save(*args, **kwargs)
        self.resource.add_job('update_queries', query_id=self.pk)


class FeatureCache(models.Model):
    query = models.ForeignKey(
        to=Query, on_delete=models.CASCADE, related_name='features', verbose_name=_('Query')
    )
    lat = models.FloatField()
    lon = models.FloatField()
    bbox_lat2 = models.FloatField(blank=True, null=True)
    bbox_lon2 = models.FloatField(blank=True, null=True)
    text = models.CharField(max_length=2048)
    data = JSONField()
