passerelle/passerelle/views.py

# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2019 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import hashlib
import inspect
import json
import logging
import uuid
from urllib.parse import quote

from dateutil import parser as date_parser
from django.apps import apps
from django.conf import settings
from django.conf.urls import url
from django.contrib.auth import logout as auth_logout
from django.contrib.auth import views as auth_views
from django.core.cache import cache
from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.db.models import Q
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, resolve_url
from django.urls import reverse
from django.utils.encoding import force_bytes, force_text
from django.utils.timezone import is_naive, make_aware
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import (
    CreateView,
    DeleteView,
    DetailView,
    ListView,
    RedirectView,
    TemplateView,
    UpdateView,
    View,
)
from django.views.generic.detail import SingleObjectMixin
from jsonschema import ValidationError, validate, validators

from passerelle.base.models import BaseResource, ResourceLog
from passerelle.utils.conversion import normalize
from passerelle.utils.json import unflatten
from passerelle.utils.jsonresponse import APIError, JSONEncoder
from passerelle.utils.paginator import InfinitePaginator

from .forms import ResourceLogSearchForm
from .utils import is_authorized, to_json

if 'mellon' in settings.INSTALLED_APPS:
    from mellon.utils import get_idps  # pylint: disable=import-error
else:

    def get_idps():
        return []


def get_all_apps():
    return [x for x in apps.get_models() if issubclass(x, BaseResource) and x.is_enabled()]


class LoginView(auth_views.LoginView):
    def dispatch(self, request, *args, **kwargs):
        if any(get_idps()):
            if 'next' not in request.GET:
                return HttpResponseRedirect(resolve_url('mellon_login'))
            return HttpResponseRedirect(
                resolve_url('mellon_login') + '?next=' + quote(request.GET.get('next'))
            )
        return super().dispatch(request, *args, **kwargs)


login = LoginView.as_view()


def logout(request, next_page=None):
    if any(get_idps()):
        return HttpResponseRedirect(resolve_url('mellon_logout'))
    auth_logout(request)
    if next_page is not None:
        next_page = resolve_url(next_page)
    else:
        next_page = '/'
    return HttpResponseRedirect(next_page)


def menu_json(request):
    label = _('Web Services')
    json_str = json.dumps(
        [
            {
                'label': force_text(label),
                'slug': 'passerelle',
                'url': request.build_absolute_uri(reverse('manage-home')),
            }
        ]
    )
    content_type = 'application/json'
    for variable in ('jsonpCallback', 'callback'):
        if variable in request.GET:
            json_str = '%s(%s);' % (request.GET[variable], json_str)
            content_type = 'application/javascript'
            break
    response = HttpResponse(content_type=content_type)
    response.write(json_str)
    return response


class HomePageView(RedirectView):
    pattern_name = 'manage-home'
    permanent = False


class ManageView(TemplateView):
    template_name = 'passerelle/manage.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        # get all app instances
        context['apps'] = []
        for app in get_all_apps():
            context['apps'].extend(app.objects.all())
        context['apps'].sort(key=lambda x: x.title.lower())
        return context


class ManageAddView(TemplateView):
    template_name = 'passerelle/manage_add.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['apps'] = [x for x in get_all_apps() if not x.is_legacy()]
        context['apps'].sort(key=lambda x: x.get_verbose_name())
        return context


class GenericConnectorMixin:
    exclude_fields = ('users',)

    def get_connector(self, **kwargs):
        return kwargs.get('connector')

    def get_form_class(self):
        return self.model.get_manager_form_class(exclude=self.exclude_fields)

    def init_stuff(self, request, *args, **kwargs):
        connector = self.get_connector(**kwargs)
        for app in apps.get_app_configs():
            if not hasattr(app, 'get_connector_model'):
                continue
            if app.get_connector_model().get_connector_slug() == connector:
                break
        else:
            raise Http404()

        self.model = app.get_connector_model()

    def dispatch(self, request, *args, **kwargs):
        self.init_stuff(request, *args, **kwargs)
        return super().dispatch(request, *args, **kwargs)


class GenericConnectorView(GenericConnectorMixin, DetailView):
    def get_context_data(self, slug=None, **kwargs):
        context = super().get_context_data(**kwargs)
        context['has_check_status'] = not hasattr(context['object'].check_status, 'not_implemented')
        context['default_tab_page'] = 'endpoints'
        if 'page' in self.request.GET:
            context['default_tab_page'] = 'logs'
        return context

    def get_template_names(self):
        template_names = super(DetailView, self).get_template_names()[:]
        if self.model.manager_view_template_name:
            template_names.append(self.model.manager_view_template_name)
        template_names.append('passerelle/manage/service_view.html')
        return template_names


class GenericCreateConnectorView(GenericConnectorMixin, CreateView):
    template_name = 'passerelle/manage/service_form.html'
    exclude_fields = ('users',)  # slug not excluded

    def form_valid(self, form):
        with transaction.atomic():
            response = super().form_valid(form)
        self.object.availability()
        return response


class GenericEditConnectorView(GenericConnectorMixin, UpdateView):
    template_name = 'passerelle/manage/service_form.html'

    def form_valid(self, form):
        with transaction.atomic():
            response = super().form_valid(form)
        self.object.availability()
        return response


class GenericDeleteConnectorView(GenericConnectorMixin, DeleteView):
    template_name = 'passerelle/manage/service_confirm_delete.html'

    def get_success_url(self):
        return reverse('manage-home')


class GenericViewLogsConnectorView(GenericConnectorMixin, ListView):
    template_name = 'passerelle/manage/service_logs.html'
    paginate_by = 25
    paginator_class = InfinitePaginator

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['object'] = self.get_object()
        context['form'] = self.form
        if self.request.GET.get('log_id'):
            try:
                context['log_target'] = ResourceLog.objects.get(
                    appname=self.kwargs['connector'], slug=self.kwargs['slug'], pk=self.request.GET['log_id']
                )
            except (ValueError, ResourceLog.DoesNotExist):
                pass
        return context

    def get_object(self):
        return get_object_or_404(self.model.objects.all(), slug=self.kwargs['slug'])

    def get_queryset(self):
        self.form = ResourceLogSearchForm(data=self.request.GET)
        qs = ResourceLog.objects.filter(appname=self.kwargs['connector'], slug=self.kwargs['slug']).order_by(
            '-timestamp'
        )
        query = None
        level = None
        if self.form.is_valid():
            query = self.form.cleaned_data['q']
            level = self.form.cleaned_data['log_level']
        if level:
            qs = qs.filter(levelno=logging.getLevelName(level))
        if query:
            try:
                date = date_parser.parse(query, dayfirst=True)
            except Exception:
                query_uuid = None
                try:
                    query_uuid = uuid.UUID(query)
                except ValueError:
                    pass
                else:
                    qs = qs.filter(transaction_id=query_uuid)
                if query_uuid is None or query_uuid is not None and not qs.exists():
                    qs = qs.filter(Q(message__icontains=query) | Q(extra__icontains=query))
            else:
                if is_naive(date):
                    date = make_aware(date)
                if date.hour == 0 and date.minute == 0 and date.second == 0:
                    # just a date: display all events for that date
                    qs = qs.filter(timestamp__gte=date, timestamp__lte=date + datetime.timedelta(days=1))
                elif date.second == 0:
                    # without seconds: display all events in this minute
                    qs = qs.filter(timestamp__gte=date, timestamp__lte=date + datetime.timedelta(seconds=60))
                else:
                    # display all events in the same second
                    qs = qs.filter(timestamp__gte=date, timestamp__lte=date + datetime.timedelta(seconds=1))
        return qs


class GenericLogView(GenericConnectorMixin, DetailView):
    template_name = 'passerelle/manage/log.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        try:
            context['logline'] = ResourceLog.objects.get(
                pk=self.kwargs['log_pk'], appname=self.kwargs['connector'], slug=self.kwargs['slug']
            )
        except ResourceLog.DoesNotExist:
            raise Http404()
        return context


class WrongParameter(Exception):
    http_status = 400
    log_error = False

    def __init__(self, missing, extra):
        self.missing = missing
        self.extra = extra

    def __str__(self):
        s = []
        if self.missing:
            s.append('missing parameters: %s.' % ', '.join(map(repr, self.missing)))
        if self.extra:
            s.append('extra parameters: %s.' % ', '.join(map(repr, self.extra)))
        return ' '.join(s)


class InvalidParameterValue(Exception):
    http_status = 400
    log_error = False

    def __init__(self, parameter_name):
        self.parameter_name = parameter_name

    def __str__(self):
        return 'invalid value for parameter "%s"' % self.parameter_name


IGNORED_PARAMS = (
    'apikey',
    'signature',
    'nonce',
    'algo',
    'timestamp',
    'orig',
    'jsonpCallback',
    'callback',
    '_',
    'raise',
    'debug',
    'decode',
    'format',
)


class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
    def get_params(self, request, *args, **kwargs):
        d = {}
        datasource = self.endpoint.endpoint_info.datasource
        for key in request.GET:
            # ignore authentication keys and JSONP params
            if key in IGNORED_PARAMS:
                continue
            if datasource and key in ['id', 'q']:
                # automatic parameter, ignore it
                continue
            d[key] = request.GET[key]
        other_params = kwargs.get('other_params', {})
        for key in other_params:
            if other_params[key] is None:
                continue
            if not d.get(key):
                d[key] = other_params[key]
        for parameter_info in self.endpoint.endpoint_info.get_params():
            # check and convert parameter values
            parameter = parameter_info['name']
            if datasource and parameter in ['id', 'q'] and parameter in request.GET:
                # automatic parameter, but also declared
                d[key] = request.GET[key]
            if parameter not in d:
                continue
            if parameter_info.get('blank') is False:
                if d[parameter] == '':
                    raise InvalidParameterValue(parameter)
            if parameter_info.get('type') in ('bool', 'boolean'):
                if d[parameter].lower() in ('true', 'on'):
                    d[parameter] = True
                elif d[parameter].lower() in ('false', 'off'):
                    d[parameter] = False
                else:
                    raise InvalidParameterValue(parameter)
            elif parameter_info.get('type') in ('int', 'integer'):
                try:
                    d[parameter] = int(d[parameter])
                except ValueError:
                    raise InvalidParameterValue(parameter)
            elif parameter_info.get('type') == 'float':
                d[parameter] = d[parameter].replace(',', '.')
                try:
                    d[parameter] = float(d[parameter])
                except ValueError:
                    raise InvalidParameterValue(parameter)
        if request.method == 'POST' and self.endpoint.endpoint_info.post:
            request_body = self.endpoint.endpoint_info.post.get('request_body', {})
            if 'application/json' in request_body.get('schema', {}):
                json_schema = request_body['schema']['application/json']
                must_unflatten = hasattr(json_schema, 'items') and json_schema.get('unflatten', False)
                merge_extra = hasattr(json_schema, 'items') and json_schema.get('merge_extra', False)
                pre_process = hasattr(json_schema, 'items') and json_schema.get('pre_process')
                try:
                    data = json.loads(request.body)
                except ValueError as e:
                    raise APIError("could not decode body to json: %s" % e, http_status=400)
                if must_unflatten:
                    data = unflatten(data)
                if merge_extra and hasattr(data, 'items'):
                    data.update(data.pop('extra', {}))
                if pre_process is not None:
                    pre_process(self.endpoint.__self__, data)

                # disable validation on description and title in order to allow lazy translation strings
                validator = validators.validator_for(json_schema)
                validator.META_SCHEMA['properties'].pop('description', None)
                validator.META_SCHEMA['properties'].pop('title', None)
                try:
                    validate(data, json_schema)
                except ValidationError as e:
                    error_msg = e.message
                    if e.path:
                        error_msg = '%s: %s' % ('/'.join(map(str, e.path)), error_msg)
                    raise APIError(error_msg, http_status=400)
                d['post_data'] = data

        return d

    @csrf_exempt
    def dispatch(self, request, *args, **kwargs):
        self.init_stuff(request, *args, **kwargs)
        self.connector = self.get_object()
        self.endpoint = None
        for dummy, method in inspect.getmembers(self.connector, inspect.ismethod):
            if not hasattr(method, 'endpoint_info'):
                continue
            if not method.endpoint_info.name == kwargs.get('endpoint'):
                continue
            if method.endpoint_info.pattern:
                pattern = url(method.endpoint_info.pattern, method)
                match = pattern.resolve(kwargs.get('rest') or '')
                if match:
                    self.endpoint = method
                    break
            else:
                self.endpoint = method
        if not self.endpoint:
            raise Http404()
        if kwargs.get('endpoint') == 'up' and hasattr(self.connector.check_status, 'not_implemented'):
            # hide automatic up endpoint if check_status method is not implemented
            raise Http404()
        return super().dispatch(request, *args, **kwargs)

    def _allowed_methods(self):
        return [x.upper() for x in self.endpoint.endpoint_info.methods]

    def check_perms(self, request):
        perm = self.endpoint.endpoint_info.perm
        if not perm:
            return True
        return is_authorized(request, self.connector, perm)

    def perform(self, request, *args, **kwargs):
        connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
        url = request.get_full_path()

        if request.method.lower() not in self.endpoint.endpoint_info.methods:
            self.connector.logger.warning(
                'endpoint %s %s (=> 405)' % (request.method, url),
                extra={
                    'request': request,
                    'connector': connector_name,
                    'connector_endpoint': endpoint_name,
                    'connector_endpoint_method': self._allowed_methods(),
                    'connector_endpoint_url': url,
                },
            )
            return self.http_method_not_allowed(request, *args, **kwargs)

        # auto log request's inputs
        payload = request.body[
            : self.connector.logging_parameters.requests_max_size or settings.LOGGED_REQUESTS_MAX_SIZE
        ]
        try:
            payload = payload.decode('utf-8')
        except UnicodeDecodeError:
            payload = '<BINARY PAYLOAD>'

        self.connector.logger.info(
            'endpoint %s %s (%r) ' % (request.method, url, payload),
            extra={
                'request': request,
                'connector': connector_name,
                'connector_endpoint': endpoint_name,
                'connector_endpoint_method': request.method,
                'connector_endpoint_url': url,
                'connector_payload': payload,
            },
        )

        if not self.check_perms(request):
            raise PermissionDenied()
        argspec = inspect.getfullargspec(self.endpoint)
        parameters = argspec.args[2:]
        params = self.get_params(request, *args, **kwargs)
        sig = inspect.signature(self.endpoint)
        try:
            sig.bind(request, **params)
        except TypeError:
            # prevent errors if using name of an ignored parameter in an endpoint argspec
            ignored = set(parameters) & set(IGNORED_PARAMS)
            assert not ignored, 'endpoint %s has ignored parameter %s' % (request.path, ignored)
            extra, missing = [], []
            for i, arg in enumerate(parameters):
                # check if the argument is optional, i.e. it has a default value
                if len(parameters) - i <= len(argspec.defaults or []):
                    continue
                if arg not in params:
                    missing.append(arg)
            for key in params:
                if key not in argspec.args:
                    extra.append(key)
            raise WrongParameter(missing, extra)

        def log_result(result):
            try:
                data = json.dumps(result, cls=JSONEncoder)
            except TypeError:
                data = str(result)
            self.connector.logger.debug(
                'endpoint %s %s (%r) ' % (request.method, url, payload),
                extra={
                    'request': request,
                    'connector': connector_name,
                    'connector_endpoint': endpoint_name,
                    'connector_endpoint_method': request.method,
                    'connector_endpoint_url': url,
                    'connector_payload': payload,
                    'connector_result': data,
                    'connector_result_in_cache': in_cache,
                },
            )

        params = self.get_params(request, *args, **kwargs)
        in_cache = False
        if request.method == 'GET' and self.endpoint.endpoint_info.cache_duration:
            cache_key = hashlib.md5(
                force_bytes(repr(self.get_object().slug) + repr(self.endpoint) + repr(params))
            ).hexdigest()
            result = cache.get(cache_key)
            if result is not None:
                # filter result after caching
                in_cache = True
                result = self.filter_result(request, params, result)
                log_result(result)
                return result

        result = self.endpoint(request, **params)
        if request.method == 'GET' and self.endpoint.endpoint_info.cache_duration:
            cache.set(cache_key, result, self.endpoint.endpoint_info.cache_duration)
        # filter result after caching
        result = self.filter_result(request, params, result)
        log_result(result)
        return result

    def filter_result(self, request, params, result):
        if not self.endpoint.endpoint_info.datasource:
            return result
        if not request.method == 'GET':
            return result
        if not result.get('data'):
            return result
        if not isinstance(result['data'], list):
            return result
        if not isinstance(result['data'][0], dict):
            return result

        if request.GET.get('id') and 'id' not in params:
            # automatic filtering on id
            result['data'] = [r for r in result['data'] if r.get('id') == request.GET['id']]
            return result

        if request.GET.get('q') and 'q' not in params:
            # automatic filtering on text
            term = normalize(request.GET['q'].lower())
            result['data'] = [r for r in result['data'] if term in normalize((r.get('text') or '').lower())]
            return result

        return result

    def get(self, request, *args, **kwargs):
        if self.endpoint.endpoint_info.pattern:
            pattern = url(self.endpoint.endpoint_info.pattern, self.endpoint)
            match = pattern.resolve(kwargs.get('rest') or '')
            if not match:
                raise Http404()
            kwargs['other_params'] = match.kwargs
        elif kwargs.get('rest'):
            raise Http404()
        return to_json(logger=self.connector.logger)(self.perform)(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def patch(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def delete(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)

    def put(self, request, *args, **kwargs):
        return self.get(request, *args, **kwargs)


class GenericExportConnectorView(GenericConnectorMixin, DetailView):
    def get(self, request, *args, **kwargs):
        response = HttpResponse(content_type='application/json')
        today = datetime.date.today()
        response['Content-Disposition'] = 'attachment; filename="export_{}_{}_{}.json"'.format(
            self.get_object().get_connector_slug(), self.get_object().slug, today.strftime('%Y%m%d')
        )
        json.dump({'resources': [self.get_object().export_json()]}, response, indent=2)
        return response