hobo/hobo/environment/models.py

# hobo - portal to configure and deploy applications
# Copyright (C) 2015-2019  Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import datetime
import json
import random
import re
import socket
import urllib.parse

import requests
from django.conf import settings
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
from django.contrib.contenttypes.models import ContentType
from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from django.db import models
from django.db.models import JSONField
from django.utils.crypto import get_random_string
from django.utils.encoding import force_str
from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _

from .utils import Zone, get_installed_services, get_local_key

SECRET_CHARS = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
FLOAT_RE = re.compile(r'^\s*[0-9]+\.[0-9]+\s*')


AUTO_VARIABLES = (
    'default_from_email',
    'global_email_prefix',
    'email_signature',
    'email_sender_name',
    'global_title',
    'robots_txt',
    'meta_description',
    'meta_keywords',
    'sms_url',
    'sms_sender',
)


class Variable(models.Model):
    name = models.CharField(max_length=100, verbose_name=_('name'))
    label = models.CharField(max_length=100, blank=True, verbose_name=_('label'))
    value = models.TextField(
        verbose_name=_('value'), blank=True, help_text=_('start with [ or { for a JSON document')
    )
    auto = models.BooleanField(default=False)
    service_type = models.ForeignKey(ContentType, null=True, on_delete=models.CASCADE)
    service_pk = models.PositiveIntegerField(null=True)
    service = GenericForeignKey('service_type', 'service_pk')
    last_update_timestamp = models.DateTimeField(auto_now=True, null=True)

    def get_field_label(self):
        if self.label:
            return self.label
        return self.name

    def _parse_value_as_json(self):
        if (
            self.value
            and not self.value.startswith(('{{', '{%'))
            and (
                self.value[0] in '{[' or self.value in ('true', 'false', 'null') or FLOAT_RE.match(self.value)
            )
        ):
            try:
                return json.loads(self.value)
            except ValueError:
                raise ValidationError('invalid JSON document')
        return self.value

    def json_getter(self):
        try:
            return self._parse_value_as_json()
        except ValidationError:
            return self.value

    def json_setter(self, value):
        self.value = json.dumps(value)

    json = property(json_getter, json_setter)

    def clean(self):
        self._parse_value_as_json()


def is_resolvable(url):
    try:
        netloc = urllib.parse.urlparse(url).netloc
        if netloc and socket.gethostbyname(netloc):
            return True
    except socket.gaierror:
        return False


def has_valid_certificate(url):
    try:
        requests.get(url, timeout=5, verify=True, allow_redirects=False)
        return True
    except requests.exceptions.SSLError:
        return False
    except requests.exceptions.ConnectionError:
        return False


class ServiceBase(models.Model):
    class Meta:
        abstract = True
        constraints = [
            models.UniqueConstraint(
                fields=['title'],
                condition=models.Q(secondary=False),
                name='%(app_label)s_%(class)s_title_idx',
            ),
        ]

    title = models.CharField(_('Title'), max_length=50)
    slug = models.SlugField(_('Slug'), max_length=200, unique=True)
    base_url = models.CharField(_('Base URL'), max_length=200, validators=[URLValidator()])
    legacy_urls = JSONField(null=True, default=list, blank=True)
    secret_key = models.CharField(_('Secret Key'), max_length=60)
    template_name = models.CharField(_('Template'), max_length=60, blank=True)
    secondary = models.BooleanField(_('Secondary Service'), default=False)

    last_operational_check_timestamp = models.DateTimeField(null=True)
    last_operational_success_timestamp = models.DateTimeField(null=True)
    last_update_timestamp = models.DateTimeField(auto_now=True, null=True)

    variables = GenericRelation(Variable, content_type_field='service_type', object_id_field='service_pk')

    @classmethod
    def is_enabled(cls):
        name = cls.__name__.lower()
        return name not in settings.HOBO_SERVICES_DISABLED or name in settings.HOBO_SERVICES_ENABLED

    def is_operational(self):
        return (
            self.last_operational_success_timestamp is not None
            and self.last_operational_success_timestamp == self.last_operational_check_timestamp
        )

    def check_operational(self):
        once_now = now()
        self.last_operational_check_timestamp = once_now
        try:
            zone = self.get_admin_zones()[0]
            response = requests.get(zone.href, timeout=5, allow_redirects=False)
            response.raise_for_status()
            self.last_operational_success_timestamp = once_now
        except requests.RequestException:
            pass
        self.save(update_fields=('last_operational_check_timestamp', 'last_operational_success_timestamp'))

    def wants_frequent_checks(self):
        # decides if a "being deployed..." spinner should be displayed (and
        # automatically hidden) next to the service.
        if self.last_operational_success_timestamp is not None:
            # if the service has been marked as operational, we don't need a
            # spinner at all.
            return False
        if self.last_operational_check_timestamp is None:
            # if the service has never been checked, sure we wants a spinner.
            return True
        two_minutes = datetime.timedelta(minutes=2)
        # monitor actively for two minutes max.
        return (self.last_operational_check_timestamp - self.last_update_timestamp) < two_minutes

    def as_dict(self):
        as_dict = {x: y for (x, y) in self.__dict__.items() if isinstance(y, (int, str))}
        as_dict['base_url'] = self.get_base_url_path()
        if self.legacy_urls:
            as_dict['legacy_urls'] = self.legacy_urls
        as_dict['service-id'] = self.Extra.service_id
        as_dict['service-label'] = force_str(self.Extra.service_label)
        as_dict['variables'] = {v.name: v.json for v in self.variables.all()}
        as_dict['secondary'] = self.secondary
        if self.get_saml_sp_metadata_url():
            as_dict['saml-sp-metadata-url'] = self.get_saml_sp_metadata_url()
        if self.get_saml_idp_metadata_url():
            as_dict['saml-idp-metadata-url'] = self.get_saml_idp_metadata_url()
        if self.get_backoffice_menu_url():
            as_dict['backoffice-menu-url'] = self.get_backoffice_menu_url()
        if self.get_provisionning_url():
            as_dict['provisionning-url'] = self.get_provisionning_url()
        return as_dict

    @property
    def name(self):
        return self.title

    def clean(self, *args, **kwargs):
        for service in get_installed_services():
            if self.pk == service.pk:
                continue
            if service.slug == self.slug:
                raise ValidationError(_('This slug is already used. It must be unique.'))
            if service.title == self.title and service.secondary is False and self.secondary is False:
                raise ValidationError(_('This title is already used. It must be unique.'))
        return super().clean(*args, **kwargs)

    def save(self, *args, **kwargs):
        self.base_url = self.base_url.strip().lower()
        if not self.base_url.endswith('/'):
            self.base_url += '/'
        if not self.secret_key:
            if self.Extra.service_id == 'hobo':
                self.secret_key = get_local_key(self.base_url)
            else:
                self.secret_key = get_random_string(50, SECRET_CHARS)

        if not self.title and self.slug:
            self.title = self.slug

        is_new = self.id is None
        super().save(*args, **kwargs)

        if is_new and settings.SERVICE_EXTRA_VARIABLES:
            for variable in settings.SERVICE_EXTRA_VARIABLES.get(self.Extra.service_id, []):
                v = Variable()
                if type(variable) is dict:
                    v.name = variable.get('name')
                    v.label = variable.get('label')
                else:
                    v.name = variable
                v.service = self
                v.save()

    def get_saml_sp_metadata_url(self):
        return None

    def get_saml_idp_metadata_url(self):
        return None

    def get_base_url_path(self):
        base_url = self.base_url
        if not base_url.endswith('/'):
            base_url += '/'
        return base_url

    def get_backoffice_menu_url(self):
        return None

    def get_provisionning_url(self):
        return self.get_base_url_path() + '__provision__/'

    def is_resolvable(self):
        return is_resolvable(self.base_url)

    def has_valid_certificate(self):
        return has_valid_certificate(self.base_url)

    def is_running(self):
        if not self.is_resolvable():
            return False
        r = requests.get(self.get_admin_zones()[0].href, timeout=5, verify=False, allow_redirects=False)
        return r.status_code >= 200 and r.status_code < 400

    def security_data(self):
        security_data = {
            'level': 'NaN',
            'label': '',
        }
        if not self.is_resolvable():
            return security_data
        security_data['level'] = 0
        resp = requests.get(self.base_url, timeout=5, verify=False, allow_redirects=False)
        missing_headers = []
        for header in (
            'X-Content-Type-Options',
            'X-Frame-Options',
            'X-XSS-Protection',
            'Strict-Transport-Security',
        ):
            if not resp.headers.get(header):
                missing_headers.append(header)
        if missing_headers:
            security_data['level'] = len(missing_headers)
            security_data['label'] = _('Missing headers: %s') % ', '.join(missing_headers)
        else:
            security_data['label'] = _('HTTP security headers are set.')
        return security_data

    def get_health_dict(self):
        properties = [
            ('is_resolvable', 120),
            ('has_valid_certificate', 3600),
            ('is_running', 60),
            ('is_operational', 60),
            ('security_data', 60),
        ]
        result = {}
        for name, cache_duration in properties:
            cache_key = '%s_%s' % (self.slug, name)
            value = cache.get(cache_key)
            if value is None:
                value = getattr(self, name)()
                cache.set(cache_key, value, cache_duration * (0.5 + random.random()))
            result[name] = value
        return result

    def change_base_url(self, base_url):
        service_dict = self.as_dict()
        legacy_urls = {
            'base_url': service_dict['base_url'],
            'timestamp': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        }
        for url_key in (
            'saml-sp-metadata-url',
            'saml-idp-metadata-url',
            'backoffice-menu-url',
            'provisionning-url',
        ):
            if url_key in service_dict:
                legacy_urls[url_key] = service_dict[url_key]
        if not self.legacy_urls:
            self.legacy_urls = []
        self.legacy_urls.insert(0, legacy_urls)
        self.base_url = base_url


class Authentic(ServiceBase):
    use_as_idp_for_self = models.BooleanField(verbose_name=_('Use as IdP'), default=False)

    class Meta:
        verbose_name = _('Authentic Identity Provider')
        verbose_name_plural = _('Authentic Identity Providers')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'authentic'
        service_label = _('Authentic')
        service_default_slug = 'idp'
        constraints = ServiceBase.Meta.constraints

    def get_admin_zones(self):
        return [
            Zone(_('User Management'), 'users', self.get_base_url_path() + 'manage/users/'),
            Zone(_('Role Management'), 'roles', self.get_base_url_path() + 'manage/roles/'),
        ]

    def get_saml_idp_metadata_url(self):
        return self.get_base_url_path() + 'idp/saml2/metadata'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'

    def get_provisionning_url(self):
        return None


class Wcs(ServiceBase):
    class Meta:
        verbose_name = _('w.c.s. Web Forms')
        verbose_name_plural = _('w.c.s. Web Forms')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'wcs'
        service_label = _('w.c.s.')
        service_default_slug = 'eservices'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'backoffice/menu.json'

    def get_admin_zones(self):
        return [
            Zone(self.title, 'webforms', self.get_base_url_path() + 'admin/'),
        ]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'saml/metadata'


class Passerelle(ServiceBase):
    class Meta:
        verbose_name = _('Passerelle')
        verbose_name_plural = _('Passerelle')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'passerelle'
        service_label = _('Passerelle')
        service_default_slug = 'passerelle'

    def get_admin_zones(self):
        return [Zone(self.title, 'webservices', self.get_base_url_path() + 'manage/')]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'


class Combo(ServiceBase):
    class Meta:
        verbose_name = _('Combo Portal')
        verbose_name_plural = _('Combo Portals')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'combo'
        service_label = _('Combo')
        service_default_slug = 'portal'

    def get_admin_zones(self):
        return [Zone(self.title, 'portal', self.get_base_url_path() + 'manage/')]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'


class Fargo(ServiceBase):
    class Meta:
        verbose_name = _('Fargo document box')
        verbose_name_plural = _('Fargo document box')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'fargo'
        service_label = _('Fargo')
        service_default_slug = 'porte-doc'

    def get_admin_zones(self):
        return [Zone(self.title, 'document-box', self.get_base_url_path() + 'admin/')]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'


class Welco(ServiceBase):
    class Meta:
        verbose_name = _('Welco Mail Channel')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'welco'
        service_label = 'Welco'
        service_default_slug = 'courrier'

    def get_admin_zones(self):
        return [Zone(self.title, 'multichannel-guichet', self.get_base_url_path() + 'admin/')]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'menu.json'


class Chrono(ServiceBase):
    class Meta:
        verbose_name = _('Chrono Agendas')
        verbose_name_plural = _('Chrono Agendas')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'chrono'
        service_label = _('Chrono')
        service_default_slug = 'agendas'

    def get_admin_zones(self):
        return [Zone(self.title, 'calendar', self.get_base_url_path() + 'manage/')]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'


class Hobo(ServiceBase):
    class Meta:
        verbose_name = _('Hobo Deployment Server')
        verbose_name_plural = _('Hobo Deployment Servers')
        ordering = ['title']
        constraints = [
            models.UniqueConstraint(
                fields=['local'],
                condition=models.Q(local=True),
                name='%(app_label)s_%(class)s_local_true_idx',
            ),
        ] + ServiceBase.Meta.constraints

    class Extra:
        service_id = 'hobo'
        service_label = _('Deployment Server')
        service_default_slug = 'hobo'

    # historically an hobo instance did not store information about itself,
    # it only stored info about other hobo's.
    # now we also store info about an instance on the instance itself, via local=True
    local = models.BooleanField(default=False)

    def get_admin_zones(self):
        return [Zone(self.title, 'hobo', self.get_base_url_path())]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        if self.local:
            return self.get_base_url_path() + 'menu.json'
        return None

    def as_dict(self):
        res = super().as_dict()
        if self.local:
            del res['id']
            del res['secondary']
            del res['service-label']
            del res['template_name']
            del res['variables']
        return res


class BiJoe(ServiceBase):
    class Meta:
        verbose_name = _('Statistics')
        verbose_name_plural = _('Statistics')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'bijoe'
        service_label = _('Statistics')
        service_default_slug = 'statistics'

    def get_admin_zones(self):
        return [Zone(self.title, 'bijoe', self.get_base_url_path())]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'


class Lingo(ServiceBase):
    class Meta:
        verbose_name = _('Lingo Billing and Payment')
        verbose_name_plural = _('Lingo Billing and Payment')
        ordering = ['title']
        constraints = ServiceBase.Meta.constraints

    class Extra:
        service_id = 'lingo'
        service_label = _('Lingo')
        service_default_slug = 'payment'

    def get_admin_zones(self):
        return [Zone(self.title, 'bankcard', self.get_base_url_path())]

    def get_saml_sp_metadata_url(self):
        return self.get_base_url_path() + 'accounts/mellon/metadata/'

    def get_backoffice_menu_url(self):
        return self.get_base_url_path() + 'manage/menu.json'


AVAILABLE_SERVICES = [Authentic, Wcs, Passerelle, Combo, Fargo, Welco, Chrono, BiJoe, Hobo, Lingo]