wcs/wcs/ctl/check_hobos.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2014  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import ConfigParser
import json
import os
import random
import sys
import tempfile
import urllib2
import urlparse
import hashlib

from quixote import cleanup
from qommon import misc
from qommon.ctl import Command, make_option
from qommon.storage import atomic_write

from wcs.admin.settings import UserFieldsFormDef
from wcs.fields import StringField, EmailField, DateField

from django.utils.encoding import force_bytes


class NoChange(Exception):
    pass

def atomic_symlink(src, dst):
    if os.path.exists(dst) and os.readlink(dst) == src:
        return
    if os.path.exists(dst + '.tmp'):
        os.unlink(dst + '.tmp')
    os.symlink(src, dst + '.tmp')
    os.rename(dst + '.tmp', dst)

class CmdCheckHobos(Command):
    name = 'hobo_deploy'

    # TODO: import this from django settings
    THEMES_DIRECTORY = os.environ.get('THEMES_DIRECTORY', '/usr/share/publik/themes')

    def __init__(self):
        Command.__init__(self, [
                make_option('--ignore-timestamp', action='store_true',
                            dest='ignore_timestamp', default=False),
                make_option('--redeploy', action='store_true', default=False),
                ])

    def execute(self, base_options, sub_options, args):
        import publisher
        publisher.WcsPublisher.configure(self.config)
        if sub_options.redeploy:
            sub_options.ignore_timestamp = True
            for tenant in publisher.WcsPublisher.get_tenants():
                hobo_json_path = os.path.join(publisher.WcsPublisher.APP_DIR, tenant, 'hobo.json')
                if not os.path.exists(hobo_json_path):
                    continue
                hobo_json = json.load(open(hobo_json_path))
                try:
                    me = [service for service in hobo_json['services']
                          if service.get('this') is True][0]
                except IndexError:
                    pass
                else:
                    cleanup()
                    self.deploy(base_options, sub_options, [me['base_url'], hobo_json_path])
        else:
            self.deploy(base_options, sub_options, args)

    def deploy(self, base_options, sub_options, args):
        import publisher

        self.base_options = base_options
        if sub_options.extra:
            if not self.config.has_section('extra'):
                self.config.add_section('extra')
            for i, extra in enumerate(sub_options.extra):
                self.config.set('extra', 'cmd_line_extra_%d' % i, extra)
        publisher.WcsPublisher.configure(self.config)
        pub = publisher.WcsPublisher.create_publisher(
                register_tld_names=False)

        global_app_dir = pub.app_dir
        base_url = args[0]

        if args[1] == '-':
            # get environment definition from stdin
            self.all_services = json.load(sys.stdin)
        else:
            self.all_services = json.load(file(args[1]))

        try:
            service = [x for x in self.all_services.get('services', []) if \
                x.get('service-id') == 'wcs' and x.get('base_url') == base_url and
                not x.get('secondary')][0]
        except IndexError:
            return

        service['this'] = True
        if base_url.endswith('/'): # wcs doesn't expect a trailing slash
            service['base_url'] = base_url[:-1]

        pub.app_dir = os.path.join(global_app_dir,
                self.get_instance_path(service))
        if not os.path.exists(pub.app_dir):
            print 'initializing instance in', pub.app_dir
            os.mkdir(pub.app_dir)
            pub.initialize_app_dir()

            if service.get('template_name'):
                skeleton_filepath = os.path.join(global_app_dir, 'skeletons',
                        service.get('template_name'))
                if os.path.exists(skeleton_filepath):
                    pub.import_zip(file(skeleton_filepath))
            new_site = True
        else:
            print 'updating instance in', pub.app_dir
            new_site = False

        try:
            self.configure_site_options(service, pub,
                    ignore_timestamp=sub_options.ignore_timestamp)
        except NoChange:
            print '  skipping'
            return

        pub.set_config(skip_sql=True)
        if new_site:
            self.configure_sql(service, pub)
        self.update_configuration(service, pub)
        self.configure_authentication_methods(service, pub)

        self.update_profile(self.all_services.get('profile', {}), pub)
        # Store hobo.json
        atomic_write(os.path.join(pub.app_dir, 'hobo.json'), json.dumps(self.all_services))

    def update_configuration(self, service, pub):
        if not pub.cfg.get('misc'):
            pub.cfg['misc'] = {'charset': 'utf-8'}
        pub.cfg['misc']['sitename'] = service.get('title').encode('utf-8')
        pub.cfg['misc']['frontoffice-url'] = service.get('base_url').encode('utf-8')
        if not pub.cfg.get('language'):
            pub.cfg['language'] = {'language': 'fr'}

        if not pub.cfg.get('emails'):
            pub.cfg['emails'] = {}

        variables = self.all_services.get('variables') or {}
        variables.update(service.get('variables') or {})

        theme_id = variables.get('theme')
        theme_data = None
        if theme_id and os.path.exists(self.THEMES_DIRECTORY):
            for theme_module in os.listdir(self.THEMES_DIRECTORY):
                try:
                    themes_json = json.load(open(
                        os.path.join(self.THEMES_DIRECTORY, theme_module, 'themes.json')))
                except IOError:
                    continue
                if not isinstance(themes_json, dict):  # compat
                    themes_json = {'themes': themes_json}
                for theme_data in themes_json.get('themes'):
                    if theme_data.get('id') == theme_id:
                        if not 'module' in theme_data:
                            theme_data['module'] = theme_module
                        break
                else:
                    theme_data = None
                    continue
                break
        if theme_data:
            pub.cfg['branding'] = {'theme': 'publik-base'}
            tenant_dir = pub.app_dir
            theme_dir = os.path.join(tenant_dir, 'theme')
            target_dir = os.path.join(self.THEMES_DIRECTORY, theme_data['module'])
            atomic_symlink(target_dir, theme_dir)
            for component in ('static', 'templates'):
                component_dir = os.path.join(tenant_dir, component)
                if not os.path.islink(component_dir) and os.path.isdir(component_dir):
                    try:
                        os.rmdir(component_dir)
                    except OSError:
                        continue
                if not theme_data.get('overlay'):
                    try:
                        os.unlink(component_dir)
                    except OSError:
                        pass
                else:
                    atomic_symlink(
                            os.path.join(self.THEMES_DIRECTORY, theme_data['overlay'], component),
                            component_dir)

        if variables.get('default_from_email'):
            pub.cfg['emails']['from'] = variables.get('default_from_email').encode('utf-8')
        if variables.get('email_signature') is not None:
            pub.cfg['emails']['footer'] = variables.get('email_signature').encode('utf-8')

        pub.write_cfg()

    def update_profile(self, profile, pub):
        formdef = UserFieldsFormDef(publisher=pub)
        profile_fields = {}
        profile_field_ids = ['_' + x['name'] for x in profile.get('fields', [])]
        for field in formdef.fields:
            if field.id in profile_field_ids:
                profile_fields[field.id] = field

        html5_autocomplete_map = {
            'first_name': 'given-name',
            'last_name': 'family-name',
            'address': 'address-line1',
            'zipcode': 'postal-code',
            'city': 'address-level2',
            'country': 'country',
            'phone': 'tel',
            'email': 'email',
        }

        # create or update profile fields
        for attribute in profile.get('fields', []):
            field_id = '_' + attribute['name']
            if not field_id in profile_fields:
                field_class = StringField
                if attribute['kind'] == 'email':
                    field_class = EmailField
                elif attribute['kind'] in ('date', 'birthdate', 'fedict_date'):
                    field_class = DateField
                new_field = field_class(label=attribute['label'].encode('utf-8'),
                        type=field_class.key,
                        varname=attribute['name'])
                new_field.id = field_id
                profile_fields[field_id] = new_field
            else:
                # remove it for the moment
                formdef.fields.remove(profile_fields[field_id])

            profile_fields[field_id].label = attribute['label'].encode('utf-8')
            profile_fields[field_id].hint = attribute['description'].encode('utf-8')
            profile_fields[field_id].required = attribute['required']

            if attribute['disabled']:
                profile_field_ids.remove('_' + attribute['name'])
            if attribute['name'] in html5_autocomplete_map:
                profile_fields[field_id].extra_css_class = (
                        'autocomplete-%s' % html5_autocomplete_map[attribute['name']])

        # insert profile fields at the beginning
        formdef.fields = [profile_fields[x] for x in profile_field_ids] + formdef.fields
        formdef.store()

        pub.cfg['users']['field_email'] = '_email'
        pub.cfg['users']['field_name'] = ['_first_name', '_last_name']
        pub.write_cfg()

        # add mapping for SAML provisioning
        for idp in pub.cfg.get('idp', {}).values():
            if not idp.get('attribute-mapping'):
                idp['attribute-mapping'] = {}
            for field in profile.get('fields', []):
                attribute_name = field['name']
                field_id = '_' + attribute_name
                if field_id in profile_field_ids:
                    idp['attribute-mapping'][str(attribute_name)] = str(field_id)
        pub.write_cfg()

    def configure_authentication_methods(self, service, pub):
        # look for an identity provider
        idps = [x for x in self.all_services.get('services', []) if x.get('service-id') == 'authentic']
        if not pub.cfg.get('identification'):
            pub.cfg['identification'] = {}
        methods = pub.cfg['identification'].get('methods', [])
        if idps and not 'idp' in methods:
            methods.append('idp')
        elif not idps and not 'password' in methods:
            methods.append('password')
        pub.cfg['identification']['methods'] = methods
        if not pub.cfg.get('sp'):
            pub.cfg['sp'] = {}
        pub.cfg['sp']['idp-manage-user-attributes'] = bool(idps)
        pub.cfg['sp']['idp-manage-roles'] = bool(idps)
        pub.write_cfg()

        if not idps:
            return

        # initialize service provider side
        if not pub.cfg['sp'].get('publickey'):
            from qommon.ident.idp import MethodAdminDirectory
            spconfig = pub.cfg['sp']
            spconfig['saml2_base_url'] = str(service.get('base_url')) + '/saml'
            spconfig['saml2_providerid'] = spconfig['saml2_base_url'] + '/metadata'
            MethodAdminDirectory().generate_rsa_keypair()

        if not 'saml_identities' in pub.cfg:
            pub.cfg['saml_identities'] = {}

        if idps:
            pub.cfg['saml_identities']['identity-creation'] = 'self'

        # write down configuration to disk as it will get reloaded
        # automatically and we don't want to lose our changes.
        pub.write_cfg()

        for idp in idps:
            if not idp['base_url'].endswith('/'):
                idp['base_url'] = idp['base_url'] + '/'
            metadata_url = '%sidp/saml2/metadata' % idp['base_url']
            try:
                rfd = misc.urlopen(metadata_url)
            except misc.ConnectionError as e:
                print >> sys.stderr, 'failed to get metadata URL', metadata_url, e
                continue

            s = rfd.read()
            (bfd, metadata_pathname) = tempfile.mkstemp('.metadata')
            atomic_write(metadata_pathname, s)

            from qommon.ident.idp import AdminIDPDir
            admin_dir = AdminIDPDir()
            key_provider_id = admin_dir.submit_new_remote(
                    metadata_pathname, None, metadata_url, None)
            admin_attribute = service.get('variables', {}).get('admin-attribute')
            if not admin_attribute:
                admin_attribute = 'is_superuser=true'
            else:
                admin_attribute = unicode(admin_attribute).encode('utf-8')
            admin_attribute_dict = dict([admin_attribute.split('=')])
            pub.cfg['idp'][key_provider_id]['admin-attributes'] = admin_attribute_dict
            pub.cfg['idp'][key_provider_id]['nameidformat'] = 'unspecified'
            pub.cfg['saml_identities']['registration-url'] = str(
                    '%saccounts/register/' % idp['base_url'])
            pub.write_cfg()

    def get_instance_path(self, service):
        parsed_url = urllib2.urlparse.urlsplit(service.get('base_url'))
        instance_path = parsed_url.netloc
        if parsed_url.path:
            instance_path += '+%s' % parsed_url.path.replace('/', '+')
        return instance_path

    def configure_site_options(self, current_service, pub, ignore_timestamp=False):
        # configure site-options.cfg
        config = ConfigParser.RawConfigParser()
        site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
        if os.path.exists(site_options_filepath):
            config.read(site_options_filepath)

        if not ignore_timestamp:
            try:
                if config.get('hobo', 'timestamp') == self.all_services.get('timestamp'):
                    raise NoChange()
            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
                pass

        if not 'hobo' in config.sections():
            config.add_section('hobo')
        config.set('hobo', 'timestamp', self.all_services.get('timestamp'))

        if not 'options' in config.sections():
            config.add_section('options')

        variables = {}
        api_secrets = {}
        for service in self.all_services.get('services', []):
            service_url = service.get('base_url')
            if not service_url.endswith('/'):
                service_url += '/'
            if service.get('slug'):
                variables['%s_url' % service.get('slug').replace('-', '_')] = service_url

            if not service.get('secret_key'):
                continue

            domain = urlparse.urlparse(service_url).netloc.split(':')[0]
            if service is current_service:
                if config.has_option('api-secrets', domain):
                    api_secrets[domain] = config.get('api-secrets', domain)
                else:
                    # custom key calcultation for "self", as the shared_secret code
                    # would do secret_key ^ secret_key = 0.
                    api_secrets[domain] = self.shared_secret(current_service.get('secret_key'),
                            str(random.SystemRandom().random()))
                continue

            api_secrets[domain] = self.shared_secret(current_service.get('secret_key'), service.get('secret_key'))

            if service.get('service-id') == 'authentic':
                variables['idp_url'] = service_url
                variables['idp_account_url'] = service_url + 'accounts/'
                variables['idp_api_url'] = service_url + 'api/'
                variables['idp_registration_url'] = service_url + 'accounts/register/'

            if service.get('secondary'):
                continue

            if service.get('service-id') == 'combo':
                if service.get('template_name') == 'portal-agent':
                    variables['portal_agent_url'] = service_url
                    variables['portal_agent_title'] = service.get('title')
                elif service.get('template_name') == 'portal-user':
                    variables['portal_url'] = service_url
                    variables['portal_user_url'] = service_url
                    variables['portal_user_title'] = service.get('title')
                    config.set('options', 'theme_skeleton_url',
                            service.get('base_url') + '__skeleton__/')

        if self.all_services.get('variables'):
            for key, value in self.all_services.get('variables').items():
                variables[key] = value
        for key, value in current_service.get('variables', {}).items():
            variables[key] = value

        if variables:
            if not 'variables' in config.sections():
                config.add_section('variables')
            for key, value in variables.items():
                key = unicode(key).encode('utf-8')
                value = unicode(value).encode('utf-8')
                config.set('variables', key, value)

        if not 'api-secrets' in config.sections():
            config.add_section('api-secrets')
        if not 'wscall-secrets' in config.sections():
            config.add_section('wscall-secrets')
        for key, value in api_secrets.items():
            config.set('api-secrets', key, value)
            # for now the secrets are the same whatever the direction is.
            config.set('wscall-secrets', key, value)

        # add known services
        for service in self.all_services.get('services', []):
            if service.get('secondary'):
                continue
            if service.get('service-id') == 'fargo':
                config.set('options', 'fargo_url', service.get('base_url'))

        try:
            portal_agent_url = config.get('variables', 'portal_agent_url')
        except ConfigParser.NoOptionError:
            pass
        else:
            if portal_agent_url.endswith('/'):
                portal_agent_url = portal_agent_url.rstrip('/')
            extra_head = '''<script src="%s/__services.js"></script>'''\
                         '''<script src="%s/static/js/publik.js"></script>''' % (
                                 portal_agent_url, portal_agent_url)
            config.set('options', 'backoffice_extra_head', extra_head)

        with open(site_options_filepath, 'wb') as site_options:
            config.write(site_options)

    def normalize_database_name(self, database_name):
        if len(database_name) > 63:
            digest = hashlib.md5(force_bytes(database_name)).hexdigest()[:4]
            database_name = '%s_%s_%s' % (database_name[:29], digest, database_name[-28:])
        return database_name

    def configure_sql(self, service, pub):
        if not pub.cfg.get('postgresql'):
            return

        if not pub.has_site_option('postgresql'):
            return

        import psycopg2
        import psycopg2.errorcodes

        # determine database name using the instance path
        domain_table_name = self.get_instance_path(service).replace(
                '-', '_').replace('.', '_').replace('+', '_')

        if pub.cfg['postgresql'].get('database-template-name'):
            database_template_name = pub.cfg['postgresql'].pop('database-template-name')
            database_name = (database_template_name % domain_table_name).strip('_')
        else:
            # legacy way to create a database name, if it contained an
            # underscore character, use the first part as a prefix
            database_name = pub.cfg['postgresql'].get('database', 'wcs')
            if not domain_table_name in database_name:
                database_name = '%s_%s' % (database_name.split('_')[0], domain_table_name)

        database_name = self.normalize_database_name(database_name)

        createdb_cfg = pub.cfg['postgresql'].get('createdb-connection-params')
        if not createdb_cfg:
            createdb_cfg = {}
            for k, v in pub.cfg['postgresql'].items():
                if v and isinstance(v, basestring):
                    createdb_cfg[k] = v

        try:
            pgconn = psycopg2.connect(**createdb_cfg)
        except psycopg2.Error as e:
            print >> sys.stderr, 'failed to connect to postgresql (%s)' % \
                    psycopg2.errorcodes.lookup(e.pgcode)
            return

        pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
        cur = pgconn.cursor()
        new_database = True
        try:
            cur.execute('''CREATE DATABASE %s''' % database_name)
        except psycopg2.Error as e:
            if e.pgcode == psycopg2.errorcodes.DUPLICATE_DATABASE:
                cur.execute("""SELECT table_name
                               FROM information_schema.tables
                               WHERE table_schema = 'public'
                               AND table_type = 'BASE TABLE'
                               AND table_name = 'wcs_meta'""")

                if cur.fetchall():
                    new_database = False
            else:
                print >> sys.stderr, 'failed to create database (%s)' % \
                        psycopg2.errorcodes.lookup(e.pgcode)
                return
        else:
            cur.close()

        pub.cfg['postgresql']['database'] = database_name
        pub.write_cfg()
        pub.set_config(skip_sql=False)

        if not new_database:
            return

        # create tables etc.
        pub.initialize_sql()

    @classmethod
    def shared_secret(cls, secret1, secret2):
        secret1 = hashlib.sha256(secret1).hexdigest()
        secret2 = hashlib.sha256(secret2).hexdigest()
        return hex(int(secret1, 16) ^ int(secret2, 16))[2:-1]


CmdCheckHobos.register()