wcs/wcs/ctl/check_hobos.py

# w.c.s. - web application for online forms
# Copyright (C) 2005-2014  Entr'ouvert
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import ConfigParser
import json
import os
import subprocess
import sys
import tempfile
import urllib2
import urlparse
import hashlib

from quixote import cleanup
from qommon.ctl import Command, make_option
from qommon.storage import atomic_write

from wcs.admin.settings import UserFieldsFormDef
from wcs.fields import StringField, EmailField


class NoChange(Exception):
    pass


class CmdCheckHobos(Command):
    name = 'hobo_deploy'

    def __init__(self):
        Command.__init__(self, [
                make_option('--ignore-timestamp', action='store_true',
                            dest='ignore_timestamp', default=False),
                make_option('--redeploy', action='store_true', default=False),
                ])

    def execute(self, base_options, sub_options, args):
        import publisher
        publisher.WcsPublisher.configure(self.config)
        if sub_options.redeploy:
            sub_options.ignore_timestamp = True
            for tenant in publisher.WcsPublisher.get_tenants():
                hobo_json_path = os.path.join(publisher.WcsPublisher.APP_DIR, tenant, 'hobo.json')
                if not os.path.exists(hobo_json_path):
                    continue
                hobo_json = json.load(open(hobo_json_path))
                try:
                    me = [service for service in hobo_json['services']
                          if service.get('this') is True][0]
                except IndexError:
                    pass
                else:
                    cleanup()
                    self.deploy(base_options, sub_options, [me['base_url'], hobo_json_path])
        else:
            self.deploy(base_options, sub_options, args)

    def deploy(self, base_options, sub_options, args):
        import publisher

        self.base_options = base_options
        if sub_options.extra:
            if not self.config.has_section('extra'):
                self.config.add_section('extra')
            for i, extra in enumerate(sub_options.extra):
                self.config.set('extra', 'cmd_line_extra_%d' % i, extra)
        publisher.WcsPublisher.configure(self.config)
        pub = publisher.WcsPublisher.create_publisher(
                register_cron=False, register_tld_names=False)

        global_app_dir = pub.app_dir
        base_url = args[0]

        if args[1] == '-':
            # get environment definition from stdin
            self.all_services = json.load(sys.stdin)
        else:
            self.all_services = json.load(file(args[1]))

        try:
            service = [x for x in self.all_services.get('services', []) if \
                x.get('service-id') == 'wcs' and x.get('base_url') == base_url and
                not x.get('secondary')][0]
        except IndexError:
            return

        service['this'] = True
        if base_url.endswith('/'): # wcs doesn't expect a trailing slash
            service['base_url'] = base_url[:-1]

        pub.app_dir = os.path.join(global_app_dir,
                self.get_instance_path(service))
        if not os.path.exists(pub.app_dir):
            print 'initializing instance in', pub.app_dir
            os.mkdir(pub.app_dir)
            pub.initialize_app_dir()

            if service.get('template_name'):
                skeleton_filepath = os.path.join(global_app_dir, 'skeletons',
                        service.get('template_name'))
                if os.path.exists(skeleton_filepath):
                    pub.import_zip(file(skeleton_filepath))
            new_site = True
        else:
            print 'updating instance in', pub.app_dir
            new_site = False

        try:
            self.configure_site_options(service, pub,
                    ignore_timestamp=sub_options.ignore_timestamp)
        except NoChange:
            print '  skipping'
            return

        pub.set_config()
        self.update_configuration(service, pub)
        self.configure_authentication_methods(service, pub)
        if new_site:
            self.configure_sql(service, pub)

        self.update_profile(self.all_services.get('profile', {}), pub)
        # Store hobo.json
        atomic_write(os.path.join(pub.app_dir, 'hobo.json'), json.dumps(self.all_services))

    def update_configuration(self, service, pub):
        if not pub.cfg.get('misc'):
            pub.cfg['misc'] = {}
        pub.cfg['misc']['sitename'] = service.get('title').encode('utf-8')
        pub.cfg['misc']['frontoffice-url'] = service.get('base_url').encode('utf-8')
        if not pub.cfg.get('language'):
            pub.cfg['language'] = {'language': 'fr'}

        if not pub.cfg.get('emails'):
            pub.cfg['emails'] = {}

        if not pub.cfg.get('branding'):
            pub.cfg['branding'] = {'theme': 'publik'}

        variables = self.all_services.get('variables') or {}
        variables.update(service.get('variables') or {})
        if variables.get('default_from_email'):
            pub.cfg['emails']['from'] = variables.get('default_from_email').encode('utf-8')
        if variables.get('email_signature') is not None:
            pub.cfg['emails']['footer'] = variables.get('email_signature').encode('utf-8')

        pub.write_cfg()

    def update_profile(self, profile, pub):
        formdef = UserFieldsFormDef(publisher=pub)
        profile_fields = {}
        profile_field_ids = ['_' + x['name'] for x in profile.get('fields', [])]
        for field in formdef.fields:
            if field.id in profile_field_ids:
                profile_fields[field.id] = field

        # create or update profile fields
        for attribute in profile.get('fields', []):
            field_id = '_' + attribute['name']
            if not field_id in profile_fields:
                field_class, field_typename = StringField, 'string'
                if attribute['kind'] == 'email':
                    field_class, field_typename = EmailField, 'email'
                new_field = field_class(label=attribute['label'].encode('utf-8'),
                        type=field_typename,
                        varname=attribute['name'])
                new_field.id = field_id
                profile_fields[field_id] = new_field
            else:
                # remove it for the moment
                formdef.fields.remove(profile_fields[field_id])

            profile_fields[field_id].label = attribute['label'].encode('utf-8')
            profile_fields[field_id].hint = attribute['description'].encode('utf-8')
            profile_fields[field_id].required = attribute['required']

            if attribute['disabled']:
                profile_field_ids.remove('_' + attribute['name'])

        # insert profile fields at the beginning
        formdef.fields = [profile_fields[x] for x in profile_field_ids] + formdef.fields
        formdef.store()

        pub.cfg['users']['field_email'] = '_email'
        pub.cfg['users']['field_name'] = ['_first_name', '_last_name']
        pub.write_cfg()

        # add mapping for SAML provisioning
        for idp in pub.cfg.get('idp', {}).values():
            if not idp.get('attribute-mapping'):
                idp['attribute-mapping'] = {}
            for field in profile.get('fields', []):
                attribute_name = field['name']
                field_id = '_' + attribute_name
                if field_id in profile_field_ids:
                    idp['attribute-mapping'][str(attribute_name)] = str(field_id)
        pub.write_cfg()

    def configure_authentication_methods(self, service, pub):
        # look for an identity provider
        idps = [x for x in self.all_services.get('services', []) if x.get('service-id') == 'authentic']
        if not pub.cfg.get('identification'):
            pub.cfg['identification'] = {}
        methods = pub.cfg['identification'].get('methods', [])
        if idps and not 'idp' in methods:
            methods.append('idp')
        elif not idps and not 'password' in methods:
            methods.append('password')
        pub.cfg['identification']['methods'] = methods
        if not pub.cfg.get('sp'):
            pub.cfg['sp'] = {}
        pub.cfg['sp']['idp-manage-user-attributes'] = bool(idps)
        pub.cfg['sp']['idp-manage-roles'] = bool(idps)
        pub.write_cfg()

        if not idps:
            return

        # initialize service provider side
        if not pub.cfg['sp'].get('publickey'):
            from qommon.ident.idp import MethodAdminDirectory
            spconfig = pub.cfg['sp']
            spconfig['saml2_base_url'] = str(service.get('base_url')) + '/saml'
            spconfig['saml2_providerid'] = spconfig['saml2_base_url'] + '/metadata'
            MethodAdminDirectory().generate_rsa_keypair()

        if not 'saml_identities' in pub.cfg:
            pub.cfg['saml_identities'] = {}

        if idps:
            pub.cfg['saml_identities']['identity-creation'] = 'self'

        # write down configuration to disk as it will get reloaded
        # automatically and we don't want to lose our changes.
        pub.write_cfg()

        for idp in idps:
            if not idp['base_url'].endswith('/'):
                idp['base_url'] = idp['base_url'] + '/'
            metadata_url = '%sidp/saml2/metadata' % idp['base_url']
            try:
                rfd = urllib2.urlopen(metadata_url)
            except (urllib2.HTTPError, urllib2.URLError), e:
                print >> sys.stderr, 'failed to get metadata URL', metadata_url, e
                continue
            except Exception, e:
                print >> sys.stderr, 'failed to get metadata URL', metadata_url, e
                continue

            s = rfd.read()
            (bfd, metadata_pathname) = tempfile.mkstemp('.metadata')
            atomic_write(metadata_pathname, s)

            from qommon.ident.idp import AdminIDPDir
            admin_dir = AdminIDPDir()
            key_provider_id = admin_dir.submit_new_remote(
                    metadata_pathname, None, metadata_url, None)
            admin_attribute = service.get('variables', {}).get('admin-attribute')
            if not admin_attribute:
                admin_attribute = 'is_superuser=true'
            else:
                admin_attribute = unicode(admin_attribute).encode('utf-8')
            admin_attribute_dict = dict([admin_attribute.split('=')])
            pub.cfg['idp'][key_provider_id]['admin-attributes'] = admin_attribute_dict
            pub.cfg['idp'][key_provider_id]['nameidformat'] = 'unspecified'
            pub.cfg['saml_identities']['registration-url'] = str(
                    '%saccounts/register/' % idp['base_url'])
            pub.write_cfg()

    def get_instance_path(self, service):
        parsed_url = urllib2.urlparse.urlsplit(service.get('base_url'))
        instance_path = parsed_url.netloc
        if parsed_url.path:
            instance_path = '%s+' % parsed_url.path.replace('/', '+')
        return instance_path

    def configure_site_options(self, current_service, pub, ignore_timestamp=False):
        # configure site-options.cfg
        config = ConfigParser.RawConfigParser()
        site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
        if os.path.exists(site_options_filepath):
            config.read(site_options_filepath)

        if not ignore_timestamp:
            try:
                if config.get('hobo', 'timestamp') == self.all_services.get('timestamp'):
                    raise NoChange()
            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
                pass

        if not 'hobo' in config.sections():
            config.add_section('hobo')
        config.set('hobo', 'timestamp', self.all_services.get('timestamp'))

        if not 'options' in config.sections():
            config.add_section('options')

        variables = {}
        api_secrets = {}
        for service in self.all_services.get('services', []):
            if service.get('slug'):
                variables['%s_url' % service.get('slug').replace('-', '_')] = service.get('base_url')
            # ignore current service to prevent self-domain = secret_key ^ secret_key = 0
            if service is current_service:
                continue
            if not service.get('secret_key'):
                continue
            domain = urlparse.urlparse(service.get('base_url')).netloc.split(':')[0]
            api_secrets[domain] = self.shared_secret(current_service.get('secret_key'), service.get('secret_key'))
            if service.get('service-id') == 'combo':
                if service.get('template_name') == 'portal-agent':
                    variables['portal_agent_url'] = service.get('base_url')
                    variables['portal_agent_title'] = service.get('title')
                elif service.get('template_name') == 'portal-user':
                    variables['portal_url'] = service.get('base_url')
                    config.set('options', 'theme_skeleton_url',
                            service.get('base_url') + '__skeleton__/')
            if service.get('service-id') == 'authentic':
                variables['idp_account_url'] = service.get('base_url') + 'accounts/'
        if self.all_services.get('variables'):
            for key, value in self.all_services.get('variables').items():
                variables[key] = value
        for key, value in current_service.get('variables', {}).items():
            variables[key] = value

        if variables:
            if not 'variables' in config.sections():
                config.add_section('variables')
            for key, value in variables.items():
                key = unicode(key).encode('utf-8')
                value = unicode(value).encode('utf-8')
                config.set('variables', key, value)

        if not 'api-secrets' in config.sections():
            config.add_section('api-secrets')
        if not 'wscall-secrets' in config.sections():
            config.add_section('wscall-secrets')
        for key, value in api_secrets.items():
            config.set('api-secrets', key, value)
            # for now the secrets are the same whatever the direction is.
            config.set('wscall-secrets', key, value)

        # add known services
        for service in self.all_services.get('services', []):
            if service.get('service-id') == 'fargo':
                config.set('options', 'fargo_url', service.get('base_url'))
            if service.get('service-id') == 'welco':
                config.set('options', 'welco_url', service.get('base_url'))

        try:
            portal_agent_url = config.get('variables', 'portal_agent_url')
        except ConfigParser.NoOptionError:
            pass
        else:
            if portal_agent_url.endswith('/'):
                portal_agent_url = portal_agent_url.rstrip('/')
            extra_head = '''<script src="%s/__services.js"></script>'''\
                         '''<script src="%s/static/js/publik.js"></script>''' % (
                                 portal_agent_url, portal_agent_url)
            config.set('options', 'backoffice_extra_head', extra_head)

        with open(site_options_filepath, 'wb') as site_options:
            config.write(site_options)

    def configure_sql(self, service, pub):
        if not pub.cfg.get('postgresql'):
            return

        if not pub.has_site_option('postgresql'):
            return

        import psycopg2
        import psycopg2.errorcodes

        # determine database name using the instance path; if the template
        # database name contained an underscore character, use the first part
        # as a prefix to the database name
        database_name = pub.cfg['postgresql'].get('database', 'wcs')
        domain_table_name = self.get_instance_path(service).replace(
                '-', '_').replace('.', '_').replace('+', '_')
        if not domain_table_name in database_name:
            database_name = '%s_%s' % (database_name.split('_')[0], domain_table_name)

        postgresql_cfg = {}
        for k, v in pub.cfg['postgresql'].items():
            if v:
                postgresql_cfg[k] = v
        try:
            pgconn = psycopg2.connect(**postgresql_cfg)
        except psycopg2.Error as e:
            print >> sys.stderr, 'failed to connect to postgresql (%s)' % \
                    psycopg2.errorcodes.lookup(e.pgcode)
            return

        pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
        cur = pgconn.cursor()
        new_database = True
        try:
            cur.execute('''CREATE DATABASE %s''' % database_name)
        except psycopg2.Error as e:
            if e.pgcode == psycopg2.errorcodes.DUPLICATE_DATABASE:
                new_database = False
            else:
                print >> sys.stderr, 'failed to create database (%s)' % \
                        psycopg2.errorcodes.lookup(e.pgcode)
                return
        else:
            cur.close()

        postgresql_cfg['database'] = database_name
        pub.cfg['postgresql'] = postgresql_cfg
        pub.write_cfg()

        if not new_database:
            return

        cmd = [sys.argv[0]]
        if self.base_options.configfile:
            cmd.extend(['-f', self.base_options.configfile])
        cmd.append('convert-to-sql')
        for param in ('database', 'user', 'password', 'host', 'port'):
            if postgresql_cfg.get(param):
                if param == 'database':
                    cmd.append('--dbname')
                else:
                    cmd.append('--' + param)
                cmd.append(str(postgresql_cfg.get(param)))
        cmd.append(str(self.get_instance_path(service)))
        subprocess.call(cmd)

    @classmethod
    def shared_secret(cls, secret1, secret2):
        secret1 = hashlib.sha256(secret1).hexdigest()
        secret2 = hashlib.sha256(secret2).hexdigest()
        return hex(int(secret1, 16) ^ int(secret2, 16))[2:-1]


CmdCheckHobos.register()