492 lines
20 KiB
Python
492 lines
20 KiB
Python
# w.c.s. - web application for online forms
|
|
# Copyright (C) 2005-2014 Entr'ouvert
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import ConfigParser
|
|
import json
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib2
|
|
import urlparse
|
|
import hashlib
|
|
|
|
from quixote import cleanup
|
|
from qommon.ctl import Command, make_option
|
|
from qommon.storage import atomic_write
|
|
|
|
from wcs.admin.settings import UserFieldsFormDef
|
|
from wcs.fields import StringField, EmailField
|
|
|
|
# TODO: import this from django settings
|
|
THEMES_DIRECTORY = os.environ.get('THEMES_DIRECTORY', '/usr/share/publik/themes')
|
|
|
|
|
|
class NoChange(Exception):
|
|
pass
|
|
|
|
def atomic_symlink(src, dst):
|
|
if os.path.exists(dst) and os.readlink(dst) == src:
|
|
return
|
|
if os.path.exists(dst + '.tmp'):
|
|
os.unlink(dst + '.tmp')
|
|
os.symlink(src, dst + '.tmp')
|
|
os.rename(dst + '.tmp', dst)
|
|
|
|
class CmdCheckHobos(Command):
|
|
name = 'hobo_deploy'
|
|
|
|
def __init__(self):
|
|
Command.__init__(self, [
|
|
make_option('--ignore-timestamp', action='store_true',
|
|
dest='ignore_timestamp', default=False),
|
|
make_option('--redeploy', action='store_true', default=False),
|
|
])
|
|
|
|
def execute(self, base_options, sub_options, args):
|
|
import publisher
|
|
publisher.WcsPublisher.configure(self.config)
|
|
if sub_options.redeploy:
|
|
sub_options.ignore_timestamp = True
|
|
for tenant in publisher.WcsPublisher.get_tenants():
|
|
hobo_json_path = os.path.join(publisher.WcsPublisher.APP_DIR, tenant, 'hobo.json')
|
|
if not os.path.exists(hobo_json_path):
|
|
continue
|
|
hobo_json = json.load(open(hobo_json_path))
|
|
try:
|
|
me = [service for service in hobo_json['services']
|
|
if service.get('this') is True][0]
|
|
except IndexError:
|
|
pass
|
|
else:
|
|
cleanup()
|
|
self.deploy(base_options, sub_options, [me['base_url'], hobo_json_path])
|
|
else:
|
|
self.deploy(base_options, sub_options, args)
|
|
|
|
def deploy(self, base_options, sub_options, args):
|
|
import publisher
|
|
|
|
self.base_options = base_options
|
|
if sub_options.extra:
|
|
if not self.config.has_section('extra'):
|
|
self.config.add_section('extra')
|
|
for i, extra in enumerate(sub_options.extra):
|
|
self.config.set('extra', 'cmd_line_extra_%d' % i, extra)
|
|
publisher.WcsPublisher.configure(self.config)
|
|
pub = publisher.WcsPublisher.create_publisher(
|
|
register_cron=False, register_tld_names=False)
|
|
|
|
global_app_dir = pub.app_dir
|
|
base_url = args[0]
|
|
|
|
if args[1] == '-':
|
|
# get environment definition from stdin
|
|
self.all_services = json.load(sys.stdin)
|
|
else:
|
|
self.all_services = json.load(file(args[1]))
|
|
|
|
try:
|
|
service = [x for x in self.all_services.get('services', []) if \
|
|
x.get('service-id') == 'wcs' and x.get('base_url') == base_url and
|
|
not x.get('secondary')][0]
|
|
except IndexError:
|
|
return
|
|
|
|
service['this'] = True
|
|
if base_url.endswith('/'): # wcs doesn't expect a trailing slash
|
|
service['base_url'] = base_url[:-1]
|
|
|
|
pub.app_dir = os.path.join(global_app_dir,
|
|
self.get_instance_path(service))
|
|
if not os.path.exists(pub.app_dir):
|
|
print 'initializing instance in', pub.app_dir
|
|
os.mkdir(pub.app_dir)
|
|
pub.initialize_app_dir()
|
|
|
|
if service.get('template_name'):
|
|
skeleton_filepath = os.path.join(global_app_dir, 'skeletons',
|
|
service.get('template_name'))
|
|
if os.path.exists(skeleton_filepath):
|
|
pub.import_zip(file(skeleton_filepath))
|
|
new_site = True
|
|
else:
|
|
print 'updating instance in', pub.app_dir
|
|
new_site = False
|
|
|
|
try:
|
|
self.configure_site_options(service, pub,
|
|
ignore_timestamp=sub_options.ignore_timestamp)
|
|
except NoChange:
|
|
print ' skipping'
|
|
return
|
|
|
|
pub.set_config(skip_sql=True)
|
|
if new_site:
|
|
self.configure_sql(service, pub)
|
|
self.update_configuration(service, pub)
|
|
self.configure_authentication_methods(service, pub)
|
|
|
|
self.update_profile(self.all_services.get('profile', {}), pub)
|
|
# Store hobo.json
|
|
atomic_write(os.path.join(pub.app_dir, 'hobo.json'), json.dumps(self.all_services))
|
|
|
|
def update_configuration(self, service, pub):
|
|
if not pub.cfg.get('misc'):
|
|
pub.cfg['misc'] = {'charset': 'utf-8'}
|
|
pub.cfg['misc']['sitename'] = service.get('title').encode('utf-8')
|
|
pub.cfg['misc']['frontoffice-url'] = service.get('base_url').encode('utf-8')
|
|
if not pub.cfg.get('language'):
|
|
pub.cfg['language'] = {'language': 'fr'}
|
|
|
|
if not pub.cfg.get('emails'):
|
|
pub.cfg['emails'] = {}
|
|
|
|
theme_id = self.all_services.get('variables', {}).get('theme')
|
|
if theme_id:
|
|
pub.cfg['branding'] = {'theme': 'publik-base'}
|
|
tenant_dir = pub.app_dir
|
|
theme_dir = os.path.join(tenant_dir, 'theme')
|
|
target_dir = os.path.join(THEMES_DIRECTORY, 'publik-base')
|
|
atomic_symlink(target_dir, theme_dir)
|
|
|
|
variables = self.all_services.get('variables') or {}
|
|
variables.update(service.get('variables') or {})
|
|
if variables.get('default_from_email'):
|
|
pub.cfg['emails']['from'] = variables.get('default_from_email').encode('utf-8')
|
|
if variables.get('email_signature') is not None:
|
|
pub.cfg['emails']['footer'] = variables.get('email_signature').encode('utf-8')
|
|
|
|
pub.write_cfg()
|
|
|
|
def update_profile(self, profile, pub):
|
|
formdef = UserFieldsFormDef(publisher=pub)
|
|
profile_fields = {}
|
|
profile_field_ids = ['_' + x['name'] for x in profile.get('fields', [])]
|
|
for field in formdef.fields:
|
|
if field.id in profile_field_ids:
|
|
profile_fields[field.id] = field
|
|
|
|
# create or update profile fields
|
|
for attribute in profile.get('fields', []):
|
|
field_id = '_' + attribute['name']
|
|
if not field_id in profile_fields:
|
|
field_class, field_typename = StringField, 'string'
|
|
if attribute['kind'] == 'email':
|
|
field_class, field_typename = EmailField, 'email'
|
|
new_field = field_class(label=attribute['label'].encode('utf-8'),
|
|
type=field_typename,
|
|
varname=attribute['name'])
|
|
new_field.id = field_id
|
|
profile_fields[field_id] = new_field
|
|
else:
|
|
# remove it for the moment
|
|
formdef.fields.remove(profile_fields[field_id])
|
|
|
|
profile_fields[field_id].label = attribute['label'].encode('utf-8')
|
|
profile_fields[field_id].hint = attribute['description'].encode('utf-8')
|
|
profile_fields[field_id].required = attribute['required']
|
|
|
|
if attribute['disabled']:
|
|
profile_field_ids.remove('_' + attribute['name'])
|
|
|
|
# insert profile fields at the beginning
|
|
formdef.fields = [profile_fields[x] for x in profile_field_ids] + formdef.fields
|
|
formdef.store()
|
|
|
|
pub.cfg['users']['field_email'] = '_email'
|
|
pub.cfg['users']['field_name'] = ['_first_name', '_last_name']
|
|
pub.write_cfg()
|
|
|
|
# add mapping for SAML provisioning
|
|
for idp in pub.cfg.get('idp', {}).values():
|
|
if not idp.get('attribute-mapping'):
|
|
idp['attribute-mapping'] = {}
|
|
for field in profile.get('fields', []):
|
|
attribute_name = field['name']
|
|
field_id = '_' + attribute_name
|
|
if field_id in profile_field_ids:
|
|
idp['attribute-mapping'][str(attribute_name)] = str(field_id)
|
|
pub.write_cfg()
|
|
|
|
def configure_authentication_methods(self, service, pub):
|
|
# look for an identity provider
|
|
idps = [x for x in self.all_services.get('services', []) if x.get('service-id') == 'authentic']
|
|
if not pub.cfg.get('identification'):
|
|
pub.cfg['identification'] = {}
|
|
methods = pub.cfg['identification'].get('methods', [])
|
|
if idps and not 'idp' in methods:
|
|
methods.append('idp')
|
|
elif not idps and not 'password' in methods:
|
|
methods.append('password')
|
|
pub.cfg['identification']['methods'] = methods
|
|
if not pub.cfg.get('sp'):
|
|
pub.cfg['sp'] = {}
|
|
pub.cfg['sp']['idp-manage-user-attributes'] = bool(idps)
|
|
pub.cfg['sp']['idp-manage-roles'] = bool(idps)
|
|
pub.write_cfg()
|
|
|
|
if not idps:
|
|
return
|
|
|
|
# initialize service provider side
|
|
if not pub.cfg['sp'].get('publickey'):
|
|
from qommon.ident.idp import MethodAdminDirectory
|
|
spconfig = pub.cfg['sp']
|
|
spconfig['saml2_base_url'] = str(service.get('base_url')) + '/saml'
|
|
spconfig['saml2_providerid'] = spconfig['saml2_base_url'] + '/metadata'
|
|
MethodAdminDirectory().generate_rsa_keypair()
|
|
|
|
if not 'saml_identities' in pub.cfg:
|
|
pub.cfg['saml_identities'] = {}
|
|
|
|
if idps:
|
|
pub.cfg['saml_identities']['identity-creation'] = 'self'
|
|
|
|
# write down configuration to disk as it will get reloaded
|
|
# automatically and we don't want to lose our changes.
|
|
pub.write_cfg()
|
|
|
|
for idp in idps:
|
|
if not idp['base_url'].endswith('/'):
|
|
idp['base_url'] = idp['base_url'] + '/'
|
|
metadata_url = '%sidp/saml2/metadata' % idp['base_url']
|
|
try:
|
|
rfd = urllib2.urlopen(metadata_url)
|
|
except (urllib2.HTTPError, urllib2.URLError), e:
|
|
print >> sys.stderr, 'failed to get metadata URL', metadata_url, e
|
|
continue
|
|
except Exception, e:
|
|
print >> sys.stderr, 'failed to get metadata URL', metadata_url, e
|
|
continue
|
|
|
|
s = rfd.read()
|
|
(bfd, metadata_pathname) = tempfile.mkstemp('.metadata')
|
|
atomic_write(metadata_pathname, s)
|
|
|
|
from qommon.ident.idp import AdminIDPDir
|
|
admin_dir = AdminIDPDir()
|
|
key_provider_id = admin_dir.submit_new_remote(
|
|
metadata_pathname, None, metadata_url, None)
|
|
admin_attribute = service.get('variables', {}).get('admin-attribute')
|
|
if not admin_attribute:
|
|
admin_attribute = 'is_superuser=true'
|
|
else:
|
|
admin_attribute = unicode(admin_attribute).encode('utf-8')
|
|
admin_attribute_dict = dict([admin_attribute.split('=')])
|
|
pub.cfg['idp'][key_provider_id]['admin-attributes'] = admin_attribute_dict
|
|
pub.cfg['idp'][key_provider_id]['nameidformat'] = 'unspecified'
|
|
pub.cfg['saml_identities']['registration-url'] = str(
|
|
'%saccounts/register/' % idp['base_url'])
|
|
pub.write_cfg()
|
|
|
|
def get_instance_path(self, service):
|
|
parsed_url = urllib2.urlparse.urlsplit(service.get('base_url'))
|
|
instance_path = parsed_url.netloc
|
|
if parsed_url.path:
|
|
instance_path += '+%s' % parsed_url.path.replace('/', '+')
|
|
return instance_path
|
|
|
|
def configure_site_options(self, current_service, pub, ignore_timestamp=False):
|
|
# configure site-options.cfg
|
|
config = ConfigParser.RawConfigParser()
|
|
site_options_filepath = os.path.join(pub.app_dir, 'site-options.cfg')
|
|
if os.path.exists(site_options_filepath):
|
|
config.read(site_options_filepath)
|
|
|
|
if not ignore_timestamp:
|
|
try:
|
|
if config.get('hobo', 'timestamp') == self.all_services.get('timestamp'):
|
|
raise NoChange()
|
|
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
|
|
pass
|
|
|
|
if not 'hobo' in config.sections():
|
|
config.add_section('hobo')
|
|
config.set('hobo', 'timestamp', self.all_services.get('timestamp'))
|
|
|
|
if not 'options' in config.sections():
|
|
config.add_section('options')
|
|
|
|
variables = {}
|
|
api_secrets = {}
|
|
for service in self.all_services.get('services', []):
|
|
service_url = service.get('base_url')
|
|
if not service_url.endswith('/'):
|
|
service_url += '/'
|
|
if service.get('slug'):
|
|
variables['%s_url' % service.get('slug').replace('-', '_')] = service_url
|
|
|
|
if not service.get('secret_key'):
|
|
continue
|
|
|
|
domain = urlparse.urlparse(service_url).netloc.split(':')[0]
|
|
if service is current_service:
|
|
# custom key calcultation for "self", as the shared_secret code
|
|
# would do secret_key ^ secret_key = 0.
|
|
api_secrets[domain] = self.shared_secret(current_service.get('secret_key'),
|
|
str(random.SystemRandom().random()))
|
|
continue
|
|
|
|
api_secrets[domain] = self.shared_secret(current_service.get('secret_key'), service.get('secret_key'))
|
|
if service.get('service-id') == 'combo':
|
|
if service.get('template_name') == 'portal-agent':
|
|
variables['portal_agent_url'] = service_url
|
|
variables['portal_agent_title'] = service.get('title')
|
|
elif service.get('template_name') == 'portal-user':
|
|
variables['portal_url'] = service_url
|
|
config.set('options', 'theme_skeleton_url',
|
|
service.get('base_url') + '__skeleton__/')
|
|
if service.get('service-id') == 'authentic':
|
|
variables['idp_account_url'] = service_url + 'accounts/'
|
|
if self.all_services.get('variables'):
|
|
for key, value in self.all_services.get('variables').items():
|
|
variables[key] = value
|
|
for key, value in current_service.get('variables', {}).items():
|
|
variables[key] = value
|
|
|
|
if variables:
|
|
if not 'variables' in config.sections():
|
|
config.add_section('variables')
|
|
for key, value in variables.items():
|
|
key = unicode(key).encode('utf-8')
|
|
value = unicode(value).encode('utf-8')
|
|
config.set('variables', key, value)
|
|
|
|
if not 'api-secrets' in config.sections():
|
|
config.add_section('api-secrets')
|
|
if not 'wscall-secrets' in config.sections():
|
|
config.add_section('wscall-secrets')
|
|
for key, value in api_secrets.items():
|
|
config.set('api-secrets', key, value)
|
|
# for now the secrets are the same whatever the direction is.
|
|
config.set('wscall-secrets', key, value)
|
|
|
|
# add known services
|
|
for service in self.all_services.get('services', []):
|
|
if service.get('secondary'):
|
|
continue
|
|
if service.get('service-id') == 'fargo':
|
|
config.set('options', 'fargo_url', service.get('base_url'))
|
|
if service.get('service-id') == 'welco':
|
|
config.set('options', 'welco_url', service.get('base_url'))
|
|
|
|
try:
|
|
portal_agent_url = config.get('variables', 'portal_agent_url')
|
|
except ConfigParser.NoOptionError:
|
|
pass
|
|
else:
|
|
if portal_agent_url.endswith('/'):
|
|
portal_agent_url = portal_agent_url.rstrip('/')
|
|
extra_head = '''<script src="%s/__services.js"></script>'''\
|
|
'''<script src="%s/static/js/publik.js"></script>''' % (
|
|
portal_agent_url, portal_agent_url)
|
|
config.set('options', 'backoffice_extra_head', extra_head)
|
|
|
|
with open(site_options_filepath, 'wb') as site_options:
|
|
config.write(site_options)
|
|
|
|
def configure_sql(self, service, pub):
|
|
if not pub.cfg.get('postgresql'):
|
|
return
|
|
|
|
if not pub.has_site_option('postgresql'):
|
|
return
|
|
|
|
import psycopg2
|
|
import psycopg2.errorcodes
|
|
|
|
# determine database name using the instance path
|
|
domain_table_name = self.get_instance_path(service).replace(
|
|
'-', '_').replace('.', '_').replace('+', '_')
|
|
|
|
if pub.cfg['postgresql'].get('database-template-name'):
|
|
database_template_name = pub.cfg['postgresql'].pop('database-template-name')
|
|
database_name = (database_template_name % domain_table_name).strip('_')
|
|
else:
|
|
# legacy way to create a database name, if it contained an
|
|
# underscore character, use the first part as a prefix
|
|
database_name = pub.cfg['postgresql'].get('database', 'wcs')
|
|
if not domain_table_name in database_name:
|
|
database_name = '%s_%s' % (database_name.split('_')[0], domain_table_name)
|
|
|
|
createdb_cfg = pub.cfg['postgresql'].get('createdb-connection-params')
|
|
if not createdb_cfg:
|
|
createdb_cfg = {}
|
|
for k, v in pub.cfg['postgresql'].items():
|
|
if v and isinstance(v, basestring):
|
|
createdb_cfg[k] = v
|
|
|
|
try:
|
|
pgconn = psycopg2.connect(**createdb_cfg)
|
|
except psycopg2.Error as e:
|
|
print >> sys.stderr, 'failed to connect to postgresql (%s)' % \
|
|
psycopg2.errorcodes.lookup(e.pgcode)
|
|
return
|
|
|
|
pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
|
|
cur = pgconn.cursor()
|
|
new_database = True
|
|
try:
|
|
cur.execute('''CREATE DATABASE %s''' % database_name)
|
|
except psycopg2.Error as e:
|
|
if e.pgcode == psycopg2.errorcodes.DUPLICATE_DATABASE:
|
|
cur.execute("""SELECT table_name
|
|
FROM information_schema.tables
|
|
WHERE table_schema = 'public'
|
|
AND table_type = 'BASE TABLE'
|
|
AND table_name = 'wcs_meta'""")
|
|
|
|
if cur.fetchall():
|
|
new_database = False
|
|
else:
|
|
print >> sys.stderr, 'failed to create database (%s)' % \
|
|
psycopg2.errorcodes.lookup(e.pgcode)
|
|
return
|
|
else:
|
|
cur.close()
|
|
|
|
pub.cfg['postgresql']['database'] = database_name
|
|
pub.write_cfg()
|
|
pub.set_config(skip_sql=False)
|
|
|
|
if not new_database:
|
|
return
|
|
|
|
cmd = [sys.argv[0]]
|
|
if self.base_options.configfile:
|
|
cmd.extend(['-f', self.base_options.configfile])
|
|
cmd.append('convert-to-sql')
|
|
for param in ('database', 'user', 'password', 'host', 'port'):
|
|
if pub.cfg['postgresql'].get(param):
|
|
if param == 'database':
|
|
cmd.append('--dbname')
|
|
else:
|
|
cmd.append('--' + param)
|
|
cmd.append(str(pub.cfg['postgresql'].get(param)))
|
|
cmd.append(str(self.get_instance_path(service)))
|
|
subprocess.call(cmd)
|
|
|
|
@classmethod
|
|
def shared_secret(cls, secret1, secret2):
|
|
secret1 = hashlib.sha256(secret1).hexdigest()
|
|
secret2 = hashlib.sha256(secret2).hexdigest()
|
|
return hex(int(secret1, 16) ^ int(secret2, 16))[2:-1]
|
|
|
|
|
|
CmdCheckHobos.register()
|