hobo, combo, authentic install

This commit is contained in:
Emmanuel Cazenave 2018-01-09 14:30:23 +01:00
parent ce9cc30a2a
commit 5da32b9072
26 changed files with 646 additions and 71 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
*.retry

View File

@ -1,27 +1,75 @@
Dev install of publik
=====================
Description
+++++++++++
This will install a publik instance on localhost using sources (git) and pip.
Ansible playbook that install and setup a multitenant publik instance using sources.
Dependencies
++++++++++++
Setup
=====
python-lasso, git, ansible.
Tested on debian testing.
Usage
+++++
Fill the varibles i the 'local' file :
pipexe: path of the pip executable to use (it can be the pip of a virtualenv)
srcdir: path of the directory where the sources are going to be cloned
Install dependencies
++++++++++++++++++++
.. code-block:: bash
ansible-playbook -i local -K main.yml
pip install --user -r requirements.txt
System requirements
+++++++++++++++++++
* postgresql server must be installed
* the system user used to run ansible must be a sudoer
* the system user used to run ansible must to be able to connect to the postgres server withpout authentication and must be authorized to create databases
* you need to have a valid SSL certificate ('ssl_certificate' and 'ssl_certificate_key' in configuration file)
Description
===========
The configuration file ('inventory' in ansible language) is inventory.yml, change it to suite your needs.
Important configuration variables :
* 'user' : must be the name of the system user used to run the playbook (DRY I know, but tell ansible about it)
* the 'ssl_certificate' used must be valid for the patterns delared in 'apps.host_pattern' and app.nginx_host_pattern
* 'apps.host_pattern' is used in the app server settings
* 'apps.nginx_host_pattern' is used in the nginx app settings
The playbook will first :
* install some system packages (see roles/base)
* create a virtualenv ('venv' variable)
Then for each 'app' registered in the 'apps' section of the configuration file, the following events will occur:
* git clone of the repository in 'src_dir' dir and install from sources
* create one configuration file for the app and one for its hobo agent (in 'venv_conf' directory)
* create app-manage, app-server and app-agent scripts in 'venv_bin' directory
* create app-server and app-agent supervisord configuration
* create /var/lib/app and /var/log/app directories
* create a nginx configuration for server-app
* create a database for the app and create its schema
Usage
=====
Install publik
.. code-block:: bash
ansible-playbook -i inventory.yml -K install.yml
Send publik to the cemetery
.. code-block:: bash
ansible-playbook -i inventory.yml -K clean.yml
Next
====
Help yourself with your DNS settings and create a tenant using cook

75
clean.yml Normal file
View File

@ -0,0 +1,75 @@
---
- hosts: local
tasks:
- name: delete venv
file:
path: "{{venv}}"
state: absent
- name: delete app /var/lib directory
file:
path: "/var/lib/{{item.value.project_name}}"
state: absent
with_dict: "{{apps}}"
become: yes
- name: delete server /var/log directory
file:
path: "/var/log/{{item.value.project_name}}-server"
state: absent
with_dict: "{{apps}}"
become: yes
- name: delete agent /var/log directory
file:
path: "/var/log/{{item.value.project_name}}-agent"
state: absent
with_dict: "{{apps}}"
become: yes
- name: delete supervisor server conf
file:
path: "/etc/supervisor/conf.d/{{item.value.project_name}}-server.conf"
state: absent
with_dict: "{{apps}}"
become: yes
- name: delete supervisor agent conf
file:
path: "/etc/supervisor/conf.d/{{item.value.project_name}}-agent.conf"
state: absent
with_dict: "{{apps}}"
become: yes
- name: reload supervisor
systemd:
name: supervisor
state: reloaded
become: yes
- name: delete nginx app server link conf
file:
path: "/etc/nginx/sites-enabled/{{item.value.project_name}}"
state: absent
with_dict: "{{apps}}"
become: yes
- name: delete nginx app server conf
file:
path: "/etc/nginx/sites-available/{{item.value.project_name}}"
state: absent
with_dict: "{{apps}}"
become: yes
- name: reload nginx
systemd:
name: nginx
state: reloaded
become: yes
- name: delete db
postgresql_db:
name: "{{item.value.db_name}}"
state: absent
with_dict: "{{apps}}"

9
install.yml Normal file
View File

@ -0,0 +1,9 @@
---
- name: publik multitenant install
hosts: local
roles:
- base
- hobo
- authentic
- combo
- end

47
inventory.yml Normal file
View File

@ -0,0 +1,47 @@
local:
hosts:
localhost:
ansible_connection: local
vars:
user: cazino
django_version: 1.8.18
src_dir: "/home/{{user}}/src"
ssl_certificate: "/etc/ssl/certs/*.local.publik.crt"
ssl_certificate_key: "/etc/ssl/private/*.local.publik.key"
venv: "/home/{{user}}/envs/publik-env"
venv_bin: "{{venv}}/bin"
venv_conf: "{{venv}}/conf"
venv_pip: "{{venv_bin}}/pip"
venv_python: "{{venv_bin}}/python"
apps:
authentic:
name: authentic
db_name: authentic_multitenant
hobo_app: "hobo.agent.authentic2"
host_pattern: "*-authentic.local.publik"
manage_cmd: "{{venv_bin}}/authentic2-ctl"
nginx_host_pattern: '~^(.*)\-authentic\.local\.publik$'
project_name: authentic-multitenant
server_port: 8000
settings_env_var: AUTHENTIC2_SETTINGS_FILE
combo:
name: combo
db_name: combo_multitenant
hobo_app: "hobo.agent.combo"
host_pattern: "*-combo.local.publik"
nginx_host_pattern: '~^(.*)\-combo\.local\.publik$'
project_name: combo-multitenant
server_port: 8004
settings_env_var: COMBO_SETTINGS_FILE
hobo:
name: hobo
db_name: hobo_multitenant
hobo_app: "hobo.agent.hobo"
host_pattern: "*-hobo.local.publik"
nginx_host_pattern: '~^(.*)\-hobo\.local\.publik$'
project_name: hobo-multitenant
server_port: 8016
settings_env_var: HOBO_SETTINGS_FILE

View File

@ -1,5 +0,0 @@
---
- name: Authentic
hosts: all
roles:
- authentic

7
pip_install_source.yml Normal file
View File

@ -0,0 +1,7 @@
# Ugly hack to perform a simple 'pip install -e /some/source/dir'
- name: pip install source
pip:
name: django
version: "{{ django_version}}"
extra_args: "-e {{ source }}"
virtualenv: "{{venv}}"

1
requirements.txt Normal file
View File

@ -0,0 +1 @@
ansible>2.4.0

View File

@ -0,0 +1,30 @@
- name: set ansible agent variables
set_fact:
agent_dash_name: "{{app['project_name']}}-agent"
agent_dot_name: "{{app['project_name']}}.agent"
agent_settings: "{{venv_conf}}/{{app['db_name']}}_agent_settings.py"
- name: create hobo agent log directory
file:
path: "/var/log/{{agent_dash_name}}"
state: directory
owner: "{{user}}"
group: "{{user}}"
become: yes
- name: hobo agent settings file
template:
src: hobo-agent-settings.j2
dest: "{{agent_settings}}"
- name: agent script
template:
src: hobo-agent.j2
dest: "{{venv_bin}}/{{agent_dash_name}}"
mode: "u=rwx,g=rx,o=rx"
- name: hobo agent supervisor configuration
template:
src: hobo-agent-supervisor.j2
dest: "/etc/supervisor/conf.d/{{agent_dash_name}}.conf"
become: yes

View File

@ -0,0 +1,9 @@
BROKER_URL = 'amqp://'
AGENT_HOST_PATTERNS = {
"{{app['name']}}": ["{{app['host_pattern']}}"],
}
{% filter upper %}{{app['name']}}{% endfilter %}_MANAGE_COMMAND = "{{venv_bin}}/{{app['project_name']}}-manage"
{% filter upper %}{{app['name']}}{% endfilter %}_MANAGE_TRY_COMMAND = {% filter upper %}{{app['name']}}{% endfilter %}_MANAGE_COMMAND

View File

@ -0,0 +1,26 @@
[program:{{agent_dash_name}}]
; Concurrency set to 1 because there is no lock around calls to hobo_notify
command={{venv_bin}}/celery worker --hostname={{agent_dot_name}}.%%h --app=hobo.agent.worker --loglevel=INFO --concurrency=1
environment=HOBO_AGENT_SETTINGS_FILE="{{agent_settings}}"
process_name={{agent_dash_name}}
user={{user}}
numprocs=1
stdout_logfile=/var/log/{{agent_dash_name}}/stdout.log
stderr_logfile=/var/log/{{agent_dash_name}}/stderr.log
autostart=true
autorestart=true
startsecs=10
; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 600
; When resorting to send SIGKILL to the program to terminate it
; send SIGKILL to its whole process group instead,
; taking care of its children as well.
killasgroup=true
; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=998

View File

@ -0,0 +1,2 @@
#!/bin/bash
HOBO_AGENT_SETTINGS_FILE="{{agent_settings}}" {{venv_bin}}/celery worker --hostname={{agent_dot_name}}.%%h --app=hobo.agent.worker --loglevel=INFO --concurrency=1

View File

@ -0,0 +1,71 @@
- name: set ansible server variables
set_fact:
app_settings: "{{venv_conf}}/{{app['db_name']}}_settings.py"
manage_app_name: "{{app['project_name']}}-manage"
server_app_name: "{{app['project_name']}}-server"
- name: create db
postgresql_db:
name: "{{app['db_name']}}"
owner: "{{user}}"
- name: conf directory
file:
path: "{{venv_conf}}"
state: directory
owner: "{{user}}"
group: "{{user}}"
become: yes
- name: app /var/lib directory
file:
path: "/var/lib/{{app['project_name']}}"
state: directory
owner: "{{user}}"
group: "{{user}}"
become: yes
- name: app /var/lib tenants directory
file:
path: "/var/lib/{{app['project_name']}}/tenants"
state: directory
owner: "{{user}}"
group: "{{user}}"
become: yes
- name: app settings file
template:
src: app-settings.j2
dest: "{{app_settings}}"
- name: manage script
template:
src: app-manage.j2
dest: "{{venv_bin}}/{{manage_app_name}}"
mode: "u=rwx,g=rx,o=rx"
- name: migrate schemas
command: "{{venv_bin}}/{{manage_app_name}} migrate_schemas"
- name: collect statics
shell: "echo yes | {{venv_bin}}/{{manage_app_name}} collectstatic"
- name: server script
template:
src: app-server.j2
dest: "{{venv_bin}}/{{server_app_name}}"
mode: "u=rwx,g=rx,o=rx"
- name: server log directory
file:
path: "/var/log/{{server_app_name}}"
state: directory
owner: "{{user}}"
group: "{{user}}"
become: yes
- name: server supervisor configuration
template:
src: server-supervisor.j2
dest: "/etc/supervisor/conf.d/{{server_app_name}}.conf"
become: yes

View File

@ -0,0 +1,2 @@
#!/bin/bash
{{app['settings_env_var']}}={{app_settings}} {% if 'manage_cmd' in app %}{{app['manage_cmd']}}{% else %}{{venv_python}} {{src_dir}}/{{app['name']}}/manage.py{% endif%} "$@"

View File

@ -0,0 +1,2 @@
#!/bin/bash
{{venv_bin}}/{{manage_app_name}} runserver {{app['server_port']}}

View File

@ -0,0 +1,20 @@
import os
PROJECT_NAME = '{{app['project_name']}}'
execfile('{{src_dir}}/hobo/debian/debian_config_common.py')
# hobo don't use multitenant mellon adapter: IdP is detected in the local
# environnment
MELLON_ADAPTER = ('hobo.utils.MellonAdapter',)
LOGGING['loggers']['']['handlers'] = ['syslog']
{% if 'hobo_app' in app %}
INSTALLED_APPS = ('{{app['hobo_app']}}', ) + INSTALLED_APPS
{% endif %}
os.environ['REQUESTS_CA_BUNDLE'] = '{{ssl_certificate}}'
ALLOWED_HOSTS = ['*']
DEBUG = True

View File

@ -0,0 +1,28 @@
[program:{{server_app_name}}]
; Concurrency set to 1 because there is no lock around calls to hobo_notify
command={% if 'manage_cmd' in app %}{{app['manage_cmd']}}{% else %}{{venv_python}} {{src_dir}}/{{app['name']}}/manage.py{% endif%} runserver {{app['server_port']}}
environment={{app['settings_env_var']}}="{{app_settings}}"
process_name={{server_app_name}}
user={{user}}
numprocs=1
stdout_logfile=/var/log/{{server_app_name}}/stdout.log
stderr_logfile=/var/log/{{server_app_name}}/stderr.log
autostart=true
autorestart=true
startsecs=10
; Need to wait for currently executing tasks to finish at shutdown.
; Increase this if you have very long running tasks.
stopwaitsecs = 600
stopsignal=KILL
; When resorting to send SIGKILL to the program to terminate it
; send SIGKILL to its whole process group instead,
; taking care of its children as well.
killasgroup=true
stopasgroup=true
; if rabbitmq is supervised, set its priority higher
; so it starts first
priority=998

View File

@ -1,55 +1,34 @@
- name: Install system depedencies
apt:
name: "{{ item }}"
state: installed
with_items:
- libsasl2-dev
- python-dev
- libldap2-dev
- libssl-dev
become: yes
- name: Git clone django-mellon
git:
repo: ssh://git@git.entrouvert.org/django-mellon.git
dest: "{{ srcdir }}/django-mellon"
- name: Pip install django
pip:
name: django
version: "{{ djangoversion}}"
executable: "{{ pipexe }}"
- name: Pip install django-mellon
pip:
name: django
version: "{{ djangoversion}}"
extra_args: "-e {{ srcdir }}/django-mellon"
executable: "{{ pipexe }}"
- name: Git clone gadjo
git:
repo: ssh://git@git.entrouvert.org/gadjo.git
dest: "{{ srcdir }}/gadjo"
- name: Pip install gadjo
pip:
name: django
version: "{{ djangoversion}}"
extra_args: "-e {{ srcdir }}/gadjo"
executable: "{{ pipexe }}"
- name: Git clone authentic
- name: git clone authentic
git:
repo: ssh://git@git.entrouvert.org/authentic.git
dest: "{{ srcdir }}/authentic"
dest: "{{ src_dir }}/authentic"
- name: Pip install authentic
- name: pip install authentic
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/authentic"
# FIXME: change authentic settings instead ?
- name: install python-memcached
pip:
name: django
version: "{{ djangoversion}}"
extra_args: "-e {{ srcdir }}/authentic"
executable: "{{ pipexe }}"
name: python-memcached
virtualenv: "{{venv}}"
- name: authentic app setup
import_role:
name: app-setup
vars:
app: "{{apps['authentic']}}"
- name: authentic hobo agent
import_role:
name: agent-setup
vars:
app: "{{apps['authentic']}}"
- name: authentic nginx setup
import_role:
name: nginx-setup
vars:
app: "{{apps['authentic']}}"

38
roles/base/tasks/main.yml Normal file
View File

@ -0,0 +1,38 @@
---
- name: Install system packages depedencies
apt:
name: "{{ item }}"
state: installed
with_items:
- libldap2-dev
- libsasl2-dev
- libssl-dev
- nginx
- python-dev
- python-lasso
- python-pip
- python-virtualenv
- rabbitmq-server
- supervisor
become: yes
- name: create the virtualenv and install django
pip:
name: django
version: "{{ django_version}}"
virtualenv: "{{venv}}"
# FIXME: needed by XXX, we should change XXX settings instead ?
- name: install raven
pip:
name: raven
virtualenv: "{{venv}}"
- name: create getlasso script
template:
src: getlasso.j2
dest: "{{venv_bin}}/getlasso.sh"
mode: "u=rwx,g=rx,o=rx"
- name: execute getlasso
command: "{{venv_bin}}/getlasso.sh"

View File

@ -0,0 +1,17 @@
#!/bin/sh
# Get venv site-packages path
DSTDIR=`{{venv_python}} -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'`
SRCDIR=`python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'`
# Clean up
rm -f $DSTDIR/lasso.*
rm -f $DSTDIR/_lasso.*
# Link
ln -sv $SRCDIR/lasso.py $DSTDIR
ln -sv $SRCDIR/_lasso.* $DSTDIR
exit 0

View File

@ -0,0 +1,28 @@
- name: Git clone combo
git:
repo: ssh://git@git.entrouvert.org/combo.git
dest: "{{ src_dir }}/combo"
- name: Pip install combo
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/combo"
- name: Combo app setup
import_role:
name: app-setup
vars:
app: "{{apps['combo']}}"
- name: Combo hobo agent
import_role:
name: agent-setup
vars:
app: "{{apps['combo']}}"
- name: combo nginx setup
import_role:
name: nginx-setup
vars:
app: "{{apps['combo']}}"

11
roles/end/tasks/main.yml Normal file
View File

@ -0,0 +1,11 @@
- name: reload supervisor
systemd:
name: supervisor
state: reloaded
become: yes
- name: reload nginx
systemd:
name: nginx
state: reloaded
become: yes

63
roles/hobo/tasks/main.yml Normal file
View File

@ -0,0 +1,63 @@
- name: git clone hobo
git:
repo: ssh://git@git.entrouvert.org/hobo.git
dest: "{{ src_dir }}/hobo"
- name: get lasso
command: "bash getlasso.sh"
args:
chdir: "{{ src_dir }}/hobo/"
- name: git clone django-mellon
git:
repo: ssh://git@git.entrouvert.org/django-mellon.git
dest: "{{ src_dir }}/django-mellon"
- name: pip install django-mellon
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/django-mellon"
- name: git clone gadjo
git:
repo: ssh://git@git.entrouvert.org/gadjo.git
dest: "{{ src_dir }}/gadjo"
- name: pip install gadjo
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/gadjo"
- name: git clone django-tenant-schemas
git:
repo: ssh://git@git.entrouvert.org/debian/django-tenant-schemas.git
dest: "{{ src_dir }}/django-tenant-schemas"
- name: pip install django-tenant-schemas
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/django-tenant-schemas"
- name: pip install hobo
import_tasks: pip_install_source.yml
vars:
source: "{{src_dir}}/hobo"
- name: hobo app setup
import_role:
name: app-setup
vars:
app: "{{apps['hobo']}}"
- name: hobo hobo agent
import_role:
name: agent-setup
vars:
app: "{{apps['hobo']}}"
- name: hobo nginx setup
import_role:
name: nginx-setup
vars:
app: "{{apps['hobo']}}"

View File

@ -0,0 +1,13 @@
- name: nginx server settings
template:
src: nginx-server.j2
dest: "/etc/nginx/sites-available/{{app['project_name']}}"
become: yes
- name: a2ensite nginx server
file:
src: "/etc/nginx/sites-available/{{app['project_name']}}"
path: "/etc/nginx/sites-enabled/{{app['project_name']}}"
state: link
become: yes

View File

@ -0,0 +1,41 @@
server {
listen 443 ssl;
server_name {{app['nginx_host_pattern']}};
ssl_certificate {{ssl_certificate}};
ssl_certificate_key {{ssl_certificate_key}};
access_log /var/log/nginx/{{app['project_name']}}-access.log combined;
error_log /var/log/nginx/{{app['project_name']}}-error.log;
location ~ ^/static/(.+)$ {
root /;
try_files /var/lib/{{app['project_name']}}/tenants/$host/static/$1
/var/lib/{{app['project_name']}}/collectstatic/$1
=404;
add_header Access-Control-Allow-Origin *;
}
location ~ ^/media/(.+)$ {
alias /var/lib/{{app['project_name']}}/tenants/$host/media/$1;
}
location / {
proxy_pass http://localhost:{{app['server_port']}};
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-SSL on;
proxy_set_header X-Forwarded-Protocol ssl;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
# catchall http → https
server {
listen 80;
server_name {{app['nginx_host_pattern']}};
access_log /var/log/nginx/{{app['project_name']}}-access.log combined;
error_log /var/log/nginx/{{app['project_name']}}-error.log;
return 301 https://$host$request_uri;
}

12
roles/wcs/tasks/main.yml Normal file
View File

@ -0,0 +1,12 @@
- name: Git clone wcs
git:
repo: ssh://git@git.entrouvert.org/wcs.git
dest: "{{ srcdir }}/wcs"
- name: Pip install wcs
pip:
name: django
version: "{{ djangoversion}}"
extra_args: "-e {{ srcdir }}/wcs"