summaryrefslogtreecommitdiffstats
path: root/archives/publik-move/publik-cluster-copy
diff options
context:
space:
mode:
Diffstat (limited to 'archives/publik-move/publik-cluster-copy')
-rwxr-xr-xarchives/publik-move/publik-cluster-copy227
1 files changed, 227 insertions, 0 deletions
diff --git a/archives/publik-move/publik-cluster-copy b/archives/publik-move/publik-cluster-copy
new file mode 100755
index 0000000..82ae8af
--- /dev/null
+++ b/archives/publik-move/publik-cluster-copy
@@ -0,0 +1,227 @@
+#!/usr/bin/python3
+"""
+ publik-cluster-copy tetragone.entrouvert.org welco --originbase /srv/backup/rsync/welco.test.entrouvert.org
+"""
+
+import argparse
+import os
+import pickle
+import socket
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument('action', choices=['sync', 'update', 'ls', 'take'])
+parser.add_argument('--tenant')
+parser.add_argument('--cached', action='store_true')
+parser.add_argument('--verbose', action='store_true')
+parser.add_argument('--originbase', default='')
+parser.add_argument('--auquo', action='store_true')
+parser.add_argument('--wcsdbname')
+args = parser.parse_args()
+
+environment = '.'.join(socket.getfqdn().split('.')[1:])
+base = '/srv/nfs'
+m = '%s/migration' % base
+data_file = '%s/data.p' % m
+if not os.path.isdir(m):
+ os.mkdir(m)
+if not os.path.isfile(data_file):
+ with open(data_file, 'w') as fh:
+ pickle.dump({}, open(data_file, "wb" ))
+
+wcs_password = open('%s/etc/wcs/password' % base).read().strip()
+
+data = pickle.load(open(data_file, 'rb'))
+
+if environment == 'test.saas.entrouvert.org':
+ old_environment = 'test.entrouvert.org'
+elif environment == 'prod.saas.entrouvert.org':
+ old_environment = 'entrouvert.org'
+else:
+ raise(Exception('unknown environment'))
+
+
+briques = [
+ {'name': 'authentic', 'service': 'authentic2-multitenant'},
+ {'name': 'bijoe', 'hostname': 'bi'},
+ {'name': 'chrono'}, {'name': 'combo'}, {'name': 'corbo'}, {'name': 'fargo'},
+ {'name': 'hobo'}, {'name': 'mandayejs'}, {'name': 'passerelle'},
+ {'name': 'wcs'}, {'name': 'welco' }]
+
+for i, b in enumerate(briques):
+ if not 'service' in b.keys():
+ briques[i]['service'] = b.get('name')
+ if 'hostname' in b.keys():
+ hostname = b['hostname']
+ else:
+ hostname = b.get('name')
+ briques[i]['origin'] = '%s.%s' % (hostname, old_environment)
+ briques[i]['originbase'] = '%s.%s:%s' % (hostname, old_environment, args.originbase)
+
+briques_list = [b['name'] for b in briques]
+
+
+def run(cmd):
+ if args.verbose:
+ print(cmd)
+ try:
+ o = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if args.verbose and o.stdout:
+ print(o.stdout.decode())
+ except subprocess.CalledProcessError as e:
+ print(e.stdout)
+ print(e.stderr)
+ raise
+ return o
+
+
+def container(ctn, cmd):
+ return run('systemd-run --machine %s --wait %s' % (ctn, cmd))
+
+
+def get_nginx_sites(name, originbase, **kwargs):
+ mp = '%s/migration/%s/nginx' % (base, name)
+ if not os.path.isdir(mp):
+ run('mkdir -p %s' % mp)
+ run('rsync -a %s/etc/nginx/sites-available %s' % (originbase, mp))
+ run('rsync -a %s/etc/nginx/sites-enabled %s' % (originbase, mp))
+
+
+def _get_brique_files(originbase, service, relpath, mp):
+ excludes = '--exclude "*.invalid/" --exclude whoosh_index/ --exclude form_tokens/'
+ run('rsync -av --info=progress %s --delete %s/var/lib/%s %s' % (excludes, originbase, relpath, mp))
+
+
+def get_brique_files(name, service, originbase, **kwargs):
+ mp = '%s/migration/%s' % (base, name)
+ if not os.path.isdir(mp):
+ run('mkdir -p %s' % mp)
+ if service == 'wcs' and args.tenant:
+ for tenant in args.tenant.split(','):
+ for e in data['tenants'][tenant]:
+ if e['service'] == 'wcs':
+ n = e['hostname']
+ print(n)
+ mp = '%s/migration/wcs/wcs' % base
+ _get_brique_files(originbase, service, 'wcs/%s' % n, mp)
+ else:
+ _get_brique_files(originbase, service, service, mp)
+
+
+def get_dumps(name, origin, originbase, **kwargs):
+ mp = '%s/migration/%s' % (base, name)
+ run('scp dump-schemas %s:/tmp' % origin)
+ run('ssh %s sudo -u postgres python /tmp/dump-schemas' % origin)
+ run('rsync -a --delete %s/var/lib/postgresql/migration/dump %s' % (originbase, mp))
+
+
+def import_schema(schema, db, tar):
+ ctn = 'database.node1.%s' % environment
+ container(ctn, '/usr/bin/sudo -u postgres /usr/bin/psql -c "drop schema if exists %s cascade;" %s' % (schema, db))
+ container(ctn, '/usr/bin/sudo -u postgres LANG=C /usr/bin/pg_restore -d %s %s' % (db, tar))
+
+
+def import_database(db, tar):
+ ctn = 'database.node1.%s' % environment
+ container(ctn, '/usr/bin/sudo -u postgres /usr/bin/dropdb --if-exists %s' % db)
+ container(ctn, '/usr/bin/sudo -u postgres /usr/bin/createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8;' % db)
+ container(ctn, '/usr/bin/sudo -u postgres LANG=C /usr/bin/pg_restore -d %s %s' % (db, tar))
+
+
+def take(tenant=args.tenant):
+
+ for e in data['tenants'][tenant]:
+ take_tenant(e['hostname'], e['service'], e['schema'])
+
+
+def take_tenant(hostname, service, schema):
+
+ if service not in briques_list:
+ print('Unknown service %s' % service)
+ return
+
+ if service == 'authentic':
+ daemon = 'authentic2-multitenant'
+ db = 'authentic2_multitenant'
+ else:
+ daemon = service
+ db = service
+
+ ctn = 'database.node1.%s' % environment
+ container(ctn, '/bin/chown -R postgres %s/migration/%s/dump' % (base, service))
+
+ if service == 'wcs':
+ dbname = 'wcs_%s' % hostname.replace('-', '_').replace('.', '_')
+ import_database(dbname, '%s/migration/wcs/dump/%s/public.tar' % (base, schema))
+ orig = '%s/migration/wcs/wcs/%s' % (base, hostname)
+ dest = '%s/lib/wcs/%s' % (base, hostname)
+ # adapt_wcs(orig, dbname)
+ # we need python2 pickle
+ run('./update-config-pck.py %s/config.pck %s' % (orig, dbname))
+ # adapt wcs-olap.ini, configuration is defined globally #26627
+ if os.path.isfile('%s/wcs-olap.ini' % orig):
+ run('sed s/^pg_dsn =/#pg_dsn =/ -i %s/wcs-olap.ini' % orig)
+ else:
+ import_schema(schema, db, '%s/migration/%s/dump/%s/%s.tar' % (base, service, db, schema))
+ orig = '%s/migration/%s/%s/tenants/%s' % (base, service, daemon, hostname)
+ dest = '%s/lib/%s/tenants/%s' % (base, daemon, hostname)
+
+ if os.path.isdir(dest):
+ run('rm -r %s' % dest)
+
+ run('mv %s %s' % (orig, dest))
+
+
+def sync():
+ for b in briques:
+ run('ssh %s hostname' % b.get('origin'))
+
+ for b in briques:
+ print(b['name'])
+ get_nginx_sites(**b)
+ get_brique_files(**b)
+ get_dumps(**b)
+
+
+def update():
+ if not args.cached:
+ hobo = 'hobo.%s' % old_environment
+ output = run('ssh %s sudo -u hobo hobo-manage tenant_command runscript /home/thomas/list-services.py --all-tenants -v0; ' % hobo)
+ data['tenants'] = eval("{ %s }" % output.stdout.decode())
+ pickle.dump(data, open(data_file, "wb" ))
+
+
+def ls():
+ for t in data['tenants']:
+ print(t)
+
+
+if __name__ == '__main__':
+ if args.auquo:
+ if not args.tenant:
+ raise(Exception('no tenant specified'))
+
+ if args.wcsdbname:
+ schemaname = args.wcsdbname
+ else:
+ schemaname = args.tenant.replace('-', '_').replace('.', '_')
+ origin = [b['origin'] for b in briques if b['service'] == 'wcs'][0]
+ originbase = [b['originbase'] for b in briques if b['service'] == 'wcs'][0]
+
+ if args.action == 'sync':
+ mp = '%s/migration/wcs/wcs' % base
+ _get_brique_files(originbase, 'wcs', 'wcs/%s' % args.tenant, mp)
+ get_dumps('wcs', origin, originbase)
+
+ elif args.action == 'take':
+ take_tenant(args.tenant, 'wcs', schemaname)
+
+ sys.exit(0)
+
+ if args.tenant:
+ for tenant in args.tenant.split(','):
+ if tenant not in data['tenants'].keys():
+ raise(Exception('tenant not found'))
+
+ locals()[args.action]()