diff --git a/publik-dump/.gitignore b/publik-dump/.gitignore new file mode 100644 index 0000000..53752db --- /dev/null +++ b/publik-dump/.gitignore @@ -0,0 +1 @@ +output diff --git a/publik-dump/bin/adapt_wcs_config.py b/publik-dump/bin/adapt_wcs_config.py new file mode 100755 index 0000000..2dd0af7 --- /dev/null +++ b/publik-dump/bin/adapt_wcs_config.py @@ -0,0 +1,19 @@ +#!/usr/bin/python2 +import argparse +import pickle + +parser = argparse.ArgumentParser() +parser.add_argument("file") +parser.add_argument("--host") +parser.add_argument("--password") +args = parser.parse_args() + +with open(args.file) as fh: + data = pickle.loads(fh.read()) + +data["postgresql"]["host"] = args.host +data["postgresql"]["password"] = args.password +print(data["postgresql"]) + +with open(args.file, "w") as fh: + fh.write(pickle.dumps(data)) diff --git a/publik-dump/bin/list_tenants.py b/publik-dump/bin/list_tenants.py new file mode 100644 index 0000000..f8fe01c --- /dev/null +++ b/publik-dump/bin/list_tenants.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 +import json +from urllib.parse import urlparse, urlsplit +from hobo.environment.models import AVAILABLE_SERVICES +from hobo.multitenant.middleware import TenantMiddleware +from django.db import connection + +tenant = connection.get_tenant() +services = [{"name": "hobo", "url": tenant.domain_url, "schema": tenant.schema_name}] + +for service in AVAILABLE_SERVICES: + for site in service.objects.all(): + service = { + "name": site.Extra.service_id, + "url": urlsplit(site.base_url).netloc.split(":")[0], + } + if site.secondary: + service["secondary"] = True + schema = TenantMiddleware.hostname2schema(service["url"]) + if service["name"] == "wcs": + schema = "wcs_%s" % schema + service["schema"] = schema + services.append(service) + +print(json.dumps({"name": tenant.domain_url, "services": services})) diff --git a/publik-dump/bin/publik_dump.py b/publik-dump/bin/publik_dump.py new file mode 100755 index 0000000..6a808c5 --- /dev/null +++ b/publik-dump/bin/publik_dump.py @@ -0,0 +1,164 @@ +#!/usr/bin/python3 +import argparse +import json +import os +import pickle +import subprocess + +parser = argparse.ArgumentParser() +parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore"]) +parser.add_argument("host", help="origin host") +parser.add_argument("tenant", help="hobo tenant url") +parser.add_argument("--update", action="store_true") +parser.add_argument("--target", help="destination host") +parser.add_argument("--dbtarget", help="destination host") +args = parser.parse_args() +host_folder = "output/%s" % args.host + + +def run(cmd): + print("+ %s" % cmd) + return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE) + + +def get_dump_folder(service): + dump_folder = "%s/%s" % (host_folder, service["url"]) + if not os.path.isdir(dump_folder): + os.mkdir(dump_folder) + return dump_folder + + +def dump_tenant_files(tenant): + for service in tenant["services"]: + dump_folder = get_dump_folder(service) + output = "%s/%s.tar.xz" % (dump_folder, service["url"]) + run( + "ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s" + % (service["name"], args.host, service["path"], service["url"], output) + ) + + +def restore_tenant_files(tenant): + assert run("ssh %s hostname -f" % args.target).stdout.decode().strip() == args.target + for service in tenant["services"]: + dump_folder = get_dump_folder(service) + input_file = "%s/%s.tar.xz" % (dump_folder, service["url"]) + run( + "cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'" + % (input_file, service["name"], args.target, service["path"]) + ) + + +def dump_tenant_databases(tenant): + for service in tenant["services"]: + dump_folder = get_dump_folder(service) + if service["name"] == "wcs": + dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) + run( + "ssh database.%s 'sudo -u postgres pg_dump -Fc %s' > %s" + % (args.host, service["database"], dump_file) + ) + else: + dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) + run( + "ssh database.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s" + % (args.host, service["schema"], service["database"], dump_file) + ) + + +def restore_tenant_databases(tenant): + assert run("ssh %s hostname -f" % args.dbtarget).stdout.decode().strip() == args.dbtarget + for service in tenant["services"]: + dump_folder = get_dump_folder(service) + if service["name"] == "wcs": + dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) + run( + "ssh %s sudo -u postgres dropdb --if-exists %s" + % (args.dbtarget, service["database"]) + ) + run( + """ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'""" + % (args.dbtarget, service["database"]) + ) + run( + "cat %s | ssh %s sudo -u postgres pg_restore -d %s" + % (dump_file, args.dbtarget, service["database"]) + ) + else: + dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) + run( + """ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'""" + % (args.dbtarget, service["schema"], service["database"]) + ) + run( + "cat %s | ssh %s sudo -u postgres pg_restore -d %s" + % (dump_file, args.dbtarget, service["database"]) + ) + + +def parse_service(service): + if service["name"] == "authentic": + path = "/var/lib/authentic2-multitenant/tenants" + database = "authentic2_multitenant" + elif service["name"] == "wcs": + path = "/var/lib/wcs" + try: + wcs_config = run( + "ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck'" + % (args.host, service["url"]) + ) + config = pickle.loads(wcs_config.stdout) + database = config["postgresql"]["database"] + except (KeyError, subprocess.CalledProcessError): + database = None + else: + database = service["name"] + path = "/var/lib/%s/tenants" % service["name"] + return dict(service, database=database, path=path) + + +def get_host_info(): + if os.path.isfile("%s/data" % host_folder) and not args.update: + return + if not os.path.isdir(host_folder): + os.makedirs(host_folder) + run("scp bin/list_tenants.py hobo.%s:" % args.host) + output = run( + "ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command " + "runscript ~/list_tenants.py --all-tenants'" % args.host + ) + tenants = [] + for line in output.stdout.decode().split("\n"): + if line: + tenant = json.loads(line) + for i, service in enumerate(tenant["services"]): + tenant["services"][i] = parse_service(service) + tenants.append(tenant) + with open("%s/data" % host_folder, "w") as fh: + fh.write(json.dumps(tenants, indent=4)) + + +def get_tenant_info(tenant_name): + with open("%s/data" % host_folder) as fh: + hobos = json.loads(fh.read()) + tenant_infos = [x for x in hobos if x["name"] == tenant_name] + if not len(tenant_infos) == 1: + raise(Exception('tenant not found')) + return tenant_infos[0] + + +if __name__ == "__main__": + if args.action == "tenantinfo": + get_host_info() + print(json.dumps(get_tenant_info(args.tenant), indent=4)) + if args.action == "tenanturls": + print(' '.join([x['url'] for x in get_tenant_info(args.tenant)['services']])) + elif args.action == "dump": + get_host_info() + tenant = get_tenant_info(args.tenant) + dump_tenant_databases(tenant) + dump_tenant_files(tenant) + elif args.action == "restore": + tenant = get_tenant_info(args.tenant) + restore_tenant_files(tenant) + restore_tenant_databases(tenant)