add project publik-dump
This commit is contained in:
Christophe Siraut
parent
a49f5b7d77
commit
faba1dbabc
|
|
@ -0,0 +1 @@
|
|||
output
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/python2
|
||||
import argparse
|
||||
import pickle
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("file")
|
||||
parser.add_argument("--host")
|
||||
parser.add_argument("--password")
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.file) as fh:
|
||||
data = pickle.loads(fh.read())
|
||||
|
||||
data["postgresql"]["host"] = args.host
|
||||
data["postgresql"]["password"] = args.password
|
||||
print(data["postgresql"])
|
||||
|
||||
with open(args.file, "w") as fh:
|
||||
fh.write(pickle.dumps(data))
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/python3
|
||||
import json
|
||||
from urllib.parse import urlparse, urlsplit
|
||||
from hobo.environment.models import AVAILABLE_SERVICES
|
||||
from hobo.multitenant.middleware import TenantMiddleware
|
||||
from django.db import connection
|
||||
|
||||
tenant = connection.get_tenant()
|
||||
services = [{"name": "hobo", "url": tenant.domain_url, "schema": tenant.schema_name}]
|
||||
|
||||
for service in AVAILABLE_SERVICES:
|
||||
for site in service.objects.all():
|
||||
service = {
|
||||
"name": site.Extra.service_id,
|
||||
"url": urlsplit(site.base_url).netloc.split(":")[0],
|
||||
}
|
||||
if site.secondary:
|
||||
service["secondary"] = True
|
||||
schema = TenantMiddleware.hostname2schema(service["url"])
|
||||
if service["name"] == "wcs":
|
||||
schema = "wcs_%s" % schema
|
||||
service["schema"] = schema
|
||||
services.append(service)
|
||||
|
||||
print(json.dumps({"name": tenant.domain_url, "services": services}))
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
#!/usr/bin/python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore"])
|
||||
parser.add_argument("host", help="origin host")
|
||||
parser.add_argument("tenant", help="hobo tenant url")
|
||||
parser.add_argument("--update", action="store_true")
|
||||
parser.add_argument("--target", help="destination host")
|
||||
parser.add_argument("--dbtarget", help="destination host")
|
||||
args = parser.parse_args()
|
||||
host_folder = "output/%s" % args.host
|
||||
|
||||
|
||||
def run(cmd):
|
||||
print("+ %s" % cmd)
|
||||
return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE)
|
||||
|
||||
|
||||
def get_dump_folder(service):
|
||||
dump_folder = "%s/%s" % (host_folder, service["url"])
|
||||
if not os.path.isdir(dump_folder):
|
||||
os.mkdir(dump_folder)
|
||||
return dump_folder
|
||||
|
||||
|
||||
def dump_tenant_files(tenant):
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
output = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
run(
|
||||
"ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s"
|
||||
% (service["name"], args.host, service["path"], service["url"], output)
|
||||
)
|
||||
|
||||
|
||||
def restore_tenant_files(tenant):
|
||||
assert run("ssh %s hostname -f" % args.target).stdout.decode().strip() == args.target
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
input_file = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
run(
|
||||
"cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'"
|
||||
% (input_file, service["name"], args.target, service["path"])
|
||||
)
|
||||
|
||||
|
||||
def dump_tenant_databases(tenant):
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
if service["name"] == "wcs":
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
run(
|
||||
"ssh database.%s 'sudo -u postgres pg_dump -Fc %s' > %s"
|
||||
% (args.host, service["database"], dump_file)
|
||||
)
|
||||
else:
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
run(
|
||||
"ssh database.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s"
|
||||
% (args.host, service["schema"], service["database"], dump_file)
|
||||
)
|
||||
|
||||
|
||||
def restore_tenant_databases(tenant):
|
||||
assert run("ssh %s hostname -f" % args.dbtarget).stdout.decode().strip() == args.dbtarget
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
if service["name"] == "wcs":
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
run(
|
||||
"ssh %s sudo -u postgres dropdb --if-exists %s"
|
||||
% (args.dbtarget, service["database"])
|
||||
)
|
||||
run(
|
||||
"""ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"""
|
||||
% (args.dbtarget, service["database"])
|
||||
)
|
||||
run(
|
||||
"cat %s | ssh %s sudo -u postgres pg_restore -d %s"
|
||||
% (dump_file, args.dbtarget, service["database"])
|
||||
)
|
||||
else:
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
run(
|
||||
"""ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'"""
|
||||
% (args.dbtarget, service["schema"], service["database"])
|
||||
)
|
||||
run(
|
||||
"cat %s | ssh %s sudo -u postgres pg_restore -d %s"
|
||||
% (dump_file, args.dbtarget, service["database"])
|
||||
)
|
||||
|
||||
|
||||
def parse_service(service):
|
||||
if service["name"] == "authentic":
|
||||
path = "/var/lib/authentic2-multitenant/tenants"
|
||||
database = "authentic2_multitenant"
|
||||
elif service["name"] == "wcs":
|
||||
path = "/var/lib/wcs"
|
||||
try:
|
||||
wcs_config = run(
|
||||
"ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck'"
|
||||
% (args.host, service["url"])
|
||||
)
|
||||
config = pickle.loads(wcs_config.stdout)
|
||||
database = config["postgresql"]["database"]
|
||||
except (KeyError, subprocess.CalledProcessError):
|
||||
database = None
|
||||
else:
|
||||
database = service["name"]
|
||||
path = "/var/lib/%s/tenants" % service["name"]
|
||||
return dict(service, database=database, path=path)
|
||||
|
||||
|
||||
def get_host_info():
|
||||
if os.path.isfile("%s/data" % host_folder) and not args.update:
|
||||
return
|
||||
if not os.path.isdir(host_folder):
|
||||
os.makedirs(host_folder)
|
||||
run("scp bin/list_tenants.py hobo.%s:" % args.host)
|
||||
output = run(
|
||||
"ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command "
|
||||
"runscript ~/list_tenants.py --all-tenants'" % args.host
|
||||
)
|
||||
tenants = []
|
||||
for line in output.stdout.decode().split("\n"):
|
||||
if line:
|
||||
tenant = json.loads(line)
|
||||
for i, service in enumerate(tenant["services"]):
|
||||
tenant["services"][i] = parse_service(service)
|
||||
tenants.append(tenant)
|
||||
with open("%s/data" % host_folder, "w") as fh:
|
||||
fh.write(json.dumps(tenants, indent=4))
|
||||
|
||||
|
||||
def get_tenant_info(tenant_name):
|
||||
with open("%s/data" % host_folder) as fh:
|
||||
hobos = json.loads(fh.read())
|
||||
tenant_infos = [x for x in hobos if x["name"] == tenant_name]
|
||||
if not len(tenant_infos) == 1:
|
||||
raise(Exception('tenant not found'))
|
||||
return tenant_infos[0]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if args.action == "tenantinfo":
|
||||
get_host_info()
|
||||
print(json.dumps(get_tenant_info(args.tenant), indent=4))
|
||||
if args.action == "tenanturls":
|
||||
print(' '.join([x['url'] for x in get_tenant_info(args.tenant)['services']]))
|
||||
elif args.action == "dump":
|
||||
get_host_info()
|
||||
tenant = get_tenant_info(args.tenant)
|
||||
dump_tenant_databases(tenant)
|
||||
dump_tenant_files(tenant)
|
||||
elif args.action == "restore":
|
||||
tenant = get_tenant_info(args.tenant)
|
||||
restore_tenant_files(tenant)
|
||||
restore_tenant_databases(tenant)
|
||||
Reference in New Issue