257 lines
10 KiB
Python
Executable File
257 lines
10 KiB
Python
Executable File
#!/usr/bin/python3
|
|
import argparse
|
|
import json
|
|
import os
|
|
import pickle
|
|
import subprocess
|
|
|
|
|
|
class PublikDump():
|
|
|
|
def __init__(self, host, hobo_tenant, update=False, target=None, dbtarget=None):
|
|
self.host = host
|
|
self.hobo_tenant = hobo_tenant
|
|
self.update = update
|
|
self.target = target
|
|
self.dbtarget = dbtarget
|
|
self.host_folder = "output/%s" % host
|
|
|
|
@classmethod
|
|
def run(cls, cmd):
|
|
print("+ %s" % cmd)
|
|
return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE)
|
|
|
|
def get_host_info(self):
|
|
host_path = "%s/data" % self.host_folder
|
|
if os.path.isfile(host_path) and not self.update:
|
|
return json.load(open(host_path))
|
|
|
|
if not os.path.isdir(self.host_folder):
|
|
os.makedirs(self.host_folder)
|
|
self.run("scp publik_dump/list_tenants.py hobo.%s:" % self.host)
|
|
output = self.run(
|
|
"ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command "
|
|
"runscript ~/list_tenants.py --all-tenants'" % self.host
|
|
)
|
|
tenants = []
|
|
for line in output.stdout.decode().split("\n"):
|
|
if line:
|
|
tenant_infos = json.loads(line)
|
|
tenants.append(tenant_infos)
|
|
|
|
if tenants == []:
|
|
if os.path.isfile(host_path):
|
|
os.remove(host_path)
|
|
raise(Exception('DISABLE_CRON_JOBS is set on this node'))
|
|
json.dump(tenants, open(host_path, "w"), indent=4)
|
|
return tenants
|
|
|
|
def parse_service(self, service):
|
|
if service["name"] == "authentic":
|
|
path = "/var/lib/authentic2-multitenant/tenants"
|
|
database = "authentic2_multitenant"
|
|
elif service["name"] == "wcs":
|
|
try:
|
|
wcs_config = self.run(
|
|
"ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck'"
|
|
% (self.host, service["url"])
|
|
)
|
|
path = "/var/lib/wcs"
|
|
except (KeyError, subprocess.CalledProcessError):
|
|
wcs_config = self.run(
|
|
"ssh wcs.%s 'cat /var/lib/wcs/tenants/%s/config.pck'"
|
|
% (self.host, service["url"])
|
|
)
|
|
path = "/var/lib/wcs/tenants"
|
|
config = pickle.loads(wcs_config.stdout)
|
|
database = config["postgresql"]["database"]
|
|
else:
|
|
path = "/var/lib/%s/tenants" % service["name"]
|
|
database = service["name"]
|
|
return dict(service, database=database, path=path)
|
|
|
|
def get_tenant_info(self):
|
|
tenant_path = '%s/%s.json' % (self.host_folder, self.hobo_tenant)
|
|
if os.path.isfile(tenant_path) and not self.update:
|
|
return json.load(open(tenant_path))
|
|
|
|
hobos = self.get_host_info()
|
|
tenants = [x for x in hobos if x["name"] == self.hobo_tenant]
|
|
if not len(tenants) == 1:
|
|
raise(Exception('tenant not found'))
|
|
tenant = tenants[0]
|
|
for service in tenant['services']:
|
|
service.update(self.parse_service(service))
|
|
|
|
json.dump(tenant, open(tenant_path, "w"), indent=4)
|
|
return tenant
|
|
|
|
def get_dump_folder(self, service):
|
|
dump_folder = "%s/%s" % (self.host_folder, service["url"])
|
|
if not os.path.isdir(dump_folder):
|
|
os.mkdir(dump_folder)
|
|
return dump_folder
|
|
|
|
|
|
def dump_tenant_files(self):
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant["services"]:
|
|
#if service["name"] == "wcs":
|
|
# continue # in order to run rsync instead, a second time on large tenant dump
|
|
dump_folder = self.get_dump_folder(service)
|
|
output = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
|
self.run(
|
|
"ssh %s.%s 'sudo tar -C %s -Jcf - %s.invalid' > %s"
|
|
% (service["name"], self.host, service["path"], service["url"], output)
|
|
)
|
|
|
|
def restore_tenant_files(self):
|
|
tenant = self.get_tenant_info()
|
|
assert self.run("ssh %s hostname -f" % self.target).stdout.decode().strip() == self.target
|
|
for service in tenant["services"]:
|
|
#if service["name"] == "wcs":
|
|
# continue # in order to run rsync instead, a second time on large tenant dump
|
|
dump_folder = self.get_dump_folder(service)
|
|
input_file = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
|
self.run(
|
|
"cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'"
|
|
% (input_file, service["name"], self.target, service["path"])
|
|
)
|
|
|
|
def dump_tenant_databases(self):
|
|
# node2.test.saas.entrouvert.org -> db1.test.saas.entrouvert.org
|
|
domain = '.'.join(self.host.split('.')[1:])
|
|
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant["services"]:
|
|
dump_folder = self.get_dump_folder(service)
|
|
if service["name"] == "wcs":
|
|
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
|
self.run(
|
|
"ssh db1.%s 'sudo -u postgres pg_dump -Fc %s' > %s"
|
|
% (domain, service["database"], dump_file)
|
|
)
|
|
else:
|
|
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
|
self.run(
|
|
"ssh db1.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s"
|
|
% (domain, service["schema"], service["database"], dump_file)
|
|
)
|
|
|
|
def restore_tenant_databases(self):
|
|
assert self.run("ssh %s hostname -f" % self.dbtarget).stdout.decode().strip() == self.dbtarget
|
|
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant["services"]:
|
|
dump_folder = self.get_dump_folder(service)
|
|
if service["name"] == "wcs":
|
|
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
|
self.run(
|
|
"ssh %s 'sudo -u postgres dropdb --if-exists %s'"
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"""
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres psql -c "alter database %s connection limit 0;"'"""
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'"
|
|
% (dump_file, self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres psql -c "alter database %s connection limit -1;"'"""
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
else:
|
|
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'"""
|
|
% (self.dbtarget, service["schema"], service["database"])
|
|
)
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres psql -c "alter database %s connection limit 0;"'"""
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'"
|
|
% (dump_file, self.dbtarget, service["database"])
|
|
)
|
|
self.run(
|
|
"""ssh %s 'sudo -u postgres psql -c "alter database %s connection limit -1;"'"""
|
|
% (self.dbtarget, service["database"])
|
|
)
|
|
|
|
def invalidate_source_tenant(self):
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant['services']:
|
|
bpath = "%s/%s" % (service['path'], service['url'])
|
|
self.run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], self.host, bpath, bpath))
|
|
|
|
def validate_source_tenant(self):
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant['services']:
|
|
bpath = "%s/%s" % (service['path'], service['url'])
|
|
self.run('ssh %s.%s sudo mv %s.invalid %s' % (service['name'], self.host, bpath, bpath))
|
|
|
|
def invalidate_target_tenant(self):
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant['services']:
|
|
bpath = "%s/%s" % (service['path'], service['url'])
|
|
self.run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], self.target, bpath, bpath))
|
|
|
|
def validate_target_tenant(self):
|
|
tenant = self.get_tenant_info()
|
|
for service in tenant['services']:
|
|
bpath = "%s/%s" % (service['path'], service['url'])
|
|
self.run('ssh %s.%s sudo mv %s.invalid %s' % (service['name'], self.target, bpath, bpath))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("action", default="dump", choices=[
|
|
"tenantinfo", "tenanturls",
|
|
"invalidate_source", "validate_source",
|
|
"dump", "restore",
|
|
"invalidate_target", "validate_target",
|
|
])
|
|
parser.add_argument("host", help="origin host")
|
|
parser.add_argument("hobo_tenant", help="hobo tenant url")
|
|
parser.add_argument("--update", action="store_true")
|
|
parser.add_argument("--target", help="destination host")
|
|
parser.add_argument("--dbtarget", help="destination host")
|
|
args = parser.parse_args()
|
|
publik_dump = PublikDump(
|
|
host=args.host,
|
|
hobo_tenant=args.hobo_tenant,
|
|
update=args.update,
|
|
target=args.target,
|
|
dbtarget=args.dbtarget,
|
|
)
|
|
|
|
if args.action == "tenantinfo":
|
|
print(json.dumps(publik_dump.get_tenant_info(), indent=4))
|
|
elif args.action == "tenanturls":
|
|
print(' '.join([x['url'] for x in publik_dump.get_tenant_info()['services']]))
|
|
elif args.action == "invalidate_source":
|
|
publik_dump.invalidate_source_tenant()
|
|
elif args.action == "validate_source":
|
|
publik_dump.validate_source_tenant()
|
|
elif args.action == "dump":
|
|
publik_dump.dump_tenant_databases()
|
|
publik_dump.dump_tenant_files()
|
|
elif args.action == "restore":
|
|
assert(publik_dump.target)
|
|
assert(publik_dump.dbtarget)
|
|
publik_dump.restore_tenant_files()
|
|
publik_dump.restore_tenant_databases()
|
|
elif args.action == "invalidate_target":
|
|
assert(publik_dump.target)
|
|
publik_dump.invalidate_target_tenant()
|
|
elif args.action == "validate_target":
|
|
assert(publik_dump.target)
|
|
publik_dump.validate_target_tenant()
|