publik-dump: add tests
This commit is contained in:
parent
de91f1cee8
commit
1e221eb962
|
|
@ -1 +1,4 @@
|
|||
output
|
||||
__pycache__
|
||||
*.pyc
|
||||
*~
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore", "invalidate"])
|
||||
parser.add_argument("host", help="origin host")
|
||||
parser.add_argument("tenant", help="hobo tenant url")
|
||||
parser.add_argument("--update", action="store_true")
|
||||
parser.add_argument("--target", help="destination host")
|
||||
parser.add_argument("--dbtarget", help="destination host")
|
||||
args = parser.parse_args()
|
||||
host_folder = "output/%s" % args.host
|
||||
|
||||
|
||||
def run(cmd):
|
||||
print("+ %s" % cmd)
|
||||
return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE)
|
||||
|
||||
|
||||
def get_dump_folder(service):
|
||||
dump_folder = "%s/%s" % (host_folder, service["url"])
|
||||
if not os.path.isdir(dump_folder):
|
||||
os.mkdir(dump_folder)
|
||||
return dump_folder
|
||||
|
||||
|
||||
def dump_tenant_files(tenant):
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
output = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
run(
|
||||
"ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s"
|
||||
% (service["name"], args.host, service["path"], service["url"], output)
|
||||
)
|
||||
|
||||
|
||||
def restore_tenant_files(tenant):
|
||||
assert run("ssh %s hostname -f" % args.target).stdout.decode().strip() == args.target
|
||||
import pdb; pdb.set_trace()
|
||||
# for service in tenant["services"]:
|
||||
# dump_folder = get_dump_folder(service)
|
||||
# input_file = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
# run(
|
||||
# "cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'"
|
||||
# % (input_file, service["name"], args.target, service["path"])
|
||||
# )
|
||||
|
||||
|
||||
def dump_tenant_databases(tenant):
|
||||
# node2.test.saas.entrouvert.org -> db2.test.saas.entrouvert.org
|
||||
domain = '.'.join(args.host.split('.')[1:])
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
if service["name"] == "wcs":
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
run(
|
||||
"ssh db2.%s 'sudo -u postgres pg_dump -Fc %s' > %s"
|
||||
% (domain, service["database"], dump_file)
|
||||
)
|
||||
else:
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
run(
|
||||
"ssh db2.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s"
|
||||
% (domain, service["schema"], service["database"], dump_file)
|
||||
)
|
||||
|
||||
|
||||
def restore_tenant_databases(tenant):
|
||||
assert run("ssh %s hostname -f" % args.dbtarget).stdout.decode().strip() == args.dbtarget
|
||||
for service in tenant["services"]:
|
||||
dump_folder = get_dump_folder(service)
|
||||
import pdb; pdb.set_trace()
|
||||
# if service["name"] == "wcs":
|
||||
# dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
# run(
|
||||
# "ssh %s sudo -u postgres dropdb --if-exists %s"
|
||||
# % (args.dbtarget, service["database"])
|
||||
# )
|
||||
# run(
|
||||
# """ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"""
|
||||
# % (args.dbtarget, service["database"])
|
||||
# )
|
||||
# run(
|
||||
# "cat %s | ssh %s sudo -u postgres pg_restore -d %s"
|
||||
# % (dump_file, args.dbtarget, service["database"])
|
||||
# )
|
||||
# else:
|
||||
# dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
# run(
|
||||
# """ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'"""
|
||||
# % (args.dbtarget, service["schema"], service["database"])
|
||||
# )
|
||||
# run(
|
||||
# "cat %s | ssh %s sudo -u postgres pg_restore -d %s"
|
||||
# % (dump_file, args.dbtarget, service["database"])
|
||||
# )
|
||||
|
||||
|
||||
def parse_service(service):
|
||||
if service["name"] == "authentic":
|
||||
path = "/var/lib/authentic2-multitenant/tenants"
|
||||
database = "authentic2_multitenant"
|
||||
elif service["name"] == "wcs":
|
||||
path = "/var/lib/wcs"
|
||||
try:
|
||||
wcs_config = run(
|
||||
"ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck 2>/dev/null || cat /var/lib/wcs/tenants/%s/config.pck'"
|
||||
% (args.host, service["url"], service["url"])
|
||||
)
|
||||
config = pickle.loads(wcs_config.stdout)
|
||||
database = config["postgresql"]["database"]
|
||||
except (KeyError, subprocess.CalledProcessError):
|
||||
database = None
|
||||
else:
|
||||
database = service["name"]
|
||||
# TODO maybe /tenants/ pour wcs
|
||||
path = "/var/lib/%s/tenants" % service["name"]
|
||||
return dict(service, database=database, path=path)
|
||||
|
||||
|
||||
def get_host_info():
|
||||
if os.path.isfile("%s/data" % host_folder) and not args.update:
|
||||
return
|
||||
if not os.path.isdir(host_folder):
|
||||
os.makedirs(host_folder)
|
||||
run("scp bin/list_tenants.py hobo.%s:" % args.host)
|
||||
output = run(
|
||||
"ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command "
|
||||
"runscript ~/list_tenants.py --all-tenants'" % args.host
|
||||
)
|
||||
tenants = []
|
||||
for line in output.stdout.decode().split("\n"):
|
||||
if line:
|
||||
tenant = json.loads(line)
|
||||
for i, service in enumerate(tenant["services"]):
|
||||
tenant["services"][i] = parse_service(service)
|
||||
tenants.append(tenant)
|
||||
with open("%s/data" % host_folder, "w") as fh:
|
||||
fh.write(json.dumps(tenants, indent=4))
|
||||
|
||||
|
||||
def get_tenant_info(tenant_name):
|
||||
with open("%s/data" % host_folder) as fh:
|
||||
hobos = json.loads(fh.read())
|
||||
tenant_infos = [x for x in hobos if x["name"] == tenant_name]
|
||||
if not len(tenant_infos) == 1:
|
||||
raise(Exception('tenant not found'))
|
||||
return tenant_infos[0]
|
||||
|
||||
|
||||
def invalidate(tenant_name):
|
||||
tenant_info = get_tenant_info(tenant_name)
|
||||
import pdb; pdb.set_trace()
|
||||
# for service in tenant_info['services']:
|
||||
# bpath = "%s/%s" % (service['path'], service['url'])
|
||||
# run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], args.host, bpath, bpath))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if args.action == "tenantinfo":
|
||||
get_host_info()
|
||||
print(json.dumps(get_tenant_info(args.tenant), indent=4))
|
||||
elif args.action == "tenanturls":
|
||||
print(' '.join([x['url'] for x in get_tenant_info(args.tenant)['services']]))
|
||||
elif args.action == "dump":
|
||||
get_host_info()
|
||||
tenant = get_tenant_info(args.tenant)
|
||||
dump_tenant_databases(tenant)
|
||||
dump_tenant_files(tenant)
|
||||
elif args.action == "restore":
|
||||
tenant = get_tenant_info(args.tenant)
|
||||
#restore_tenant_files(tenant)
|
||||
#restore_tenant_databases(tenant)
|
||||
elif args.action == "invalidate":
|
||||
invalidate(args.tenant)
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/python3
|
||||
import json
|
||||
from urllib.parse import urlparse, urlsplit
|
||||
from urllib.parse import urlsplit
|
||||
from hobo.environment.models import AVAILABLE_SERVICES
|
||||
from hobo.multitenant.middleware import TenantMiddleware
|
||||
from django.db import connection
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
#!/usr/bin/python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import subprocess
|
||||
|
||||
|
||||
class PublikDump():
|
||||
|
||||
def __init__(self, host, hobo_tenant, update=False, target=None, dbtarget=None):
|
||||
self.host = host
|
||||
self.hobo_tenant = hobo_tenant
|
||||
self.update = update
|
||||
self.target = target
|
||||
self.dbtarget = dbtarget
|
||||
self.host_folder = "output/%s" % host
|
||||
|
||||
@classmethod
|
||||
def run(cls, cmd):
|
||||
print("+ %s" % cmd)
|
||||
return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE)
|
||||
|
||||
def get_host_info(self):
|
||||
host_path = "%s/data" % self.host_folder
|
||||
if os.path.isfile(host_path) and not self.update:
|
||||
return json.load(open(host_path))
|
||||
|
||||
if not os.path.isdir(self.host_folder):
|
||||
os.makedirs(self.host_folder)
|
||||
self.run("scp publik_dump/list_tenants.py hobo.%s:" % self.host)
|
||||
output = self.run(
|
||||
"ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command "
|
||||
"runscript ~/list_tenants.py --all-tenants'" % self.host
|
||||
)
|
||||
tenants = []
|
||||
for line in output.stdout.decode().split("\n"):
|
||||
if line:
|
||||
tenant_infos = json.loads(line)
|
||||
tenants.append(tenant_infos)
|
||||
|
||||
json.dump(tenants, open(host_path, "w"), indent=4)
|
||||
return tenants
|
||||
|
||||
def parse_service(self, service):
|
||||
if service["name"] == "authentic":
|
||||
path = "/var/lib/authentic2-multitenant/tenants"
|
||||
database = "authentic2_multitenant"
|
||||
elif service["name"] == "wcs":
|
||||
try:
|
||||
wcs_config = self.run(
|
||||
"ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck'"
|
||||
% (self.host, service["url"])
|
||||
)
|
||||
path = "/var/lib/wcs"
|
||||
except (KeyError, subprocess.CalledProcessError):
|
||||
wcs_config = self.run(
|
||||
"ssh wcs.%s 'cat /var/lib/wcs/tenants/%s/config.pck'"
|
||||
% (self.host, service["url"])
|
||||
)
|
||||
path = "/var/lib/wcs/tenants"
|
||||
config = pickle.loads(wcs_config.stdout)
|
||||
database = config["postgresql"]["database"]
|
||||
else:
|
||||
path = "/var/lib/%s/tenants" % service["name"]
|
||||
database = service["name"]
|
||||
return dict(service, database=database, path=path)
|
||||
|
||||
def get_tenant_info(self):
|
||||
tenant_path = '%s/%s.json' % (self.host_folder, self.hobo_tenant)
|
||||
if os.path.isfile(tenant_path) and not self.update:
|
||||
return json.load(open(tenant_path))
|
||||
|
||||
hobos = self.get_host_info()
|
||||
tenants = [x for x in hobos if x["name"] == self.hobo_tenant]
|
||||
if not len(tenants) == 1:
|
||||
raise(Exception('tenant not found'))
|
||||
tenant = tenants[0]
|
||||
for service in tenant['services']:
|
||||
service.update(self.parse_service(service))
|
||||
|
||||
json.dump(tenant, open(tenant_path, "w"), indent=4)
|
||||
return tenant
|
||||
|
||||
def get_dump_folder(self, service):
|
||||
dump_folder = "%s/%s" % (self.host_folder, service["url"])
|
||||
if not os.path.isdir(dump_folder):
|
||||
os.mkdir(dump_folder)
|
||||
return dump_folder
|
||||
|
||||
|
||||
def dump_tenant_files(self):
|
||||
tenant = self.get_tenant_info()
|
||||
for service in tenant["services"]:
|
||||
dump_folder = self.get_dump_folder(service)
|
||||
output = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
self.run(
|
||||
"ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s"
|
||||
% (service["name"], self.host, service["path"], service["url"], output)
|
||||
)
|
||||
|
||||
def restore_tenant_files(self):
|
||||
tenant = self.get_tenant_info()
|
||||
assert self.run("ssh %s hostname -f" % self.target).stdout.decode().strip() == self.target
|
||||
for service in tenant["services"]:
|
||||
dump_folder = self.get_dump_folder(service)
|
||||
input_file = "%s/%s.tar.xz" % (dump_folder, service["url"])
|
||||
self.run(
|
||||
"cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'"
|
||||
% (input_file, service["name"], self.target, service["path"])
|
||||
)
|
||||
|
||||
def dump_tenant_databases(self):
|
||||
# node2.test.saas.entrouvert.org -> db2.test.saas.entrouvert.org
|
||||
domain = '.'.join(self.host.split('.')[1:])
|
||||
|
||||
tenant = self.get_tenant_info()
|
||||
for service in tenant["services"]:
|
||||
dump_folder = self.get_dump_folder(service)
|
||||
if service["name"] == "wcs":
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
self.run(
|
||||
"ssh db2.%s 'sudo -u postgres pg_dump -Fc %s' > %s"
|
||||
% (domain, service["database"], dump_file)
|
||||
)
|
||||
else:
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
self.run(
|
||||
"ssh db2.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s"
|
||||
% (domain, service["schema"], service["database"], dump_file)
|
||||
)
|
||||
|
||||
def restore_tenant_databases(self):
|
||||
assert self.run("ssh %s hostname -f" % self.dbtarget).stdout.decode().strip() == self.dbtarget
|
||||
|
||||
tenant = self.get_tenant_info()
|
||||
for service in tenant["services"]:
|
||||
dump_folder = self.get_dump_folder(service)
|
||||
if service["name"] == "wcs":
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"])
|
||||
self.run(
|
||||
"ssh %s 'sudo -u postgres dropdb --if-exists %s'"
|
||||
% (self.dbtarget, service["database"])
|
||||
)
|
||||
self.run(
|
||||
"""ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"""
|
||||
% (self.dbtarget, service["database"])
|
||||
)
|
||||
self.run(
|
||||
"cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'"
|
||||
% (dump_file, self.dbtarget, service["database"])
|
||||
)
|
||||
else:
|
||||
dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"])
|
||||
self.run(
|
||||
"""ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'"""
|
||||
% (self.dbtarget, service["schema"], service["database"])
|
||||
)
|
||||
self.run(
|
||||
"cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'"
|
||||
% (dump_file, self.dbtarget, service["database"])
|
||||
)
|
||||
|
||||
def invalidate_tenant(self):
|
||||
tenant = self.get_tenant_info()
|
||||
for service in tenant['services']:
|
||||
bpath = "%s/%s" % (service['path'], service['url'])
|
||||
self.run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], self.host, bpath, bpath))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore", "invalidate"])
|
||||
parser.add_argument("host", help="origin host")
|
||||
parser.add_argument("hobo_tenant", help="hobo tenant url")
|
||||
parser.add_argument("--update", action="store_true")
|
||||
parser.add_argument("--target", help="destination host")
|
||||
parser.add_argument("--dbtarget", help="destination host")
|
||||
args = parser.parse_args()
|
||||
publik_dump = PublikDump(
|
||||
host=args.host,
|
||||
hobo_tenant=args.hobo_tenant,
|
||||
update=args.update,
|
||||
target=args.target,
|
||||
dbtarget=args.dbtarget,
|
||||
)
|
||||
|
||||
if args.action == "tenantinfo":
|
||||
print(json.dumps(publik_dump.get_tenant_info(), indent=4))
|
||||
elif args.action == "tenanturls":
|
||||
print(' '.join([x['url'] for x in publik_dump.get_tenant_info()['services']]))
|
||||
elif args.action == "dump":
|
||||
publik_dump.dump_tenant_databases()
|
||||
publik_dump.dump_tenant_files()
|
||||
elif args.action == "restore":
|
||||
publik_dump.restore_tenant_files()
|
||||
publik_dump.restore_tenant_databases()
|
||||
elif args.action == "invalidate":
|
||||
publik_dump.invalidate_tenant()
|
||||
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
|||
{"name": "hobo-eurelien.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-eurelien.test.entrouvert.org", "schema": "hobo_eurelien_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-eurelien.test.entrouvert.org", "schema": "connexion_eurelien_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-eurelien.test.entrouvert.org", "schema": "wcs_demarches_eurelien_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-eurelien.test.entrouvert.org", "schema": "passerelle_eurelien_test_entrouvert_org"}, {"name": "combo", "url": "agents-eurelien.test.entrouvert.org", "schema": "agents_eurelien_test_entrouvert_org"}, {"name": "combo", "url": "portail-eurelien.test.entrouvert.org", "schema": "portail_eurelien_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-eurelien.test.entrouvert.org", "schema": "portedoc_eurelien_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-eurelien.test.entrouvert.org", "schema": "agendas_eurelien_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-eurelien.test.entrouvert.org", "schema": "statistiques_eurelien_test_entrouvert_org"}]}
|
||||
{"name": "hobo-tours-ville.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-tours-ville.test.entrouvert.org", "schema": "hobo_tours_ville_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-tours.test.entrouvert.org", "secondary": true, "schema": "connexion_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours.test.entrouvert.org", "secondary": true, "schema": "wcs_demarches_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours-ville.test.entrouvert.org", "schema": "wcs_demarches_tours_ville_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours.test.entrouvert.org", "secondary": true, "schema": "passerelle_tours_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours-ville.test.entrouvert.org", "schema": "passerelle_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours-ville.test.entrouvert.org", "schema": "agents_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours.test.entrouvert.org", "secondary": true, "schema": "agents_tours_test_entrouvert_org"}, {"name": "combo", "url": "tours.test.entrouvert.org", "secondary": true, "schema": "tours_test_entrouvert_org"}, {"name": "combo", "url": "tours-ville.test.entrouvert.org", "schema": "tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours-ville.test.entrouvert.org", "schema": "portedoc_tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours.test.entrouvert.org", "secondary": true, "schema": "portedoc_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours.test.entrouvert.org", "secondary": true, "schema": "agendas_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours-ville.test.entrouvert.org", "schema": "agendas_tours_ville_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-tours.test.entrouvert.org", "secondary": true, "schema": "statistiques_tours_test_entrouvert_org"}, {"name": "bijoe", "url": "stats-tours-ville.test.entrouvert.org", "schema": "stats_tours_ville_test_entrouvert_org"}, {"name": "hobo", "url": "hobo-tours.test.entrouvert.org", "secondary": true, "schema": "hobo_tours_test_entrouvert_org"}]}
|
||||
{"name": "hobo-tours.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-tours.test.entrouvert.org", "schema": "hobo_tours_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-tours.test.entrouvert.org", "schema": "connexion_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours.test.entrouvert.org", "schema": "wcs_demarches_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours-ville.test.entrouvert.org", "secondary": true, "schema": "wcs_demarches_tours_ville_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours.test.entrouvert.org", "schema": "passerelle_tours_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours-ville.test.entrouvert.org", "secondary": true, "schema": "passerelle_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours.test.entrouvert.org", "schema": "agents_tours_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours-ville.test.entrouvert.org", "secondary": true, "schema": "agents_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "tours.test.entrouvert.org", "schema": "tours_test_entrouvert_org"}, {"name": "combo", "url": "tours-ville.test.entrouvert.org", "secondary": true, "schema": "tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours-ville.test.entrouvert.org", "secondary": true, "schema": "portedoc_tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours.test.entrouvert.org", "schema": "portedoc_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours.test.entrouvert.org", "schema": "agendas_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours-ville.test.entrouvert.org", "secondary": true, "schema": "agendas_tours_ville_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-tours.test.entrouvert.org", "schema": "statistiques_tours_test_entrouvert_org"}, {"name": "bijoe", "url": "stats-tours-ville.test.entrouvert.org", "secondary": true, "schema": "stats_tours_ville_test_entrouvert_org"}, {"name": "hobo", "url": "hobo-tours-ville.test.entrouvert.org", "schema": "hobo_tours_ville_test_entrouvert_org"}]}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import pytest
|
||||
import os
|
||||
|
||||
from publik_dump.publik_dump import PublikDump
|
||||
|
||||
@pytest.fixture
|
||||
def output(tmpdir):
|
||||
return str(tmpdir)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service():
|
||||
return ""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def publik_dump():
|
||||
return PublikDump
|
||||
|
||||
def test_get_dump_folder(output, publik_dump, service):
|
||||
#filename = os.path.join(str(tmpdir), 'file.tar')
|
||||
assert publik_dump.coucou() == "nico"
|
||||
|
||||
|
||||
def test_dump_tenant_files():
|
||||
print("coucou")
|
||||
|
||||
|
||||
# patch run
|
||||
def get_host_info():
|
||||
pass
|
||||
|
|
@ -0,0 +1,343 @@
|
|||
import json
|
||||
import mock
|
||||
import os
|
||||
import pytest
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
from publik_dump.publik_dump import PublikDump
|
||||
|
||||
|
||||
HOST_INFO_0 = {
|
||||
"name": "hobo-eurelien.test.entrouvert.org",
|
||||
"services": [
|
||||
{
|
||||
"name": "hobo",
|
||||
"url": "hobo-eurelien.test.entrouvert.org",
|
||||
"schema": "hobo_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "authentic",
|
||||
"url": "connexion-eurelien.test.entrouvert.org",
|
||||
"schema": "connexion_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "wcs",
|
||||
"url": "demarches-eurelien.test.entrouvert.org",
|
||||
"schema": "wcs_demarches_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "passerelle",
|
||||
"url": "passerelle-eurelien.test.entrouvert.org",
|
||||
"schema": "passerelle_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "combo",
|
||||
"url": "agents-eurelien.test.entrouvert.org",
|
||||
"schema": "agents_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "combo",
|
||||
"url": "portail-eurelien.test.entrouvert.org",
|
||||
"schema": "portail_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "fargo",
|
||||
"url": "portedoc-eurelien.test.entrouvert.org",
|
||||
"schema": "portedoc_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "chrono",
|
||||
"url": "agendas-eurelien.test.entrouvert.org",
|
||||
"schema": "agendas_eurelien_test_entrouvert_org"
|
||||
},
|
||||
{
|
||||
"name": "bijoe",
|
||||
"url": "statistiques-eurelien.test.entrouvert.org",
|
||||
"schema": "statistiques_eurelien_test_entrouvert_org"
|
||||
}
|
||||
]
|
||||
}
|
||||
HOST_INFO_0_HOBO = HOST_INFO_0['services'][0]
|
||||
HOST_INFO_0_AUTHENTIC = HOST_INFO_0['services'][1]
|
||||
HOST_INFO_0_WCS = HOST_INFO_0['services'][2]
|
||||
|
||||
TENANT_INFO = {
|
||||
"name": "hobo-eurelien.test.entrouvert.org",
|
||||
"services": [
|
||||
{
|
||||
"name": "hobo",
|
||||
"url": "hobo-eurelien.test.entrouvert.org",
|
||||
"schema": "hobo_eurelien_test_entrouvert_org",
|
||||
"database": "hobo",
|
||||
"path": "/var/lib/hobo/tenants"
|
||||
},
|
||||
{
|
||||
"name": "authentic",
|
||||
"url": "connexion-eurelien.test.entrouvert.org",
|
||||
"schema": "connexion_eurelien_test_entrouvert_org",
|
||||
"database": "authentic2_multitenant",
|
||||
"path": "/var/lib/authentic2-multitenant/tenants"
|
||||
},
|
||||
{
|
||||
"name": "wcs",
|
||||
"url": "demarches-eurelien.test.entrouvert.org",
|
||||
"schema": "wcs_demarches_eurelien_test_entrouvert_org",
|
||||
"database": "wcs_demarches_eurelien_test_entrouvert_org",
|
||||
"path": "/var/lib/wcs"
|
||||
},
|
||||
{
|
||||
"name": "passerelle",
|
||||
"url": "passerelle-eurelien.test.entrouvert.org",
|
||||
"schema": "passerelle_eurelien_test_entrouvert_org",
|
||||
"database": "passerelle",
|
||||
"path": "/var/lib/passerelle/tenants"
|
||||
},
|
||||
{
|
||||
"name": "combo",
|
||||
"url": "agents-eurelien.test.entrouvert.org",
|
||||
"schema": "agents_eurelien_test_entrouvert_org",
|
||||
"database": "combo",
|
||||
"path": "/var/lib/combo/tenants"
|
||||
},
|
||||
{
|
||||
"name": "combo",
|
||||
"url": "portail-eurelien.test.entrouvert.org",
|
||||
"schema": "portail_eurelien_test_entrouvert_org",
|
||||
"database": "combo",
|
||||
"path": "/var/lib/combo/tenants"
|
||||
},
|
||||
{
|
||||
"name": "fargo",
|
||||
"url": "portedoc-eurelien.test.entrouvert.org",
|
||||
"schema": "portedoc_eurelien_test_entrouvert_org",
|
||||
"database": "fargo",
|
||||
"path": "/var/lib/fargo/tenants"
|
||||
},
|
||||
{
|
||||
"name": "chrono",
|
||||
"url": "agendas-eurelien.test.entrouvert.org",
|
||||
"schema": "agendas_eurelien_test_entrouvert_org",
|
||||
"database": "chrono",
|
||||
"path": "/var/lib/chrono/tenants"
|
||||
},
|
||||
{
|
||||
"name": "bijoe",
|
||||
"url": "statistiques-eurelien.test.entrouvert.org",
|
||||
"schema": "statistiques_eurelien_test_entrouvert_org",
|
||||
"database": "bijoe",
|
||||
"path": "/var/lib/bijoe/tenants"
|
||||
}
|
||||
]
|
||||
}
|
||||
TENANT_HOBO = TENANT_INFO['services'][0]
|
||||
TENANT_AUTHENTIC = TENANT_INFO['services'][1]
|
||||
TENANT_WCS = TENANT_INFO['services'][2]
|
||||
|
||||
TARGET = 'node1.test-hds.saas.entrouvert'
|
||||
DB_TARGET = 'sql3.test-hds.saas.entrouvert'
|
||||
|
||||
@pytest.fixture
|
||||
def publik_dump(tmpdir):
|
||||
obj = PublikDump('node2.test.saas.entrouvert.org', 'hobo-eurelien.test.entrouvert.org')
|
||||
obj.host_folder = str(tmpdir)
|
||||
#obj.run = lambda x: x
|
||||
return obj
|
||||
|
||||
|
||||
class MockedCompletedProcess():
|
||||
'''simulate subprocess.run return'''
|
||||
def __init__(self, stdout):
|
||||
self.stdout = stdout
|
||||
|
||||
|
||||
def test_run(publik_dump):
|
||||
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
|
||||
mocked_run.return_value = MockedCompletedProcess('foo result')
|
||||
output = publik_dump.run('foo query')
|
||||
assert mocked_run.mock_calls[0][1][0] == 'foo query'
|
||||
assert output.stdout == 'foo result'
|
||||
|
||||
|
||||
def get_list_tenants():
|
||||
with open(os.path.join(os.path.dirname(__file__), 'list_tenants.json'), 'rb') as desc:
|
||||
return desc.read()
|
||||
|
||||
def get_config_pck():
|
||||
with open(os.path.join(os.path.dirname(__file__), 'config.pck'), 'rb') as desc:
|
||||
return desc.read()
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(get_list_tenants()))
|
||||
def test_get_host_info(mocked_run, publik_dump):
|
||||
tenants = publik_dump.get_host_info()
|
||||
assert len(mocked_run.mock_calls) == 2
|
||||
assert mocked_run.mock_calls[0][1][0] == \
|
||||
'scp publik_dump/list_tenants.py hobo.node2.test.saas.entrouvert.org:'
|
||||
assert mocked_run.mock_calls[1][1][0] == \
|
||||
"ssh hobo.node2.test.saas.entrouvert.org"\
|
||||
" 'sudo -u hobo HOME=$HOME hobo-manage tenant_command runscript ~/list_tenants.py --all-tenants'"
|
||||
assert tenants[0] == HOST_INFO_0
|
||||
|
||||
# result from cache
|
||||
tenants = publik_dump.get_host_info()
|
||||
assert len(mocked_run.mock_calls) == 2
|
||||
assert tenants[0] == HOST_INFO_0
|
||||
|
||||
# renew cache
|
||||
publik_dump.update = True
|
||||
tenants = publik_dump.get_host_info()
|
||||
assert len(mocked_run.mock_calls) == 4
|
||||
assert tenants[0] == HOST_INFO_0
|
||||
|
||||
|
||||
def test_parse_service(publik_dump):
|
||||
# hobo
|
||||
assert publik_dump.parse_service(HOST_INFO_0_HOBO) == TENANT_HOBO
|
||||
|
||||
# authentic
|
||||
assert publik_dump.parse_service(HOST_INFO_0_AUTHENTIC) == TENANT_AUTHENTIC
|
||||
|
||||
# wcs old tenant directory
|
||||
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
|
||||
mocked_run.return_value = MockedCompletedProcess(get_config_pck())
|
||||
assert publik_dump.parse_service(HOST_INFO_0_WCS) == TENANT_WCS
|
||||
|
||||
# wcs new tenant directory
|
||||
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
|
||||
mocked_run.side_effect = [CalledProcessError(1, ''), MockedCompletedProcess(get_config_pck())]
|
||||
result = publik_dump.parse_service(HOST_INFO_0_WCS)
|
||||
assert result['path'] == '/var/lib/wcs/tenants'
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(get_config_pck()))
|
||||
@mock.patch('publik_dump.publik_dump.PublikDump.get_host_info', return_value=[HOST_INFO_0])
|
||||
def test_get_tenant_info(mocked_get_host_info, mocked_run, publik_dump):
|
||||
tenant = publik_dump.get_tenant_info()
|
||||
assert len(mocked_run.mock_calls) == 1
|
||||
assert tenant == TENANT_INFO
|
||||
|
||||
# result from cache
|
||||
tenant = publik_dump.get_tenant_info()
|
||||
assert len(mocked_run.mock_calls) == 1
|
||||
assert tenant == TENANT_INFO
|
||||
|
||||
# renew cache
|
||||
publik_dump.update = True
|
||||
tenant = publik_dump.get_tenant_info()
|
||||
assert tenant == TENANT_INFO
|
||||
|
||||
|
||||
def test_get_dump_folder(publik_dump):
|
||||
service = HOST_INFO_0_HOBO
|
||||
assert publik_dump.get_dump_folder(service).strip('/').split('/')[0] == 'tmp'
|
||||
assert publik_dump.get_dump_folder(service).strip('/').split('/')[-1] == service['url']
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run')
|
||||
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
|
||||
def test_dump_tenant_files(mocked_tenant_info, mocked_run, publik_dump):
|
||||
publik_dump.dump_tenant_files()
|
||||
assert len(mocked_run.mock_calls) == 9
|
||||
|
||||
# hobo
|
||||
assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh hobo.node2.test.saas.entrouvert.org '" \
|
||||
"sudo tar -C /var/lib/hobo/tenants -Jcf - hobo-eurelien.test.entrouvert.org" \
|
||||
"' > HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz"
|
||||
|
||||
# authentic
|
||||
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh authentic.node2.test.saas.entrouvert.org '"\
|
||||
"sudo tar -C /var/lib/authentic2-multitenant/tenants -Jcf - connexion-eurelien.test.entrouvert.org" \
|
||||
"' > HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz"
|
||||
|
||||
# wcs
|
||||
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh wcs.node2.test.saas.entrouvert.org '" \
|
||||
"sudo tar -C /var/lib/wcs -Jcf - demarches-eurelien.test.entrouvert.org" \
|
||||
"' > HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz"
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(TARGET.encode()))
|
||||
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
|
||||
def test_restore_tenant_files(mocked_tenant_info, mocked_run, publik_dump):
|
||||
publik_dump.target = TARGET
|
||||
publik_dump.restore_tenant_files()
|
||||
assert len(mocked_run.mock_calls) == 1 + 9
|
||||
assert mocked_run.mock_calls[0][1][0] == \
|
||||
'ssh node1.test-hds.saas.entrouvert hostname -f'
|
||||
|
||||
# hobo
|
||||
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"cat HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz"\
|
||||
" | ssh hobo.node1.test-hds.saas.entrouvert"\
|
||||
" 'sudo tar -C /var/lib/hobo/tenants -Jxf -'"
|
||||
|
||||
# authentic
|
||||
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"cat HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz"\
|
||||
" | ssh authentic.node1.test-hds.saas.entrouvert"\
|
||||
" 'sudo tar -C /var/lib/authentic2-multitenant/tenants -Jxf -'"
|
||||
|
||||
# wcs
|
||||
assert mocked_run.mock_calls[3][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"cat HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz"\
|
||||
" | ssh wcs.node1.test-hds.saas.entrouvert"\
|
||||
" 'sudo tar -C /var/lib/wcs -Jxf -'"
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run')
|
||||
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
|
||||
def test_dump_tenant_databases(mocked_tenant_info, mocked_run, publik_dump):
|
||||
publik_dump.dump_tenant_databases()
|
||||
assert len(mocked_run.mock_calls) == 9
|
||||
|
||||
# hobo
|
||||
assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh db2.test.saas.entrouvert.org"\
|
||||
" 'sudo -u postgres pg_dump -n hobo_eurelien_test_entrouvert_org -Fc hobo'"\
|
||||
" > HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz"
|
||||
|
||||
# authentic
|
||||
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh db2.test.saas.entrouvert.org"\
|
||||
" 'sudo -u postgres pg_dump -n connexion_eurelien_test_entrouvert_org -Fc authentic2_multitenant'"\
|
||||
" > HF/connexion-eurelien.test.entrouvert.org/connexion_eurelien_test_entrouvert_org.sql.gz"
|
||||
|
||||
# wcs
|
||||
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh db2.test.saas.entrouvert.org"\
|
||||
" 'sudo -u postgres pg_dump -Fc wcs_demarches_eurelien_test_entrouvert_org'"\
|
||||
" > HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz"
|
||||
|
||||
|
||||
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(DB_TARGET.encode()))
|
||||
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
|
||||
def test_restore_tenant_database(mocked_tenant_info, mocked_run, publik_dump):
|
||||
publik_dump.dbtarget = DB_TARGET
|
||||
publik_dump.restore_tenant_databases()
|
||||
assert len(mocked_run.mock_calls) == 1 + 3 + 2*8
|
||||
assert mocked_run.mock_calls[0][1][0] == \
|
||||
'ssh sql3.test-hds.saas.entrouvert hostname -f'
|
||||
|
||||
# hobo
|
||||
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh sql3.test-hds.saas.entrouvert"\
|
||||
" 'sudo -u postgres psql -c \"drop schema if exists hobo_eurelien_test_entrouvert_org cascade\""\
|
||||
" hobo'"
|
||||
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"cat HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz"\
|
||||
" | ssh sql3.test-hds.saas.entrouvert 'sudo -u postgres pg_restore -d hobo'"
|
||||
|
||||
# wcs
|
||||
assert mocked_run.mock_calls[5][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh sql3.test-hds.saas.entrouvert"\
|
||||
" 'sudo -u postgres dropdb --if-exists wcs_demarches_eurelien_test_entrouvert_org'"
|
||||
assert mocked_run.mock_calls[6][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"ssh sql3.test-hds.saas.entrouvert"\
|
||||
" 'sudo -u postgres createdb wcs_demarches_eurelien_test_entrouvert_org"\
|
||||
" --owner wcs --template=\"template0\" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"
|
||||
assert mocked_run.mock_calls[7][1][0].replace(publik_dump.host_folder, 'HF') == \
|
||||
"cat HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz"\
|
||||
" | ssh sql3.test-hds.saas.entrouvert"\
|
||||
" 'sudo -u postgres pg_restore -d wcs_demarches_eurelien_test_entrouvert_org'"
|
||||
Loading…
Reference in New Issue