From 1e221eb962512c965d0835b128d75d00d6089a9c Mon Sep 17 00:00:00 2001 From: Nicolas ROCHE Date: Sat, 23 Oct 2021 06:15:47 +0200 Subject: [PATCH] publik-dump: add tests --- publik-dump/.gitignore | 3 + publik-dump/bin/publik_dump.py | 179 --------- publik-dump/publik_dump/__init__.py | 0 .../{bin => publik_dump}/adapt_wcs_config.py | 0 .../{bin => publik_dump}/list_tenants.py | 2 +- publik-dump/publik_dump/publik_dump.py | 199 ++++++++++ publik-dump/tests/__init__.py | 0 publik-dump/tests/config.pck | Bin 0 -> 7162 bytes publik-dump/tests/list_tenants.json | 3 + publik-dump/tests/test.py | 31 ++ publik-dump/tests/test_publik_dump.py | 343 ++++++++++++++++++ 11 files changed, 580 insertions(+), 180 deletions(-) delete mode 100755 publik-dump/bin/publik_dump.py create mode 100644 publik-dump/publik_dump/__init__.py rename publik-dump/{bin => publik_dump}/adapt_wcs_config.py (100%) rename publik-dump/{bin => publik_dump}/list_tenants.py (95%) create mode 100755 publik-dump/publik_dump/publik_dump.py create mode 100644 publik-dump/tests/__init__.py create mode 100644 publik-dump/tests/config.pck create mode 100644 publik-dump/tests/list_tenants.json create mode 100644 publik-dump/tests/test.py create mode 100644 publik-dump/tests/test_publik_dump.py diff --git a/publik-dump/.gitignore b/publik-dump/.gitignore index 53752db..cc8e6ef 100644 --- a/publik-dump/.gitignore +++ b/publik-dump/.gitignore @@ -1 +1,4 @@ output +__pycache__ +*.pyc +*~ diff --git a/publik-dump/bin/publik_dump.py b/publik-dump/bin/publik_dump.py deleted file mode 100755 index 680a543..0000000 --- a/publik-dump/bin/publik_dump.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/python3 -import argparse -import json -import os -import pickle -import subprocess - -parser = argparse.ArgumentParser() -parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore", "invalidate"]) -parser.add_argument("host", help="origin host") -parser.add_argument("tenant", help="hobo tenant url") -parser.add_argument("--update", action="store_true") -parser.add_argument("--target", help="destination host") -parser.add_argument("--dbtarget", help="destination host") -args = parser.parse_args() -host_folder = "output/%s" % args.host - - -def run(cmd): - print("+ %s" % cmd) - return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE) - - -def get_dump_folder(service): - dump_folder = "%s/%s" % (host_folder, service["url"]) - if not os.path.isdir(dump_folder): - os.mkdir(dump_folder) - return dump_folder - - -def dump_tenant_files(tenant): - for service in tenant["services"]: - dump_folder = get_dump_folder(service) - output = "%s/%s.tar.xz" % (dump_folder, service["url"]) - run( - "ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s" - % (service["name"], args.host, service["path"], service["url"], output) - ) - - -def restore_tenant_files(tenant): - assert run("ssh %s hostname -f" % args.target).stdout.decode().strip() == args.target - import pdb; pdb.set_trace() - # for service in tenant["services"]: - # dump_folder = get_dump_folder(service) - # input_file = "%s/%s.tar.xz" % (dump_folder, service["url"]) - # run( - # "cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'" - # % (input_file, service["name"], args.target, service["path"]) - # ) - - -def dump_tenant_databases(tenant): - # node2.test.saas.entrouvert.org -> db2.test.saas.entrouvert.org - domain = '.'.join(args.host.split('.')[1:]) - for service in tenant["services"]: - dump_folder = get_dump_folder(service) - if service["name"] == "wcs": - dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) - run( - "ssh db2.%s 'sudo -u postgres pg_dump -Fc %s' > %s" - % (domain, service["database"], dump_file) - ) - else: - dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) - run( - "ssh db2.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s" - % (domain, service["schema"], service["database"], dump_file) - ) - - -def restore_tenant_databases(tenant): - assert run("ssh %s hostname -f" % args.dbtarget).stdout.decode().strip() == args.dbtarget - for service in tenant["services"]: - dump_folder = get_dump_folder(service) - import pdb; pdb.set_trace() - # if service["name"] == "wcs": - # dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) - # run( - # "ssh %s sudo -u postgres dropdb --if-exists %s" - # % (args.dbtarget, service["database"]) - # ) - # run( - # """ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'""" - # % (args.dbtarget, service["database"]) - # ) - # run( - # "cat %s | ssh %s sudo -u postgres pg_restore -d %s" - # % (dump_file, args.dbtarget, service["database"]) - # ) - # else: - # dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) - # run( - # """ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'""" - # % (args.dbtarget, service["schema"], service["database"]) - # ) - # run( - # "cat %s | ssh %s sudo -u postgres pg_restore -d %s" - # % (dump_file, args.dbtarget, service["database"]) - # ) - - -def parse_service(service): - if service["name"] == "authentic": - path = "/var/lib/authentic2-multitenant/tenants" - database = "authentic2_multitenant" - elif service["name"] == "wcs": - path = "/var/lib/wcs" - try: - wcs_config = run( - "ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck 2>/dev/null || cat /var/lib/wcs/tenants/%s/config.pck'" - % (args.host, service["url"], service["url"]) - ) - config = pickle.loads(wcs_config.stdout) - database = config["postgresql"]["database"] - except (KeyError, subprocess.CalledProcessError): - database = None - else: - database = service["name"] - # TODO maybe /tenants/ pour wcs - path = "/var/lib/%s/tenants" % service["name"] - return dict(service, database=database, path=path) - - -def get_host_info(): - if os.path.isfile("%s/data" % host_folder) and not args.update: - return - if not os.path.isdir(host_folder): - os.makedirs(host_folder) - run("scp bin/list_tenants.py hobo.%s:" % args.host) - output = run( - "ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command " - "runscript ~/list_tenants.py --all-tenants'" % args.host - ) - tenants = [] - for line in output.stdout.decode().split("\n"): - if line: - tenant = json.loads(line) - for i, service in enumerate(tenant["services"]): - tenant["services"][i] = parse_service(service) - tenants.append(tenant) - with open("%s/data" % host_folder, "w") as fh: - fh.write(json.dumps(tenants, indent=4)) - - -def get_tenant_info(tenant_name): - with open("%s/data" % host_folder) as fh: - hobos = json.loads(fh.read()) - tenant_infos = [x for x in hobos if x["name"] == tenant_name] - if not len(tenant_infos) == 1: - raise(Exception('tenant not found')) - return tenant_infos[0] - - -def invalidate(tenant_name): - tenant_info = get_tenant_info(tenant_name) - import pdb; pdb.set_trace() - # for service in tenant_info['services']: - # bpath = "%s/%s" % (service['path'], service['url']) - # run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], args.host, bpath, bpath)) - - -if __name__ == "__main__": - if args.action == "tenantinfo": - get_host_info() - print(json.dumps(get_tenant_info(args.tenant), indent=4)) - elif args.action == "tenanturls": - print(' '.join([x['url'] for x in get_tenant_info(args.tenant)['services']])) - elif args.action == "dump": - get_host_info() - tenant = get_tenant_info(args.tenant) - dump_tenant_databases(tenant) - dump_tenant_files(tenant) - elif args.action == "restore": - tenant = get_tenant_info(args.tenant) - #restore_tenant_files(tenant) - #restore_tenant_databases(tenant) - elif args.action == "invalidate": - invalidate(args.tenant) diff --git a/publik-dump/publik_dump/__init__.py b/publik-dump/publik_dump/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/publik-dump/bin/adapt_wcs_config.py b/publik-dump/publik_dump/adapt_wcs_config.py similarity index 100% rename from publik-dump/bin/adapt_wcs_config.py rename to publik-dump/publik_dump/adapt_wcs_config.py diff --git a/publik-dump/bin/list_tenants.py b/publik-dump/publik_dump/list_tenants.py similarity index 95% rename from publik-dump/bin/list_tenants.py rename to publik-dump/publik_dump/list_tenants.py index f8fe01c..af1b450 100644 --- a/publik-dump/bin/list_tenants.py +++ b/publik-dump/publik_dump/list_tenants.py @@ -1,6 +1,6 @@ #!/usr/bin/python3 import json -from urllib.parse import urlparse, urlsplit +from urllib.parse import urlsplit from hobo.environment.models import AVAILABLE_SERVICES from hobo.multitenant.middleware import TenantMiddleware from django.db import connection diff --git a/publik-dump/publik_dump/publik_dump.py b/publik-dump/publik_dump/publik_dump.py new file mode 100755 index 0000000..efb23fa --- /dev/null +++ b/publik-dump/publik_dump/publik_dump.py @@ -0,0 +1,199 @@ +#!/usr/bin/python3 +import argparse +import json +import os +import pickle +import subprocess + + +class PublikDump(): + + def __init__(self, host, hobo_tenant, update=False, target=None, dbtarget=None): + self.host = host + self.hobo_tenant = hobo_tenant + self.update = update + self.target = target + self.dbtarget = dbtarget + self.host_folder = "output/%s" % host + + @classmethod + def run(cls, cmd): + print("+ %s" % cmd) + return subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE) + + def get_host_info(self): + host_path = "%s/data" % self.host_folder + if os.path.isfile(host_path) and not self.update: + return json.load(open(host_path)) + + if not os.path.isdir(self.host_folder): + os.makedirs(self.host_folder) + self.run("scp publik_dump/list_tenants.py hobo.%s:" % self.host) + output = self.run( + "ssh hobo.%s 'sudo -u hobo HOME=$HOME hobo-manage tenant_command " + "runscript ~/list_tenants.py --all-tenants'" % self.host + ) + tenants = [] + for line in output.stdout.decode().split("\n"): + if line: + tenant_infos = json.loads(line) + tenants.append(tenant_infos) + + json.dump(tenants, open(host_path, "w"), indent=4) + return tenants + + def parse_service(self, service): + if service["name"] == "authentic": + path = "/var/lib/authentic2-multitenant/tenants" + database = "authentic2_multitenant" + elif service["name"] == "wcs": + try: + wcs_config = self.run( + "ssh wcs.%s 'cat /var/lib/wcs/%s/config.pck'" + % (self.host, service["url"]) + ) + path = "/var/lib/wcs" + except (KeyError, subprocess.CalledProcessError): + wcs_config = self.run( + "ssh wcs.%s 'cat /var/lib/wcs/tenants/%s/config.pck'" + % (self.host, service["url"]) + ) + path = "/var/lib/wcs/tenants" + config = pickle.loads(wcs_config.stdout) + database = config["postgresql"]["database"] + else: + path = "/var/lib/%s/tenants" % service["name"] + database = service["name"] + return dict(service, database=database, path=path) + + def get_tenant_info(self): + tenant_path = '%s/%s.json' % (self.host_folder, self.hobo_tenant) + if os.path.isfile(tenant_path) and not self.update: + return json.load(open(tenant_path)) + + hobos = self.get_host_info() + tenants = [x for x in hobos if x["name"] == self.hobo_tenant] + if not len(tenants) == 1: + raise(Exception('tenant not found')) + tenant = tenants[0] + for service in tenant['services']: + service.update(self.parse_service(service)) + + json.dump(tenant, open(tenant_path, "w"), indent=4) + return tenant + + def get_dump_folder(self, service): + dump_folder = "%s/%s" % (self.host_folder, service["url"]) + if not os.path.isdir(dump_folder): + os.mkdir(dump_folder) + return dump_folder + + + def dump_tenant_files(self): + tenant = self.get_tenant_info() + for service in tenant["services"]: + dump_folder = self.get_dump_folder(service) + output = "%s/%s.tar.xz" % (dump_folder, service["url"]) + self.run( + "ssh %s.%s 'sudo tar -C %s -Jcf - %s' > %s" + % (service["name"], self.host, service["path"], service["url"], output) + ) + + def restore_tenant_files(self): + tenant = self.get_tenant_info() + assert self.run("ssh %s hostname -f" % self.target).stdout.decode().strip() == self.target + for service in tenant["services"]: + dump_folder = self.get_dump_folder(service) + input_file = "%s/%s.tar.xz" % (dump_folder, service["url"]) + self.run( + "cat %s | ssh %s.%s 'sudo tar -C %s -Jxf -'" + % (input_file, service["name"], self.target, service["path"]) + ) + + def dump_tenant_databases(self): + # node2.test.saas.entrouvert.org -> db2.test.saas.entrouvert.org + domain = '.'.join(self.host.split('.')[1:]) + + tenant = self.get_tenant_info() + for service in tenant["services"]: + dump_folder = self.get_dump_folder(service) + if service["name"] == "wcs": + dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) + self.run( + "ssh db2.%s 'sudo -u postgres pg_dump -Fc %s' > %s" + % (domain, service["database"], dump_file) + ) + else: + dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) + self.run( + "ssh db2.%s 'sudo -u postgres pg_dump -n %s -Fc %s' > %s" + % (domain, service["schema"], service["database"], dump_file) + ) + + def restore_tenant_databases(self): + assert self.run("ssh %s hostname -f" % self.dbtarget).stdout.decode().strip() == self.dbtarget + + tenant = self.get_tenant_info() + for service in tenant["services"]: + dump_folder = self.get_dump_folder(service) + if service["name"] == "wcs": + dump_file = "%s/%s.sql.gz" % (dump_folder, service["database"]) + self.run( + "ssh %s 'sudo -u postgres dropdb --if-exists %s'" + % (self.dbtarget, service["database"]) + ) + self.run( + """ssh %s 'sudo -u postgres createdb %s --owner wcs --template="template0" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'""" + % (self.dbtarget, service["database"]) + ) + self.run( + "cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'" + % (dump_file, self.dbtarget, service["database"]) + ) + else: + dump_file = "%s/%s.sql.gz" % (dump_folder, service["schema"]) + self.run( + """ssh %s 'sudo -u postgres psql -c "drop schema if exists %s cascade" %s'""" + % (self.dbtarget, service["schema"], service["database"]) + ) + self.run( + "cat %s | ssh %s 'sudo -u postgres pg_restore -d %s'" + % (dump_file, self.dbtarget, service["database"]) + ) + + def invalidate_tenant(self): + tenant = self.get_tenant_info() + for service in tenant['services']: + bpath = "%s/%s" % (service['path'], service['url']) + self.run('ssh %s.%s sudo mv %s %s.invalid' % (service['name'], self.host, bpath, bpath)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("action", default="dump", choices=["tenantinfo", "tenanturls", "dump", "restore", "invalidate"]) + parser.add_argument("host", help="origin host") + parser.add_argument("hobo_tenant", help="hobo tenant url") + parser.add_argument("--update", action="store_true") + parser.add_argument("--target", help="destination host") + parser.add_argument("--dbtarget", help="destination host") + args = parser.parse_args() + publik_dump = PublikDump( + host=args.host, + hobo_tenant=args.hobo_tenant, + update=args.update, + target=args.target, + dbtarget=args.dbtarget, + ) + + if args.action == "tenantinfo": + print(json.dumps(publik_dump.get_tenant_info(), indent=4)) + elif args.action == "tenanturls": + print(' '.join([x['url'] for x in publik_dump.get_tenant_info()['services']])) + elif args.action == "dump": + publik_dump.dump_tenant_databases() + publik_dump.dump_tenant_files() + elif args.action == "restore": + publik_dump.restore_tenant_files() + publik_dump.restore_tenant_databases() + elif args.action == "invalidate": + publik_dump.invalidate_tenant() diff --git a/publik-dump/tests/__init__.py b/publik-dump/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/publik-dump/tests/config.pck b/publik-dump/tests/config.pck new file mode 100644 index 0000000000000000000000000000000000000000..edb4710b0f49e4b83352f8ab3c0a4c683cc14cd9 GIT binary patch literal 7162 zcmdT}`+plp5fACxZTbi;&{9AokQgT?*_I!2b5$GifYR0tC53|`j@8{t+W7AFW_Qny z3ZgBKHZA4-{*7PzcjO;qc2=iX?3(!FAL5^MJ3Bi&^PSn*nbmv82XJgYc=*_{V{OH< zSY%!J+yIWx2M-|0^mxkQzTi>(&Wm;;w!Aje9PSUEL5j$tq=e=Lcz6JhkVLMOR3T59NMJ5_9+$8<71=q(BW?_TnP*1H zVw)>djbsXs>fk}-A$|>y4dBFla6b|{3MU^A&f{msy_{)J@myxH;^%mwcp`Y_8LrKw zJgK_y1c~cNX}E$XgLiTL<=da;Oc^vfGnM%46GQ!8 zq=C$g>~uuLNzhkI#{mI~7Yce0>0?T$IMlFp9y`r1VJ8Hwc>^%7t$RiztnD*No{I z6{;At8eKk1hchI>;sR&sb9JS@3@*kBjWP1A;69YQ6KPm*do!(fq>ABO@TA?a$3^1} z*=AyMgYj$)&KJD{s^aw~G^qB*8`pY!?6pW5jlfw77M!Z#$S+~(m(U^;j{Oo^zbqGI zoJOqc16TobS zWtojZi=fAhQKDU-0^r{w&?Y#POpyNKE!tbNG+7RPWOQCf_)cptt^io3Cgglf)Hxa7iJiERvR%XOK~Qtk*N zwkvdCc^0XDZov+R40gp;IvnFUZ+7u_f~V1zF+(`k>AS(RQ&ZlIdaq1?Ju>Wj!AV?1 zka;DsdK0mHwIJo~Oz48c*o^<$0A8ODp29UlV&9{qY=dT#PLk$H2i_o0DBcy?C<-Y` zBiDk9xUzID9sQg3n!%c&U`?(#gIuJsXPC zM#J?1yyX&r4NmystHm7u_nf@@@T?vefn-)S1>fKqh13mt$T9|R?|hz`N>IB0(o zoIs670Yb9h-Xzb^#8`X(Du}i<~RUfoNzbTqBO>0EDsb!bI9CeIh164BupQ2k{;4G z(0AUG83*NdasyYgq_Qp0Ktqzl;izyq7E3P7Nku4R%rtD%0Sm=6b%&f5ORWW0L(ZGD zJe(%>i$c3H0JE)zcLwkaS0q^H#JmH)j6!Qxq~|bQ;Kv*cf{*a>@`nsf--6wI3zt`} z3ZBH;uMx$MvFa#L0hu*BEYaMrO;G$8lOmcJyJ}81ZFnHZ4Alx@ZDgAe;l{qql77gN zMDBzLVmUuU^|3y|3Ws=|nK3Az#p3sv4kMNuEZIt>CrQ5E5_xFL*|5I07;X!`Q?l(c z<74Tu)j_~2k z*1UP0B6mFY5h<6G{2G4xH3De{qif2+lgae|Ldb2hH$dYocTb_jZ{o=dBaKmvSyne~ zFNB?0o^ZpxuGq##%lg-^%`)zX@1(tZg1mW1;Epbcql0aSr6TDc5ZnoS_r&%hU4^{j z!Z^X^OUcakxYOZG*XJGQeZOcPuDXYg`)qtp1@chcSsqFR!@2!%oJQ@SX>wSR9Gx&7 zD(od9<9NB;f#5i+?wVys_U;MqB`jGgw&iEZ;XpjB>y8e@&UkDz;sd1Fk+q)zaGDm` z%&9+U{Myp^=pZ`^UcSwf<-3nwXYHQIULU4b_e0&m;e_x|)KSTl6rb9!lg?Lwt&D=V+qX7h01DrLI1^eDkLtNFh04SVgw zL~p8nKT6=uC_agu>yWUilJu0RZRdwi2l%%MwX`(uaye@D?0l_!e}G@n^GSl8UbY19 zegN;i3GdU>kLz2h0E1sozUKU9_*7)uTHAiWZ(H!Y`8%GAKA3$T`j8jyWy=2PKl#KcnweB%En52quyCrhCgt6riqxscl1BD;7{|tPkKnS z|C^QVx`scu;4i}o`){0me$(*R7W|DCyZ7?Hx8NUS8Me;6E9oz|f41OX!*%wLJ^XhF Q(d=sYcMJaG=tWWeA4o_7WB>pF literal 0 HcmV?d00001 diff --git a/publik-dump/tests/list_tenants.json b/publik-dump/tests/list_tenants.json new file mode 100644 index 0000000..1abbf5e --- /dev/null +++ b/publik-dump/tests/list_tenants.json @@ -0,0 +1,3 @@ +{"name": "hobo-eurelien.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-eurelien.test.entrouvert.org", "schema": "hobo_eurelien_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-eurelien.test.entrouvert.org", "schema": "connexion_eurelien_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-eurelien.test.entrouvert.org", "schema": "wcs_demarches_eurelien_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-eurelien.test.entrouvert.org", "schema": "passerelle_eurelien_test_entrouvert_org"}, {"name": "combo", "url": "agents-eurelien.test.entrouvert.org", "schema": "agents_eurelien_test_entrouvert_org"}, {"name": "combo", "url": "portail-eurelien.test.entrouvert.org", "schema": "portail_eurelien_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-eurelien.test.entrouvert.org", "schema": "portedoc_eurelien_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-eurelien.test.entrouvert.org", "schema": "agendas_eurelien_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-eurelien.test.entrouvert.org", "schema": "statistiques_eurelien_test_entrouvert_org"}]} +{"name": "hobo-tours-ville.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-tours-ville.test.entrouvert.org", "schema": "hobo_tours_ville_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-tours.test.entrouvert.org", "secondary": true, "schema": "connexion_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours.test.entrouvert.org", "secondary": true, "schema": "wcs_demarches_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours-ville.test.entrouvert.org", "schema": "wcs_demarches_tours_ville_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours.test.entrouvert.org", "secondary": true, "schema": "passerelle_tours_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours-ville.test.entrouvert.org", "schema": "passerelle_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours-ville.test.entrouvert.org", "schema": "agents_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours.test.entrouvert.org", "secondary": true, "schema": "agents_tours_test_entrouvert_org"}, {"name": "combo", "url": "tours.test.entrouvert.org", "secondary": true, "schema": "tours_test_entrouvert_org"}, {"name": "combo", "url": "tours-ville.test.entrouvert.org", "schema": "tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours-ville.test.entrouvert.org", "schema": "portedoc_tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours.test.entrouvert.org", "secondary": true, "schema": "portedoc_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours.test.entrouvert.org", "secondary": true, "schema": "agendas_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours-ville.test.entrouvert.org", "schema": "agendas_tours_ville_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-tours.test.entrouvert.org", "secondary": true, "schema": "statistiques_tours_test_entrouvert_org"}, {"name": "bijoe", "url": "stats-tours-ville.test.entrouvert.org", "schema": "stats_tours_ville_test_entrouvert_org"}, {"name": "hobo", "url": "hobo-tours.test.entrouvert.org", "secondary": true, "schema": "hobo_tours_test_entrouvert_org"}]} +{"name": "hobo-tours.test.entrouvert.org", "services": [{"name": "hobo", "url": "hobo-tours.test.entrouvert.org", "schema": "hobo_tours_test_entrouvert_org"}, {"name": "authentic", "url": "connexion-tours.test.entrouvert.org", "schema": "connexion_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours.test.entrouvert.org", "schema": "wcs_demarches_tours_test_entrouvert_org"}, {"name": "wcs", "url": "demarches-tours-ville.test.entrouvert.org", "secondary": true, "schema": "wcs_demarches_tours_ville_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours.test.entrouvert.org", "schema": "passerelle_tours_test_entrouvert_org"}, {"name": "passerelle", "url": "passerelle-tours-ville.test.entrouvert.org", "secondary": true, "schema": "passerelle_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours.test.entrouvert.org", "schema": "agents_tours_test_entrouvert_org"}, {"name": "combo", "url": "agents-tours-ville.test.entrouvert.org", "secondary": true, "schema": "agents_tours_ville_test_entrouvert_org"}, {"name": "combo", "url": "tours.test.entrouvert.org", "schema": "tours_test_entrouvert_org"}, {"name": "combo", "url": "tours-ville.test.entrouvert.org", "secondary": true, "schema": "tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours-ville.test.entrouvert.org", "secondary": true, "schema": "portedoc_tours_ville_test_entrouvert_org"}, {"name": "fargo", "url": "portedoc-tours.test.entrouvert.org", "schema": "portedoc_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours.test.entrouvert.org", "schema": "agendas_tours_test_entrouvert_org"}, {"name": "chrono", "url": "agendas-tours-ville.test.entrouvert.org", "secondary": true, "schema": "agendas_tours_ville_test_entrouvert_org"}, {"name": "bijoe", "url": "statistiques-tours.test.entrouvert.org", "schema": "statistiques_tours_test_entrouvert_org"}, {"name": "bijoe", "url": "stats-tours-ville.test.entrouvert.org", "secondary": true, "schema": "stats_tours_ville_test_entrouvert_org"}, {"name": "hobo", "url": "hobo-tours-ville.test.entrouvert.org", "schema": "hobo_tours_ville_test_entrouvert_org"}]} diff --git a/publik-dump/tests/test.py b/publik-dump/tests/test.py new file mode 100644 index 0000000..462c032 --- /dev/null +++ b/publik-dump/tests/test.py @@ -0,0 +1,31 @@ +import pytest +import os + +from publik_dump.publik_dump import PublikDump + +@pytest.fixture +def output(tmpdir): + return str(tmpdir) + + +@pytest.fixture +def service(): + return "" + + +@pytest.fixture +def publik_dump(): + return PublikDump + +def test_get_dump_folder(output, publik_dump, service): + #filename = os.path.join(str(tmpdir), 'file.tar') + assert publik_dump.coucou() == "nico" + + +def test_dump_tenant_files(): + print("coucou") + + +# patch run +def get_host_info(): + pass diff --git a/publik-dump/tests/test_publik_dump.py b/publik-dump/tests/test_publik_dump.py new file mode 100644 index 0000000..e43ee34 --- /dev/null +++ b/publik-dump/tests/test_publik_dump.py @@ -0,0 +1,343 @@ +import json +import mock +import os +import pytest +from subprocess import CalledProcessError + +from publik_dump.publik_dump import PublikDump + + +HOST_INFO_0 = { + "name": "hobo-eurelien.test.entrouvert.org", + "services": [ + { + "name": "hobo", + "url": "hobo-eurelien.test.entrouvert.org", + "schema": "hobo_eurelien_test_entrouvert_org" + }, + { + "name": "authentic", + "url": "connexion-eurelien.test.entrouvert.org", + "schema": "connexion_eurelien_test_entrouvert_org" + }, + { + "name": "wcs", + "url": "demarches-eurelien.test.entrouvert.org", + "schema": "wcs_demarches_eurelien_test_entrouvert_org" + }, + { + "name": "passerelle", + "url": "passerelle-eurelien.test.entrouvert.org", + "schema": "passerelle_eurelien_test_entrouvert_org" + }, + { + "name": "combo", + "url": "agents-eurelien.test.entrouvert.org", + "schema": "agents_eurelien_test_entrouvert_org" + }, + { + "name": "combo", + "url": "portail-eurelien.test.entrouvert.org", + "schema": "portail_eurelien_test_entrouvert_org" + }, + { + "name": "fargo", + "url": "portedoc-eurelien.test.entrouvert.org", + "schema": "portedoc_eurelien_test_entrouvert_org" + }, + { + "name": "chrono", + "url": "agendas-eurelien.test.entrouvert.org", + "schema": "agendas_eurelien_test_entrouvert_org" + }, + { + "name": "bijoe", + "url": "statistiques-eurelien.test.entrouvert.org", + "schema": "statistiques_eurelien_test_entrouvert_org" + } + ] +} +HOST_INFO_0_HOBO = HOST_INFO_0['services'][0] +HOST_INFO_0_AUTHENTIC = HOST_INFO_0['services'][1] +HOST_INFO_0_WCS = HOST_INFO_0['services'][2] + +TENANT_INFO = { + "name": "hobo-eurelien.test.entrouvert.org", + "services": [ + { + "name": "hobo", + "url": "hobo-eurelien.test.entrouvert.org", + "schema": "hobo_eurelien_test_entrouvert_org", + "database": "hobo", + "path": "/var/lib/hobo/tenants" + }, + { + "name": "authentic", + "url": "connexion-eurelien.test.entrouvert.org", + "schema": "connexion_eurelien_test_entrouvert_org", + "database": "authentic2_multitenant", + "path": "/var/lib/authentic2-multitenant/tenants" + }, + { + "name": "wcs", + "url": "demarches-eurelien.test.entrouvert.org", + "schema": "wcs_demarches_eurelien_test_entrouvert_org", + "database": "wcs_demarches_eurelien_test_entrouvert_org", + "path": "/var/lib/wcs" + }, + { + "name": "passerelle", + "url": "passerelle-eurelien.test.entrouvert.org", + "schema": "passerelle_eurelien_test_entrouvert_org", + "database": "passerelle", + "path": "/var/lib/passerelle/tenants" + }, + { + "name": "combo", + "url": "agents-eurelien.test.entrouvert.org", + "schema": "agents_eurelien_test_entrouvert_org", + "database": "combo", + "path": "/var/lib/combo/tenants" + }, + { + "name": "combo", + "url": "portail-eurelien.test.entrouvert.org", + "schema": "portail_eurelien_test_entrouvert_org", + "database": "combo", + "path": "/var/lib/combo/tenants" + }, + { + "name": "fargo", + "url": "portedoc-eurelien.test.entrouvert.org", + "schema": "portedoc_eurelien_test_entrouvert_org", + "database": "fargo", + "path": "/var/lib/fargo/tenants" + }, + { + "name": "chrono", + "url": "agendas-eurelien.test.entrouvert.org", + "schema": "agendas_eurelien_test_entrouvert_org", + "database": "chrono", + "path": "/var/lib/chrono/tenants" + }, + { + "name": "bijoe", + "url": "statistiques-eurelien.test.entrouvert.org", + "schema": "statistiques_eurelien_test_entrouvert_org", + "database": "bijoe", + "path": "/var/lib/bijoe/tenants" + } + ] +} +TENANT_HOBO = TENANT_INFO['services'][0] +TENANT_AUTHENTIC = TENANT_INFO['services'][1] +TENANT_WCS = TENANT_INFO['services'][2] + +TARGET = 'node1.test-hds.saas.entrouvert' +DB_TARGET = 'sql3.test-hds.saas.entrouvert' + +@pytest.fixture +def publik_dump(tmpdir): + obj = PublikDump('node2.test.saas.entrouvert.org', 'hobo-eurelien.test.entrouvert.org') + obj.host_folder = str(tmpdir) + #obj.run = lambda x: x + return obj + + +class MockedCompletedProcess(): + '''simulate subprocess.run return''' + def __init__(self, stdout): + self.stdout = stdout + + +def test_run(publik_dump): + with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run: + mocked_run.return_value = MockedCompletedProcess('foo result') + output = publik_dump.run('foo query') + assert mocked_run.mock_calls[0][1][0] == 'foo query' + assert output.stdout == 'foo result' + + +def get_list_tenants(): + with open(os.path.join(os.path.dirname(__file__), 'list_tenants.json'), 'rb') as desc: + return desc.read() + +def get_config_pck(): + with open(os.path.join(os.path.dirname(__file__), 'config.pck'), 'rb') as desc: + return desc.read() + + +@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(get_list_tenants())) +def test_get_host_info(mocked_run, publik_dump): + tenants = publik_dump.get_host_info() + assert len(mocked_run.mock_calls) == 2 + assert mocked_run.mock_calls[0][1][0] == \ + 'scp publik_dump/list_tenants.py hobo.node2.test.saas.entrouvert.org:' + assert mocked_run.mock_calls[1][1][0] == \ + "ssh hobo.node2.test.saas.entrouvert.org"\ + " 'sudo -u hobo HOME=$HOME hobo-manage tenant_command runscript ~/list_tenants.py --all-tenants'" + assert tenants[0] == HOST_INFO_0 + + # result from cache + tenants = publik_dump.get_host_info() + assert len(mocked_run.mock_calls) == 2 + assert tenants[0] == HOST_INFO_0 + + # renew cache + publik_dump.update = True + tenants = publik_dump.get_host_info() + assert len(mocked_run.mock_calls) == 4 + assert tenants[0] == HOST_INFO_0 + + +def test_parse_service(publik_dump): + # hobo + assert publik_dump.parse_service(HOST_INFO_0_HOBO) == TENANT_HOBO + + # authentic + assert publik_dump.parse_service(HOST_INFO_0_AUTHENTIC) == TENANT_AUTHENTIC + + # wcs old tenant directory + with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run: + mocked_run.return_value = MockedCompletedProcess(get_config_pck()) + assert publik_dump.parse_service(HOST_INFO_0_WCS) == TENANT_WCS + + # wcs new tenant directory + with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run: + mocked_run.side_effect = [CalledProcessError(1, ''), MockedCompletedProcess(get_config_pck())] + result = publik_dump.parse_service(HOST_INFO_0_WCS) + assert result['path'] == '/var/lib/wcs/tenants' + + +@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(get_config_pck())) +@mock.patch('publik_dump.publik_dump.PublikDump.get_host_info', return_value=[HOST_INFO_0]) +def test_get_tenant_info(mocked_get_host_info, mocked_run, publik_dump): + tenant = publik_dump.get_tenant_info() + assert len(mocked_run.mock_calls) == 1 + assert tenant == TENANT_INFO + + # result from cache + tenant = publik_dump.get_tenant_info() + assert len(mocked_run.mock_calls) == 1 + assert tenant == TENANT_INFO + + # renew cache + publik_dump.update = True + tenant = publik_dump.get_tenant_info() + assert tenant == TENANT_INFO + + +def test_get_dump_folder(publik_dump): + service = HOST_INFO_0_HOBO + assert publik_dump.get_dump_folder(service).strip('/').split('/')[0] == 'tmp' + assert publik_dump.get_dump_folder(service).strip('/').split('/')[-1] == service['url'] + + +@mock.patch('publik_dump.publik_dump.subprocess.run') +@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO) +def test_dump_tenant_files(mocked_tenant_info, mocked_run, publik_dump): + publik_dump.dump_tenant_files() + assert len(mocked_run.mock_calls) == 9 + + # hobo + assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh hobo.node2.test.saas.entrouvert.org '" \ + "sudo tar -C /var/lib/hobo/tenants -Jcf - hobo-eurelien.test.entrouvert.org" \ + "' > HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz" + + # authentic + assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh authentic.node2.test.saas.entrouvert.org '"\ + "sudo tar -C /var/lib/authentic2-multitenant/tenants -Jcf - connexion-eurelien.test.entrouvert.org" \ + "' > HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz" + + # wcs + assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh wcs.node2.test.saas.entrouvert.org '" \ + "sudo tar -C /var/lib/wcs -Jcf - demarches-eurelien.test.entrouvert.org" \ + "' > HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz" + + +@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(TARGET.encode())) +@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO) +def test_restore_tenant_files(mocked_tenant_info, mocked_run, publik_dump): + publik_dump.target = TARGET + publik_dump.restore_tenant_files() + assert len(mocked_run.mock_calls) == 1 + 9 + assert mocked_run.mock_calls[0][1][0] == \ + 'ssh node1.test-hds.saas.entrouvert hostname -f' + + # hobo + assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \ + "cat HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz"\ + " | ssh hobo.node1.test-hds.saas.entrouvert"\ + " 'sudo tar -C /var/lib/hobo/tenants -Jxf -'" + + # authentic + assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \ + "cat HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz"\ + " | ssh authentic.node1.test-hds.saas.entrouvert"\ + " 'sudo tar -C /var/lib/authentic2-multitenant/tenants -Jxf -'" + + # wcs + assert mocked_run.mock_calls[3][1][0].replace(publik_dump.host_folder, 'HF') == \ + "cat HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz"\ + " | ssh wcs.node1.test-hds.saas.entrouvert"\ + " 'sudo tar -C /var/lib/wcs -Jxf -'" + + +@mock.patch('publik_dump.publik_dump.subprocess.run') +@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO) +def test_dump_tenant_databases(mocked_tenant_info, mocked_run, publik_dump): + publik_dump.dump_tenant_databases() + assert len(mocked_run.mock_calls) == 9 + + # hobo + assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh db2.test.saas.entrouvert.org"\ + " 'sudo -u postgres pg_dump -n hobo_eurelien_test_entrouvert_org -Fc hobo'"\ + " > HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz" + + # authentic + assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh db2.test.saas.entrouvert.org"\ + " 'sudo -u postgres pg_dump -n connexion_eurelien_test_entrouvert_org -Fc authentic2_multitenant'"\ + " > HF/connexion-eurelien.test.entrouvert.org/connexion_eurelien_test_entrouvert_org.sql.gz" + + # wcs + assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh db2.test.saas.entrouvert.org"\ + " 'sudo -u postgres pg_dump -Fc wcs_demarches_eurelien_test_entrouvert_org'"\ + " > HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz" + + +@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(DB_TARGET.encode())) +@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO) +def test_restore_tenant_database(mocked_tenant_info, mocked_run, publik_dump): + publik_dump.dbtarget = DB_TARGET + publik_dump.restore_tenant_databases() + assert len(mocked_run.mock_calls) == 1 + 3 + 2*8 + assert mocked_run.mock_calls[0][1][0] == \ + 'ssh sql3.test-hds.saas.entrouvert hostname -f' + + # hobo + assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh sql3.test-hds.saas.entrouvert"\ + " 'sudo -u postgres psql -c \"drop schema if exists hobo_eurelien_test_entrouvert_org cascade\""\ + " hobo'" + assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \ + "cat HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz"\ + " | ssh sql3.test-hds.saas.entrouvert 'sudo -u postgres pg_restore -d hobo'" + + # wcs + assert mocked_run.mock_calls[5][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh sql3.test-hds.saas.entrouvert"\ + " 'sudo -u postgres dropdb --if-exists wcs_demarches_eurelien_test_entrouvert_org'" + assert mocked_run.mock_calls[6][1][0].replace(publik_dump.host_folder, 'HF') == \ + "ssh sql3.test-hds.saas.entrouvert"\ + " 'sudo -u postgres createdb wcs_demarches_eurelien_test_entrouvert_org"\ + " --owner wcs --template=\"template0\" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'" + assert mocked_run.mock_calls[7][1][0].replace(publik_dump.host_folder, 'HF') == \ + "cat HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz"\ + " | ssh sql3.test-hds.saas.entrouvert"\ + " 'sudo -u postgres pg_restore -d wcs_demarches_eurelien_test_entrouvert_org'"