misc-nroche/publik-dump/tests/test_publik_dump.py

447 lines
19 KiB
Python

import json
import mock
import os
import pytest
from subprocess import CalledProcessError
from publik_dump.publik_dump import PublikDump
HOST_INFO_0 = {
"name": "hobo-eurelien.test.entrouvert.org",
"services": [
{
"name": "hobo",
"url": "hobo-eurelien.test.entrouvert.org",
"schema": "hobo_eurelien_test_entrouvert_org"
},
{
"name": "authentic",
"url": "connexion-eurelien.test.entrouvert.org",
"schema": "connexion_eurelien_test_entrouvert_org"
},
{
"name": "wcs",
"url": "demarches-eurelien.test.entrouvert.org",
"schema": "wcs_demarches_eurelien_test_entrouvert_org"
},
{
"name": "passerelle",
"url": "passerelle-eurelien.test.entrouvert.org",
"schema": "passerelle_eurelien_test_entrouvert_org"
},
{
"name": "combo",
"url": "agents-eurelien.test.entrouvert.org",
"schema": "agents_eurelien_test_entrouvert_org"
},
{
"name": "combo",
"url": "portail-eurelien.test.entrouvert.org",
"schema": "portail_eurelien_test_entrouvert_org"
},
{
"name": "fargo",
"url": "portedoc-eurelien.test.entrouvert.org",
"schema": "portedoc_eurelien_test_entrouvert_org"
},
{
"name": "chrono",
"url": "agendas-eurelien.test.entrouvert.org",
"schema": "agendas_eurelien_test_entrouvert_org"
},
{
"name": "bijoe",
"url": "statistiques-eurelien.test.entrouvert.org",
"schema": "statistiques_eurelien_test_entrouvert_org"
}
]
}
HOST_INFO_0_HOBO = HOST_INFO_0['services'][0]
HOST_INFO_0_AUTHENTIC = HOST_INFO_0['services'][1]
HOST_INFO_0_WCS = HOST_INFO_0['services'][2]
TENANT_INFO = {
"name": "hobo-eurelien.test.entrouvert.org",
"services": [
{
"name": "hobo",
"url": "hobo-eurelien.test.entrouvert.org",
"schema": "hobo_eurelien_test_entrouvert_org",
"database": "hobo",
"path": "/var/lib/hobo/tenants"
},
{
"name": "authentic",
"url": "connexion-eurelien.test.entrouvert.org",
"schema": "connexion_eurelien_test_entrouvert_org",
"database": "authentic2_multitenant",
"path": "/var/lib/authentic2-multitenant/tenants"
},
{
"name": "wcs",
"url": "demarches-eurelien.test.entrouvert.org",
"schema": "wcs_demarches_eurelien_test_entrouvert_org",
"database": "wcs_demarches_eurelien_test_entrouvert_org",
"path": "/var/lib/wcs"
},
{
"name": "passerelle",
"url": "passerelle-eurelien.test.entrouvert.org",
"schema": "passerelle_eurelien_test_entrouvert_org",
"database": "passerelle",
"path": "/var/lib/passerelle/tenants"
},
{
"name": "combo",
"url": "agents-eurelien.test.entrouvert.org",
"schema": "agents_eurelien_test_entrouvert_org",
"database": "combo",
"path": "/var/lib/combo/tenants"
},
{
"name": "combo",
"url": "portail-eurelien.test.entrouvert.org",
"schema": "portail_eurelien_test_entrouvert_org",
"database": "combo",
"path": "/var/lib/combo/tenants"
},
{
"name": "fargo",
"url": "portedoc-eurelien.test.entrouvert.org",
"schema": "portedoc_eurelien_test_entrouvert_org",
"database": "fargo",
"path": "/var/lib/fargo/tenants"
},
{
"name": "chrono",
"url": "agendas-eurelien.test.entrouvert.org",
"schema": "agendas_eurelien_test_entrouvert_org",
"database": "chrono",
"path": "/var/lib/chrono/tenants"
},
{
"name": "bijoe",
"url": "statistiques-eurelien.test.entrouvert.org",
"schema": "statistiques_eurelien_test_entrouvert_org",
"database": "bijoe",
"path": "/var/lib/bijoe/tenants"
}
]
}
TENANT_HOBO = TENANT_INFO['services'][0]
TENANT_AUTHENTIC = TENANT_INFO['services'][1]
TENANT_WCS = TENANT_INFO['services'][2]
TARGET = 'node1.test-hds.saas.entrouvert'
DB_TARGET = 'sql3.test-hds.saas.entrouvert'
@pytest.fixture
def publik_dump(tmpdir):
obj = PublikDump('node2.test.saas.entrouvert.org', 'hobo-eurelien.test.entrouvert.org')
obj.host_folder = str(tmpdir)
#obj.run = lambda x: x
return obj
class MockedCompletedProcess():
'''simulate subprocess.run return'''
def __init__(self, stdout):
self.stdout = stdout
def test_run(publik_dump):
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
mocked_run.return_value = MockedCompletedProcess('foo result')
output = publik_dump.run('foo query')
assert mocked_run.mock_calls[0][1][0] == 'foo query'
assert output.stdout == 'foo result'
def get_list_tenants():
with open(os.path.join(os.path.dirname(__file__), 'list_tenants.json'), 'rb') as desc:
return desc.read()
def get_config_pck():
with open(os.path.join(os.path.dirname(__file__), 'config.pck'), 'rb') as desc:
return desc.read()
@mock.patch('publik_dump.publik_dump.subprocess.run')
def test_get_host_info(mocked_run, publik_dump):
mocked_run.return_value=MockedCompletedProcess(get_list_tenants())
tenants = publik_dump.get_host_info()
assert len(mocked_run.mock_calls) == 2
assert mocked_run.mock_calls[0][1][0] == \
'scp publik_dump/list_tenants.py hobo.node2.test.saas.entrouvert.org:'
assert mocked_run.mock_calls[1][1][0] == \
"ssh hobo.node2.test.saas.entrouvert.org"\
" 'sudo -u hobo HOME=$HOME hobo-manage tenant_command runscript ~/list_tenants.py --all-tenants'"
assert tenants[0] == HOST_INFO_0
assert os.path.isfile("%s/data" % publik_dump.host_folder)
# result from cache
tenants = publik_dump.get_host_info()
assert len(mocked_run.mock_calls) == 2
assert tenants[0] == HOST_INFO_0
# renew cache
publik_dump.update = True
tenants = publik_dump.get_host_info()
assert len(mocked_run.mock_calls) == 4
assert tenants[0] == HOST_INFO_0
# cron disabled error
mocked_run.return_value=MockedCompletedProcess(b'')
with pytest.raises(Exception, match=r'DISABLE_CRON_JOBS is set on this node'):
tenants = publik_dump.get_host_info()
assert not os.path.isfile("%s/data" % publik_dump.host_folder)
def test_parse_service(publik_dump):
# hobo
assert publik_dump.parse_service(HOST_INFO_0_HOBO) == TENANT_HOBO
# authentic
assert publik_dump.parse_service(HOST_INFO_0_AUTHENTIC) == TENANT_AUTHENTIC
# wcs old tenant directory
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
mocked_run.return_value = MockedCompletedProcess(get_config_pck())
assert publik_dump.parse_service(HOST_INFO_0_WCS) == TENANT_WCS
# wcs new tenant directory
with mock.patch('publik_dump.publik_dump.subprocess.run') as mocked_run:
mocked_run.side_effect = [CalledProcessError(1, ''), MockedCompletedProcess(get_config_pck())]
result = publik_dump.parse_service(HOST_INFO_0_WCS)
assert result['path'] == '/var/lib/wcs/tenants'
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(get_config_pck()))
@mock.patch('publik_dump.publik_dump.PublikDump.get_host_info', return_value=[HOST_INFO_0])
def test_get_tenant_info(mocked_get_host_info, mocked_run, publik_dump):
tenant = publik_dump.get_tenant_info()
assert len(mocked_run.mock_calls) == 1
assert tenant == TENANT_INFO
# result from cache
tenant = publik_dump.get_tenant_info()
assert len(mocked_run.mock_calls) == 1
assert tenant == TENANT_INFO
# renew cache
publik_dump.update = True
tenant = publik_dump.get_tenant_info()
assert tenant == TENANT_INFO
def test_get_dump_folder(publik_dump):
service = HOST_INFO_0_HOBO
assert publik_dump.get_dump_folder(service).strip('/').split('/')[0] == 'tmp'
assert publik_dump.get_dump_folder(service).strip('/').split('/')[-1] == service['url']
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_dump_tenant_files(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.dump_tenant_files()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh hobo.node2.test.saas.entrouvert.org '" \
"sudo tar -C /var/lib/hobo/tenants -Jcf - hobo-eurelien.test.entrouvert.org" \
"' > HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz"
# authentic
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh authentic.node2.test.saas.entrouvert.org '"\
"sudo tar -C /var/lib/authentic2-multitenant/tenants -Jcf - connexion-eurelien.test.entrouvert.org" \
"' > HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz"
# wcs
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh wcs.node2.test.saas.entrouvert.org '" \
"sudo tar -C /var/lib/wcs -Jcf - demarches-eurelien.test.entrouvert.org" \
"' > HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz"
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(TARGET.encode()))
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_restore_tenant_files(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.target = TARGET
publik_dump.restore_tenant_files()
assert len(mocked_run.mock_calls) == 1 + 9
assert mocked_run.mock_calls[0][1][0] == \
'ssh node1.test-hds.saas.entrouvert hostname -f'
# hobo
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
"cat HF/hobo-eurelien.test.entrouvert.org/hobo-eurelien.test.entrouvert.org.tar.xz"\
" | ssh hobo.node1.test-hds.saas.entrouvert"\
" 'sudo tar -C /var/lib/hobo/tenants -Jxf -'"
# authentic
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
"cat HF/connexion-eurelien.test.entrouvert.org/connexion-eurelien.test.entrouvert.org.tar.xz"\
" | ssh authentic.node1.test-hds.saas.entrouvert"\
" 'sudo tar -C /var/lib/authentic2-multitenant/tenants -Jxf -'"
# wcs
assert mocked_run.mock_calls[3][1][0].replace(publik_dump.host_folder, 'HF') == \
"cat HF/demarches-eurelien.test.entrouvert.org/demarches-eurelien.test.entrouvert.org.tar.xz"\
" | ssh wcs.node1.test-hds.saas.entrouvert"\
" 'sudo tar -C /var/lib/wcs -Jxf -'"
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_dump_tenant_databases(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.dump_tenant_databases()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh db1.test.saas.entrouvert.org"\
" 'sudo -u postgres pg_dump -n hobo_eurelien_test_entrouvert_org -Fc hobo'"\
" > HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz"
# authentic
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh db1.test.saas.entrouvert.org"\
" 'sudo -u postgres pg_dump -n connexion_eurelien_test_entrouvert_org -Fc authentic2_multitenant'"\
" > HF/connexion-eurelien.test.entrouvert.org/connexion_eurelien_test_entrouvert_org.sql.gz"
# wcs
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh db1.test.saas.entrouvert.org"\
" 'sudo -u postgres pg_dump -Fc wcs_demarches_eurelien_test_entrouvert_org'"\
" > HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz"
@mock.patch('publik_dump.publik_dump.subprocess.run', return_value=MockedCompletedProcess(DB_TARGET.encode()))
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_restore_tenant_database(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.dbtarget = DB_TARGET
publik_dump.restore_tenant_databases()
assert len(mocked_run.mock_calls) == 1 + 3 + 2*8
assert mocked_run.mock_calls[0][1][0] == \
'ssh sql3.test-hds.saas.entrouvert hostname -f'
# hobo
assert mocked_run.mock_calls[1][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh sql3.test-hds.saas.entrouvert"\
" 'sudo -u postgres psql -c \"drop schema if exists hobo_eurelien_test_entrouvert_org cascade\""\
" hobo'"
assert mocked_run.mock_calls[2][1][0].replace(publik_dump.host_folder, 'HF') == \
"cat HF/hobo-eurelien.test.entrouvert.org/hobo_eurelien_test_entrouvert_org.sql.gz"\
" | ssh sql3.test-hds.saas.entrouvert 'sudo -u postgres pg_restore -d hobo'"
# wcs
assert mocked_run.mock_calls[5][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh sql3.test-hds.saas.entrouvert"\
" 'sudo -u postgres dropdb --if-exists wcs_demarches_eurelien_test_entrouvert_org'"
assert mocked_run.mock_calls[6][1][0].replace(publik_dump.host_folder, 'HF') == \
"ssh sql3.test-hds.saas.entrouvert"\
" 'sudo -u postgres createdb wcs_demarches_eurelien_test_entrouvert_org"\
" --owner wcs --template=\"template0\" --lc-collate=fr_FR.utf8 --lc-ctype=fr_FR.utf8'"
assert mocked_run.mock_calls[7][1][0].replace(publik_dump.host_folder, 'HF') == \
"cat HF/demarches-eurelien.test.entrouvert.org/wcs_demarches_eurelien_test_entrouvert_org.sql.gz"\
" | ssh sql3.test-hds.saas.entrouvert"\
" 'sudo -u postgres pg_restore -d wcs_demarches_eurelien_test_entrouvert_org'"
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_invalidate_source_tenant(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.invalidate_source_tenant()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0] == \
'ssh hobo.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org'\
' /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org.invalid'
# authentic
assert mocked_run.mock_calls[1][1][0] == \
'ssh authentic.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org'\
' /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org.invalid'
# wcs
assert mocked_run.mock_calls[2][1][0] == \
'ssh wcs.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/wcs/demarches-eurelien.test.entrouvert.org'\
' /var/lib/wcs/demarches-eurelien.test.entrouvert.org.invalid'
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_validate_source_tenant(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.validate_source_tenant()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0] == \
'ssh hobo.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org.invalid'\
' /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org'
# authentic
assert mocked_run.mock_calls[1][1][0] == \
'ssh authentic.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org.invalid'\
' /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org'
# wcs
assert mocked_run.mock_calls[2][1][0] == \
'ssh wcs.node2.test.saas.entrouvert.org'\
' sudo mv /var/lib/wcs/demarches-eurelien.test.entrouvert.org.invalid'\
' /var/lib/wcs/demarches-eurelien.test.entrouvert.org'
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_invalidate_target_tenant(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.target = TARGET
publik_dump.invalidate_target_tenant()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0] == \
'ssh hobo.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org' \
' /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org.invalid'
# authentic
assert mocked_run.mock_calls[1][1][0] == \
'ssh authentic.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org'\
' /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org.invalid'
# wcs
assert mocked_run.mock_calls[2][1][0] == \
'ssh wcs.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/wcs/demarches-eurelien.test.entrouvert.org'\
' /var/lib/wcs/demarches-eurelien.test.entrouvert.org.invalid'
@mock.patch('publik_dump.publik_dump.subprocess.run')
@mock.patch('publik_dump.publik_dump.PublikDump.get_tenant_info', return_value=TENANT_INFO)
def test_validate_target_tenant(mocked_tenant_info, mocked_run, publik_dump):
publik_dump.target = TARGET
publik_dump.validate_target_tenant()
assert len(mocked_run.mock_calls) == 9
# hobo
assert mocked_run.mock_calls[0][1][0] == \
'ssh hobo.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org.invalid' \
' /var/lib/hobo/tenants/hobo-eurelien.test.entrouvert.org'
# authentic
assert mocked_run.mock_calls[1][1][0] == \
'ssh authentic.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org.invalid'\
' /var/lib/authentic2-multitenant/tenants/connexion-eurelien.test.entrouvert.org'
# wcs
assert mocked_run.mock_calls[2][1][0] == \
'ssh wcs.node1.test-hds.saas.entrouvert' \
' sudo mv /var/lib/wcs/demarches-eurelien.test.entrouvert.org.invalid'\
' /var/lib/wcs/demarches-eurelien.test.entrouvert.org'