From fb8ede1a8840bc91d9f5bf3f7852f59d05e23b9e Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Thu, 17 Jan 2019 17:46:55 +0100 Subject: [PATCH] use a Jenkinsfile (fixes #29862) --- Jenkinsfile | 35 ++++++++ get_wcs.sh | 4 + tests/conftest.py | 200 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_wcs.py | 91 +++++++++++++++++++++ tox.ini | 15 +++- wcs_olap/cmd.py | 4 + wcs_olap/feeder.py | 13 ++- 7 files changed, 355 insertions(+), 7 deletions(-) create mode 100644 Jenkinsfile create mode 100755 get_wcs.sh create mode 100644 tests/conftest.py create mode 100644 tests/test_wcs.py diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..f835c6e --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,35 @@ +@Library('eo-jenkins-lib@master') import eo.Utils + +pipeline { + agent any + stages { + stage('Unit Tests') { + steps { + sh 'tox -rv' + } + } + stage('Packaging') { + steps { + script { + if (env.JOB_NAME == 'wcs-olap' && env.GIT_BRANCH == 'origin/master') { + sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder -d jessie,stretch wcs-olap' + } + } + } + } + } + post { + always { + script { + utils = new Utils() + utils.mail_notify(currentBuild, env, 'admin+jenkins-wcs-olap@entrouvert.com') + utils.publish_coverage('coverage.xml') + utils.publish_coverage_native() + } + junit 'junit.xml' + } + success { + cleanWs() + } + } +} diff --git a/get_wcs.sh b/get_wcs.sh new file mode 100755 index 0000000..d939cfd --- /dev/null +++ b/get_wcs.sh @@ -0,0 +1,4 @@ +#!/bin/sh -xue + +test -d wcs || git clone http://git.entrouvert.org/wcs.git +(cd wcs && git pull) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c87c619 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- + +import sys +import subprocess +import time +import os +import shutil +import random +import socket +from contextlib import closing +from collections import namedtuple + +import psycopg2 + +import pytest + +Wcs = namedtuple('Wcs', ['url', 'appdir', 'pid']) + + +class Database(object): + def __init__(self): + self.db_name = 'db%s' % random.getrandbits(20) + self.dsn = 'dbname=%s' % self.db_name + with closing(psycopg2.connect('')) as conn: + conn.set_isolation_level(0) + with conn.cursor() as cursor: + cursor.execute('CREATE DATABASE %s' % self.db_name) + + def conn(self): + return closing(psycopg2.connect(self.dsn)) + + def delete(self): + with closing(psycopg2.connect('')) as conn: + conn.set_isolation_level(0) + with conn.cursor() as cursor: + cursor.execute('DROP DATABASE IF EXISTS %s' % self.db_name) + + def __repr__(self): + return '' % self.db_name + + +@pytest.fixture +def postgres_db(): + db = Database() + try: + yield db + finally: + db.delete() + + +WCS_SCRIPTS = { + 'setup-auth': u""" +from quixote import get_publisher + +get_publisher().cfg['identification'] = {'methods': ['password']} +get_publisher().cfg['debug'] = {'display_exceptions': 'text'} +get_publisher().write_cfg() +""", + 'create-user': u""" +from quixote import get_publisher +from qommon.ident.password_accounts import PasswordAccount + +user = get_publisher().user_class() +user.name = 'foo bar' +user.email = 'foo@example.net' +user.store() +account = PasswordAccount(id='user') +account.set_password('user') +account.user_id = user.id +account.store() +""", + 'create-data': u""" +import datetime +import random +from quixote import get_publisher + +from wcs.categories import Category +from wcs.formdef import FormDef +from wcs.roles import Role +from wcs import fields + +cat = Category() +cat.name = 'Catégorie' +cat.description = '' +cat.store() + +formdef = FormDef() +formdef.name = 'Demande' +formdef.category_id = cat.id +formdef.fields = [ + fields.StringField(id='1', label='1st field', type='string', anonymise=False, varname='field_string'), + fields.ItemField(id='2', label='2nd field', type='item', + items=['foo', 'bar', 'baz'], varname='field_item'), +] +formdef.store() + +user = get_publisher().user_class.select()[0] + +for i in range(50): + formdata = formdef.data_class()() + formdata.just_created() + formdata.receipt_time = datetime.datetime(2018, random.randrange(1, 13), random.randrange(1, 29)).timetuple() + formdata.data = {'1': 'FOO BAR %d' % i} + if i%4 == 0: + formdata.data['2'] = 'foo' + formdata.data['2_display'] = 'foo' + elif i%4 == 1: + formdata.data['2'] = 'bar' + formdata.data['2_display'] = 'bar' + else: + formdata.data['2'] = 'baz' + formdata.data['2_display'] = 'baz' + if i%3 == 0: + formdata.jump_status('new') + else: + formdata.jump_status('finished') + if i%7 == 0: + formdata.user_id = user.id + formdata.store() +""", +} + + +@pytest.fixture(scope='session') +def wcs(tmp_path_factory): + '''Session scoped wcs fixture, so read-only.''' + if 'WCSCTL' not in os.environ or not os.path.exists(os.environ['WCSCTL']): + pytest.skip('WCSCTL not defined in environment') + WCSCTL = os.environ.get('WCSCTL') + WCS_DIR = tmp_path_factory.mktemp('wcs') + HOSTNAME = '127.0.0.1' + PORT = 8899 + ADDRESS = '0.0.0.0' + WCS_PID = None + + def run_wcs_script(script, hostname): + '''Run python script inside w.c.s. environment''' + + script_path = WCS_DIR / (script + '.py') + with script_path.open('w') as fd: + fd.write(WCS_SCRIPTS[script]) + + subprocess.check_call( + [WCSCTL, 'runscript', '--app-dir', str(WCS_DIR), '--vhost', hostname, + str(script_path)]) + + tenant_dir = WCS_DIR / HOSTNAME + tenant_dir.mkdir() + + run_wcs_script('setup-auth', HOSTNAME) + run_wcs_script('create-user', HOSTNAME) + run_wcs_script('create-data', HOSTNAME) + + with (tenant_dir / 'site-options.cfg').open('w') as fd: + fd.write(u'''[api-secrets] +olap = olap +''') + + with (WCS_DIR / 'wcs.cfg').open('w') as fd: + fd.write(u'''[main] +app_dir = %s\n''' % WCS_DIR) + + with (WCS_DIR / 'local_settings.py').open('w') as fd: + fd.write(u''' +WCS_LEGACY_CONFIG_FILE = '%s/wcs.cfg' +THEMES_DIRECTORY = '/' +ALLOWED_HOSTS = ['%s'] +''' % (WCS_DIR, HOSTNAME)) + + # launch a Django worker for running w.c.s. + WCS_PID = os.fork() + if not WCS_PID: + os.chdir(os.path.dirname(WCSCTL)) + os.environ['DJANGO_SETTINGS_MODULE'] = 'wcs.settings' + os.environ['WCS_SETTINGS_FILE'] = str(WCS_DIR / 'local_settings.py') + os.execvp('python', ['python', 'manage.py', 'runserver', '--noreload', '%s:%s' % (ADDRESS, PORT)]) + sys.exit(0) + + # verify w.c.s. is launched + s = socket.socket() + i = 0 + while True: + i += 1 + try: + s.connect((ADDRESS, PORT)) + except Exception: + time.sleep(0.1) + else: + s.close() + break + assert i < 50, 'no connection found after 5 seconds' + + # verify w.c.s. is still running + pid, exit_code = os.waitpid(WCS_PID, os.WNOHANG) + if pid: + assert False, 'w.c.s. stopped with exit-code %s' % exit_code + + yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID) + os.kill(WCS_PID, 9) + shutil.rmtree(str(WCS_DIR)) diff --git a/tests/test_wcs.py b/tests/test_wcs.py new file mode 100644 index 0000000..17fa5d9 --- /dev/null +++ b/tests/test_wcs.py @@ -0,0 +1,91 @@ +import subprocess + + +def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog): + config_ini = tmpdir / 'config.ini' + model_dir = tmpdir / 'model_dir' + model_dir.mkdir() + with config_ini.open('w') as fd: + fd.write(u''' +[wcs-olap] +cubes_model_dirs = {model_dir} +pg_dsn = {dsn} + +[{wcs.url}] +orig = olap +key = olap +schema = olap +'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn)) + + from wcs_olap import cmd + import sys + + sys.argv = ['', '--no-log-errors', str(config_ini)] + cmd.main2() + + expected_schema = [ + ('agent', 'id'), + ('agent', 'label'), + ('category', 'id'), + ('category', 'label'), + ('channel', 'id'), + ('channel', 'label'), + ('evolution', 'id'), + ('evolution', 'generic_status_id'), + ('evolution', 'formdata_id'), + ('evolution', 'time'), + ('evolution', 'date'), + ('evolution', 'hour_id'), + ('evolution_demande', 'id'), + ('evolution_demande', 'status_id'), + ('evolution_demande', 'formdata_id'), + ('evolution_demande', 'time'), + ('evolution_demande', 'date'), + ('evolution_demande', 'hour_id'), + ('formdata', 'id'), + ('formdata', 'formdef_id'), + ('formdata', 'receipt_time'), + ('formdata', 'hour_id'), + ('formdata', 'channel_id'), + ('formdata', 'backoffice'), + ('formdata', 'generic_status_id'), + ('formdata', 'endpoint_delay'), + ('formdata', 'first_agent_id'), + ('formdata', 'geolocation_base'), + ('formdata', 'json_data'), + ('formdata_demande', 'id'), + ('formdata_demande', 'formdef_id'), + ('formdata_demande', 'receipt_time'), + ('formdata_demande', 'hour_id'), + ('formdata_demande', 'channel_id'), + ('formdata_demande', 'backoffice'), + ('formdata_demande', 'generic_status_id'), + ('formdata_demande', 'endpoint_delay'), + ('formdata_demande', 'first_agent_id'), + ('formdata_demande', 'geolocation_base'), + ('formdata_demande', 'json_data'), + ('formdata_demande', 'status_id'), + ('formdata_demande', 'field_field_item'), + ('formdata_demande', 'function__receiver'), + ('formdata_demande_field_field_item', 'id'), + ('formdata_demande_field_field_item', 'label'), + ('formdef', 'id'), + ('formdef', 'category_id'), + ('formdef', 'label'), + ('hour', 'id'), + ('hour', 'label'), + ('role', 'id'), + ('role', 'label'), + ('status', 'id'), + ('status', 'label'), + ('status_demande', 'id'), + ('status_demande', 'label') + ] + + with postgres_db.conn() as conn: + with conn.cursor() as c: + c.execute('SELECT table_name, column_name ' + 'FROM information_schema.columns ' + 'WHERE table_schema = \'olap\' ORDER BY table_name, ordinal_position') + + assert list(c.fetchall()) == expected_schema diff --git a/tox.ini b/tox.ini index 9e61007..8269fa4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,16 +4,25 @@ # and then run "tox" from this directory. [tox] -toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/publik-bi/ +toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/wcs-olap/ +envlist = coverage [testenv] usedevelop = true +basepython = python2 setenv = - coverage: COVERAGE=--junit-xml=junit.xml --cov=src --cov-report xml + coverage: COVERAGE=--junit-xml=junit.xml --cov=wcs_olap --cov-report xml --cov-report html + WCSCTL=wcs/wcsctl.py deps = coverage pytest pytest-cov pytest-random + quixote<3.0 + psycopg2-binary + vobject + gadjo + django>=1.11,<1.12 commands = - py.test {env:COVERAGE:} {posargs:--random tests} + ./get_wcs.sh + py.test {env:COVERAGE:} {posargs:--random-group tests} diff --git a/wcs_olap/cmd.py b/wcs_olap/cmd.py index 50928e8..9e6ceb8 100644 --- a/wcs_olap/cmd.py +++ b/wcs_olap/cmd.py @@ -49,6 +49,8 @@ def main2(): group = parser.add_mutually_exclusive_group() parser.add_argument('--no-feed', dest='feed', help='only produce the model', action='store_false', default=True) + parser.add_argument('--no-log-errors', dest='no_log_errors', + action='store_true', default=False) parser.add_argument('--fake', action='store_true', default=False) group.add_argument("-a", "--all", help="synchronize all wcs", action='store_true', default=False) @@ -111,6 +113,8 @@ def main2(): logger.info('finished') feed_result = False except: + if args.no_log_errors: + raise feed_result = True logger.exception('failed to synchronize with %s', url) failure = failure or feed_result diff --git a/wcs_olap/feeder.py b/wcs_olap/feeder.py index 13e83c6..1ad6179 100644 --- a/wcs_olap/feeder.py +++ b/wcs_olap/feeder.py @@ -435,17 +435,18 @@ CREATE TABLE public.dates AS (SELECT try: formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed) formdef_feeder.feed() - except WcsApiError, e: + except WcsApiError as e: # ignore authorization errors - if (len(e.args) > 2 and getattr(e.args[2], 'response', None) - and e.args[2].response.status_code == 403): + if (len(e.args) > 2 and + getattr(e.args[2], 'response', None) and + e.args[2].response.status_code == 403): continue self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e) if 'cubes_model_dirs' in self.config: model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema) with open(model_path, 'w') as f: json.dump(self.model, f, indent=2, sort_keys=True) - except: + except Exception: raise else: if self.do_feed: @@ -454,6 +455,10 @@ CREATE TABLE public.dates AS (SELECT self.ex('DROP SCHEMA IF EXISTS {schema} CASCADE') self.logger.debug('dropping schema %s to %s', self.schema + '_temp', self.schema) self.ex('ALTER SCHEMA {schema_temp} RENAME TO {schema}') + finally: + # prevent connection from remaining open + self.cur.close() + self.connection.close() def insert_agent(self, name): self.ex('INSERT INTO {agent_table} (label) VALUES (%s) RETURNING (id)', vars=[name])