diff --git a/tests/conftest.py b/tests/conftest.py index 15dc699..2c87ce2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- +import configparser import sys import time import os import shutil import random import socket -from contextlib import closing +from unittest import mock +from contextlib import closing, contextmanager, ExitStack from collections import namedtuple import psycopg2 @@ -228,6 +230,15 @@ ALLOWED_HOSTS = ['%s'] shutil.rmtree(str(wcs_dir)) +@contextmanager +def config_manager(filename): + config = configparser.ConfigParser() + config.read([filename]) + yield config + with open(filename, 'w') as fd: + config.write(fd) + + @pytest.fixture def olap_cmd(wcs, tmpdir, postgres_db): config_ini = tmpdir / 'config.ini' @@ -256,7 +267,41 @@ cubes_slug = olap-slug if no_log_errors: sys.argv.insert(1, '--no-log-errors') cmd.main2() + return 0 + except SystemExit as e: + return e.code finally: sys.argv = old_argv + + def config(): + return config_manager(str(config_ini)) + f.model_dir = model_dir + f.config = config return f + + +@pytest.fixture +def mock_cursor_execute(): + @contextmanager + def do(**execute_mock_kwargs): + import psycopg2 + + with ExitStack() as stack: + old_connect = psycopg2.connect + + def connect(*args, **kwargs): + conn = old_connect(*args, **kwargs) + mocked_conn = mock.Mock(wraps=conn) + old_cursor = conn.cursor + + def cursor(*args, **kwargs): + cur = old_cursor(*args, **kwargs) + mocked_cur = mock.Mock(wraps=cur) + mocked_cur.execute = mock.Mock(wraps=cur.execute, **execute_mock_kwargs) + return mocked_cur + mocked_conn.cursor = cursor + return mocked_conn + stack.enter_context(mock.patch.object(psycopg2, 'connect', connect)) + yield None + yield do diff --git a/tests/test_wcs.py b/tests/test_wcs.py index 5d96c98..64e3f54 100644 --- a/tests/test_wcs.py +++ b/tests/test_wcs.py @@ -199,8 +199,7 @@ def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog): raise requests.RequestException('wat!') with httmock.HTTMock(requests_raise): - with pytest.raises(SystemExit): - olap_cmd(no_log_errors=False) + assert olap_cmd(no_log_errors=False) != 0 assert 'wat!' in caplog.text @@ -210,8 +209,7 @@ def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog): return {'status_code': 401, 'content': {"err": 1, "err_desc": "invalid signature"}} with httmock.HTTMock(return_401): - with pytest.raises(SystemExit): - olap_cmd(no_log_errors=False) + assert olap_cmd(no_log_errors=False) != 0 assert 'invalid signature' in caplog.text @@ -221,8 +219,7 @@ def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog): return 'x' with httmock.HTTMock(return_invalid_json): - with pytest.raises(SystemExit): - olap_cmd(no_log_errors=False) + assert olap_cmd(no_log_errors=False) != 0 assert 'Invalid JSON content' in caplog.text diff --git a/tox.ini b/tox.ini index ead8160..d324119 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,9 @@ deps = pytest-random httmock +# wcs-olap optional dependencies + sentry_sdk<0.12.3 + # w.c.s. dependencies, as it cannot b installed through pip quixote>=3,<3.2 psycopg2-binary<2.9 diff --git a/wcs_olap/cmd.py b/wcs_olap/cmd.py index 9965891..84d0cc7 100644 --- a/wcs_olap/cmd.py +++ b/wcs_olap/cmd.py @@ -1,11 +1,19 @@ import argparse import configparser +import contextlib import locale import logging import logging.config import os import sys +try: + import sentry_sdk +except ImportError: + sentry_sdk = None +else: + from sentry_sdk.integrations.logging import LoggingIntegration + from . import wcs_api, feeder @@ -32,6 +40,42 @@ def get_config(path=None): return config +scopes = [] + + +@contextlib.contextmanager +def scope(): + with contextlib.ExitStack() as stack: + for scope in scopes: + stack.enter_context(scope()) + yield + + +def configure_sentry(config): + logger = logging.getLogger('wcs-olap') + + if not config.get('sentry', 'dsn', fallback=None): + return + if not sentry_sdk: + logger.error('sentry DSN configured but sentry_sdk library is not available') + return + + # get DEBUG level logs as breadcrumbs + logger.level = logging.DEBUG + + sentry_logging = LoggingIntegration( + level=logging.DEBUG, + event_level=logging.ERROR) + + sentry_sdk.init( + dsn=config.get('sentry', 'dsn'), + environment=config.get('sentry', 'environment', fallback=None), + attach_stacktrace=True, + integrations=[sentry_logging]) + + scopes.append(sentry_sdk.push_scope) + + def main2(): try: locale.setlocale(locale.LC_ALL, '') @@ -54,6 +98,7 @@ def main2(): feed = args.feed fake = args.fake config = get_config(path=args.config_path) + configure_sentry(config) # list all known urls urls = [url for url in config.sections() if url.startswith('http://') or url.startswith('https://')] @@ -104,7 +149,7 @@ def main2(): pg_dsn) olap_feeder = feeder.WcsOlapFeeder( api=api, schema=schema, pg_dsn=pg_dsn, logger=logger, - config=defaults, do_feed=feed, fake=fake, slugs=slugs) + config=defaults, do_feed=feed, fake=fake, slugs=slugs, scope=scope) olap_feeder.feed() logger.info('finished') feed_result = False diff --git a/wcs_olap/feeder.py b/wcs_olap/feeder.py index 9eb0a08..bd1dee4 100644 --- a/wcs_olap/feeder.py +++ b/wcs_olap/feeder.py @@ -90,10 +90,11 @@ class WcsOlapFeeder(object): status_to_id = dict((c[1], c[0]) for c in channels) id_to_status = dict((c[0], c[1]) for c in channels) - def __init__(self, api, pg_dsn, schema, logger=None, config=None, do_feed=True, fake=False, slugs=None): + def __init__(self, api, pg_dsn, schema, logger=None, config=None, do_feed=True, fake=False, slugs=None, scope=None): self.api = api self.slugs = slugs self.fake = fake + self.scope = scope or contextlib.nullcontext() self.logger = logger or Whatever() if len(schema) > 63: raise ValueError('schema name length must < 64 characters: %r' % schema) @@ -337,7 +338,7 @@ class WcsOlapFeeder(object): try: self.cur.execute(sql, vars=vars) except Exception as e: - self.logger.error('Failed to execute %r with vars %s, raised %s', sql, reprlib.repr(vars or []), e) + self.logger.warning('Failed to execute %r with vars %s, raised %s', sql, reprlib.repr(vars or []), e) raise def do_schema(self): @@ -608,12 +609,13 @@ class WcsOlapFeeder(object): self.do_dates_table() self.do_base_table() for formdef in self.formdefs: - self.api.cache = {} - try: - formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed) - formdef_feeder.feed() - except WcsApiError as e: - self.logger.error('failed to retrieve formdef %s, %s', formdef.slug, e) + with self.scope(): + self.api.cache = {} + try: + formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed) + formdef_feeder.feed() + except WcsApiError as e: + self.logger.error('failed to retrieve formdef %s, %s', formdef.slug, e) except Exception: # keep temporary schema alive for debugging raise