wcs-olap/wcs_olap/cmd.py

import argparse
import configparser
import contextlib
import locale
import logging
import logging.config
import os
import sys

try:
    import sentry_sdk
except ImportError:
    sentry_sdk = None
else:
    from sentry_sdk.integrations.logging import LoggingIntegration

from . import feeder, wcs_api


def main():
    try:
        main2()
    except SystemExit:
        raise


def get_config(path=None):
    config = configparser.ConfigParser()
    global_config_path = '/etc/wcs-olap/config.ini'
    user_config_path = os.path.expanduser('~/.wcs-olap.ini')
    if not path:
        paths = [global_config_path, user_config_path]
    else:
        paths = [global_config_path, path, user_config_path]
    read_files = config.read(paths)
    if config.has_section('loggers'):
        logging.config.fileConfig(paths)
        logger = logging.getLogger('wcs-olap')
        logger.info('config files: %s', ', '.join(read_files))
    return config


scopes = []


@contextlib.contextmanager
def scope():
    with contextlib.ExitStack() as stack:
        for scope in scopes:
            stack.enter_context(scope())
        yield


def configure_sentry(config):
    logger = logging.getLogger('wcs-olap')

    if not config.get('sentry', 'dsn', fallback=None):
        return
    if not sentry_sdk:
        logger.error('sentry DSN configured but sentry_sdk library is not available')
        return

    # get DEBUG level logs as breadcrumbs
    logger.setLevel(logging.DEBUG)

    sentry_logging = LoggingIntegration(level=logging.DEBUG, event_level=logging.ERROR)

    sentry_sdk.init(
        dsn=config.get('sentry', 'dsn'),
        environment=config.get('sentry', 'environment', fallback=None),
        attach_stacktrace=True,
        integrations=[sentry_logging],
    )

    scopes.append(sentry_sdk.push_scope)


def main2():
    try:
        locale.setlocale(locale.LC_ALL, '')
    except locale.Error:
        # current locale does not exist
        pass
    parser = argparse.ArgumentParser(
        description='Export W.C.S. data as a star schema in a ' 'postgresql DB', add_help=False
    )
    parser.add_argument('config_path', nargs='?', default=None)
    group = parser.add_mutually_exclusive_group()
    parser.add_argument(
        '--no-feed', dest='feed', help='only produce the model', action='store_false', default=True
    )
    parser.add_argument('--no-log-errors', dest='no_log_errors', action='store_true', default=False)
    parser.add_argument('--fake', action='store_true', default=False)
    group.add_argument('-a', '--all', help='synchronize all wcs', action='store_true', default=False)
    group.add_argument('--url', help='url of the w.c.s. instance', required=False, default=None)
    args, rest = parser.parse_known_args()
    feed = args.feed
    fake = args.fake
    config = get_config(path=args.config_path)
    configure_sentry(config)
    # list all known urls
    urls = [url for url in config.sections() if url.startswith('http://') or url.startswith('https://')]
    defaults = {}
    if not args.all:
        try:
            url = args.url or urls[0]
        except IndexError:
            print('no url found')
            raise SystemExit(1)
        urls = [url]
        if config.has_section(args.url):
            defaults = dict(config.items(args.url))
        parser.add_argument('-h', '--help', action='help', help='show this help message and exit')
        parser.add_argument('--orig', help='origin of the request for signatures')
        parser.add_argument('--slug', action='append', default=[])
        parser.add_argument('--key', help='HMAC key for signatures')
        parser.add_argument('--pg-dsn', help='Psycopg2 DB DSN')
        parser.add_argument('--schema', help='schema name')
        args = parser.parse_args()
        for key in ('orig', 'key', 'pg_dsn', 'schema', 'batch_size'):
            if getattr(args, key, None):
                defaults[key] = getattr(args, key)

    logger = logging.getLogger('wcs-olap')
    failure = False
    for url in urls:
        if config.has_section('wcs-olap'):
            defaults.update(config.items('wcs-olap'))
        if config.has_section(url):
            defaults.update(config.items(url))
        try:
            key = defaults['key']
            orig = defaults['orig']
            schema = defaults['schema']
            pg_dsn = defaults['pg_dsn']
            slugs = defaults.get('slugs', '').strip().split() or getattr(args, 'slug', [])
            batch_size = int(defaults.get('batch_size', 500))
        except KeyError as e:
            failure = True
            logger.error('configuration incomplete for %s: %s', url, e)
        else:
            try:
                api = wcs_api.WcsApi(
                    url=url,
                    orig=orig,
                    key=key,
                    batch_size=batch_size,
                    verify=(defaults.get('verify', 'True') == 'True'),
                )
                logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url, pg_dsn)
                olap_feeder = feeder.WcsOlapFeeder(
                    api=api,
                    schema=schema,
                    pg_dsn=pg_dsn,
                    logger=logger,
                    config=defaults,
                    do_feed=feed,
                    fake=fake,
                    slugs=slugs,
                    scope=scope,
                )
                olap_feeder.feed()
                logger.info('finished')
                feed_result = False
            except Exception:
                if args.no_log_errors:
                    raise
                feed_result = True
                logger.exception('failed to synchronize with %s', url)
            failure = failure or feed_result
        defaults = {}
    if failure:
        sys.exit(1)


if __name__ == '__main__':
    main()