wcs-olap/wcs_olap/cmd.py

import argparse
import configparser
import contextlib
import locale
import logging
import logging.config
import os
import sys

try:
    import sentry_sdk
except ImportError:
    sentry_sdk = None
else:
    from sentry_sdk.integrations.logging import LoggingIntegration

from . import wcs_api, feeder


def main():
    try:
        main2()
    except SystemExit:
        raise


def get_config(path=None):
    config = configparser.ConfigParser()
    global_config_path = '/etc/wcs-olap/config.ini'
    user_config_path = os.path.expanduser('~/.wcs-olap.ini')
    if not path:
        paths = [global_config_path, user_config_path]
    else:
        paths = [global_config_path, path, user_config_path]
    read_files = config.read(paths)
    if config.has_section('loggers'):
        logging.config.fileConfig(paths)
        logger = logging.getLogger('wcs-olap')
        logger.info('config files: %s', ', '.join(read_files))
    return config


scopes = []


@contextlib.contextmanager
def scope():
    with contextlib.ExitStack() as stack:
        for scope in scopes:
            stack.enter_context(scope())
        yield


def configure_sentry(config):
    logger = logging.getLogger('wcs-olap')

    if not config.get('sentry', 'dsn', fallback=None):
        return
    if not sentry_sdk:
        logger.error('sentry DSN configured but sentry_sdk library is not available')
        return

    # get DEBUG level logs as breadcrumbs
    logger.level = logging.DEBUG

    sentry_logging = LoggingIntegration(
        level=logging.DEBUG,
        event_level=logging.ERROR)

    sentry_sdk.init(
        dsn=config.get('sentry', 'dsn'),
        environment=config.get('sentry', 'environment', fallback=None),
        attach_stacktrace=True,
        integrations=[sentry_logging])

    scopes.append(sentry_sdk.push_scope)


def main2():
    try:
        locale.setlocale(locale.LC_ALL, '')
    except locale.Error:
        # current locale does not exist
        pass
    parser = argparse.ArgumentParser(description='Export W.C.S. data as a star schema in a '
                                     'postgresql DB', add_help=False)
    parser.add_argument('config_path', nargs='?', default=None)
    group = parser.add_mutually_exclusive_group()
    parser.add_argument('--no-feed', dest='feed', help='only produce the model',
                        action='store_false', default=True)
    parser.add_argument('--no-log-errors', dest='no_log_errors',
                        action='store_true', default=False)
    parser.add_argument('--fake', action='store_true', default=False)
    group.add_argument("-a", "--all", help="synchronize all wcs", action='store_true',
                       default=False)
    group.add_argument('--url', help='url of the w.c.s. instance', required=False, default=None)
    args, rest = parser.parse_known_args()
    feed = args.feed
    fake = args.fake
    config = get_config(path=args.config_path)
    configure_sentry(config)
    # list all known urls
    urls = [url for url in config.sections() if url.startswith('http://')
            or url.startswith('https://')]
    defaults = {}
    if not args.all:
        try:
            url = args.url or urls[0]
        except IndexError:
            print('no url found')
            raise SystemExit(1)
        urls = [url]
        if config.has_section(args.url):
            defaults = dict(config.items(args.url))
        parser.add_argument("-h", "--help", action="help", help="show this help message and exit")
        parser.add_argument('--orig', help='origin of the request for signatures')
        parser.add_argument('--slug', action='append', default=[])
        parser.add_argument('--key', help='HMAC key for signatures')
        parser.add_argument('--pg-dsn', help='Psycopg2 DB DSN')
        parser.add_argument('--schema', help='schema name')
        args = parser.parse_args()
        for key in ('orig', 'key', 'pg_dsn', 'schema', 'batch_size'):
            if getattr(args, key, None):
                defaults[key] = getattr(args, key)

    logger = logging.getLogger('wcs-olap')
    failure = False
    for url in urls:
        if config.has_section('wcs-olap'):
            defaults.update(config.items('wcs-olap'))
        if config.has_section(url):
            defaults.update((config.items(url)))
        try:
            key = defaults['key']
            orig = defaults['orig']
            schema = defaults['schema']
            pg_dsn = defaults['pg_dsn']
            slugs = defaults.get('slugs', '').strip().split() or getattr(args, 'slug', [])
            batch_size = int(defaults.get('batch_size', 500))
        except KeyError as e:
            failure = True
            logger.error('configuration incomplete for %s: %s', url, e)
        else:
            try:
                api = wcs_api.WcsApi(url=url, orig=orig, key=key,
                                     batch_size=batch_size,
                                     verify=(defaults.get('verify', 'True') == 'True'))
                logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url,
                            pg_dsn)
                olap_feeder = feeder.WcsOlapFeeder(
                    api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,
                    config=defaults, do_feed=feed, fake=fake, slugs=slugs, scope=scope)
                olap_feeder.feed()
                logger.info('finished')
                feed_result = False
            except Exception:
                if args.no_log_errors:
                    raise
                feed_result = True
                logger.exception('failed to synchronize with %s', url)
            failure = failure or feed_result
        defaults = {}
    if failure:
        sys.exit(1)

if __name__ == '__main__':
    main()