178 lines
5.7 KiB
Python
178 lines
5.7 KiB
Python
import argparse
|
|
import configparser
|
|
import contextlib
|
|
import locale
|
|
import logging
|
|
import logging.config
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
import sentry_sdk
|
|
except ImportError:
|
|
sentry_sdk = None
|
|
else:
|
|
from sentry_sdk.integrations.logging import LoggingIntegration
|
|
|
|
from . import feeder, wcs_api
|
|
|
|
|
|
def main():
|
|
try:
|
|
main2()
|
|
except SystemExit:
|
|
raise
|
|
|
|
|
|
def get_config(path=None):
|
|
config = configparser.ConfigParser()
|
|
global_config_path = '/etc/wcs-olap/config.ini'
|
|
user_config_path = os.path.expanduser('~/.wcs-olap.ini')
|
|
if not path:
|
|
paths = [global_config_path, user_config_path]
|
|
else:
|
|
paths = [global_config_path, path, user_config_path]
|
|
read_files = config.read(paths)
|
|
if config.has_section('loggers'):
|
|
logging.config.fileConfig(paths)
|
|
logger = logging.getLogger('wcs-olap')
|
|
logger.info('config files: %s', ', '.join(read_files))
|
|
return config
|
|
|
|
|
|
scopes = []
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def scope():
|
|
with contextlib.ExitStack() as stack:
|
|
for scope in scopes:
|
|
stack.enter_context(scope())
|
|
yield
|
|
|
|
|
|
def configure_sentry(config):
|
|
logger = logging.getLogger('wcs-olap')
|
|
|
|
if not config.get('sentry', 'dsn', fallback=None):
|
|
return
|
|
if not sentry_sdk:
|
|
logger.error('sentry DSN configured but sentry_sdk library is not available')
|
|
return
|
|
|
|
# get DEBUG level logs as breadcrumbs
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
sentry_logging = LoggingIntegration(level=logging.DEBUG, event_level=logging.ERROR)
|
|
|
|
sentry_sdk.init(
|
|
dsn=config.get('sentry', 'dsn'),
|
|
environment=config.get('sentry', 'environment', fallback=None),
|
|
attach_stacktrace=True,
|
|
integrations=[sentry_logging],
|
|
)
|
|
|
|
scopes.append(sentry_sdk.push_scope)
|
|
|
|
|
|
def main2():
|
|
try:
|
|
locale.setlocale(locale.LC_ALL, '')
|
|
except locale.Error:
|
|
# current locale does not exist
|
|
pass
|
|
parser = argparse.ArgumentParser(
|
|
description='Export W.C.S. data as a star schema in a ' 'postgresql DB', add_help=False
|
|
)
|
|
parser.add_argument('config_path', nargs='?', default=None)
|
|
group = parser.add_mutually_exclusive_group()
|
|
parser.add_argument(
|
|
'--no-feed', dest='feed', help='only produce the model', action='store_false', default=True
|
|
)
|
|
parser.add_argument('--no-log-errors', dest='no_log_errors', action='store_true', default=False)
|
|
parser.add_argument('--fake', action='store_true', default=False)
|
|
group.add_argument("-a", "--all", help="synchronize all wcs", action='store_true', default=False)
|
|
group.add_argument('--url', help='url of the w.c.s. instance', required=False, default=None)
|
|
args, rest = parser.parse_known_args()
|
|
feed = args.feed
|
|
fake = args.fake
|
|
config = get_config(path=args.config_path)
|
|
configure_sentry(config)
|
|
# list all known urls
|
|
urls = [url for url in config.sections() if url.startswith('http://') or url.startswith('https://')]
|
|
defaults = {}
|
|
if not args.all:
|
|
try:
|
|
url = args.url or urls[0]
|
|
except IndexError:
|
|
print('no url found')
|
|
raise SystemExit(1)
|
|
urls = [url]
|
|
if config.has_section(args.url):
|
|
defaults = dict(config.items(args.url))
|
|
parser.add_argument("-h", "--help", action="help", help="show this help message and exit")
|
|
parser.add_argument('--orig', help='origin of the request for signatures')
|
|
parser.add_argument('--slug', action='append', default=[])
|
|
parser.add_argument('--key', help='HMAC key for signatures')
|
|
parser.add_argument('--pg-dsn', help='Psycopg2 DB DSN')
|
|
parser.add_argument('--schema', help='schema name')
|
|
args = parser.parse_args()
|
|
for key in ('orig', 'key', 'pg_dsn', 'schema', 'batch_size'):
|
|
if getattr(args, key, None):
|
|
defaults[key] = getattr(args, key)
|
|
|
|
logger = logging.getLogger('wcs-olap')
|
|
failure = False
|
|
for url in urls:
|
|
if config.has_section('wcs-olap'):
|
|
defaults.update(config.items('wcs-olap'))
|
|
if config.has_section(url):
|
|
defaults.update(config.items(url))
|
|
try:
|
|
key = defaults['key']
|
|
orig = defaults['orig']
|
|
schema = defaults['schema']
|
|
pg_dsn = defaults['pg_dsn']
|
|
slugs = defaults.get('slugs', '').strip().split() or getattr(args, 'slug', [])
|
|
batch_size = int(defaults.get('batch_size', 500))
|
|
except KeyError as e:
|
|
failure = True
|
|
logger.error('configuration incomplete for %s: %s', url, e)
|
|
else:
|
|
try:
|
|
api = wcs_api.WcsApi(
|
|
url=url,
|
|
orig=orig,
|
|
key=key,
|
|
batch_size=batch_size,
|
|
verify=(defaults.get('verify', 'True') == 'True'),
|
|
)
|
|
logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url, pg_dsn)
|
|
olap_feeder = feeder.WcsOlapFeeder(
|
|
api=api,
|
|
schema=schema,
|
|
pg_dsn=pg_dsn,
|
|
logger=logger,
|
|
config=defaults,
|
|
do_feed=feed,
|
|
fake=fake,
|
|
slugs=slugs,
|
|
scope=scope,
|
|
)
|
|
olap_feeder.feed()
|
|
logger.info('finished')
|
|
feed_result = False
|
|
except Exception:
|
|
if args.no_log_errors:
|
|
raise
|
|
feed_result = True
|
|
logger.exception('failed to synchronize with %s', url)
|
|
failure = failure or feed_result
|
|
defaults = {}
|
|
if failure:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|