127 lines
4.7 KiB
Python
127 lines
4.7 KiB
Python
import sys
|
|
import argparse
|
|
import ConfigParser
|
|
import os
|
|
import logging
|
|
import logging.config
|
|
from . import wcs_api
|
|
from .feeder import WcsOlapFeeder
|
|
import locale
|
|
|
|
from . import tb
|
|
|
|
|
|
def main():
|
|
try:
|
|
main2()
|
|
except SystemExit:
|
|
raise
|
|
except:
|
|
tb.print_tb()
|
|
raise SystemExit(1)
|
|
|
|
|
|
def get_config(path=None):
|
|
config = ConfigParser.ConfigParser()
|
|
global_config_path = '/etc/wcs-olap/config.ini'
|
|
user_config_path = os.path.expanduser('~/.wcs-olap.ini')
|
|
if not path:
|
|
paths = [global_config_path, user_config_path]
|
|
else:
|
|
paths = [global_config_path, path, user_config_path]
|
|
read_files = config.read(paths)
|
|
if config.has_section('loggers'):
|
|
logging.config.fileConfig(paths)
|
|
logger = logging.getLogger('wcs-olap')
|
|
logger.info('config files: %s', ', '.join(read_files))
|
|
return config
|
|
|
|
|
|
def main2():
|
|
try:
|
|
locale.setlocale(locale.LC_ALL, '')
|
|
except locale.Error:
|
|
# current locale does not exist
|
|
pass
|
|
parser = argparse.ArgumentParser(description='Export W.C.S. data as a star schema in a '
|
|
'postgresql DB', add_help=False)
|
|
parser.add_argument('config_path', nargs='?', default=None)
|
|
group = parser.add_mutually_exclusive_group()
|
|
parser.add_argument('--no-feed', dest='feed', help='only produce the model',
|
|
action='store_false', default=True)
|
|
parser.add_argument('--no-log-errors', dest='no_log_errors',
|
|
action='store_true', default=False)
|
|
parser.add_argument('--fake', action='store_true', default=False)
|
|
group.add_argument("-a", "--all", help="synchronize all wcs", action='store_true',
|
|
default=False)
|
|
group.add_argument('--url', help='url of the w.c.s. instance', required=False, default=None)
|
|
args, rest = parser.parse_known_args()
|
|
feed = args.feed
|
|
fake = args.fake
|
|
config = get_config(path=args.config_path)
|
|
# list all known urls
|
|
urls = [url for url in config.sections() if url.startswith('http://') or
|
|
url.startswith('https://')]
|
|
defaults = {}
|
|
if not args.all:
|
|
try:
|
|
url = args.url or urls[0]
|
|
except IndexError:
|
|
print 'no url found'
|
|
raise SystemExit(1)
|
|
urls = [url]
|
|
if config.has_section(args.url):
|
|
defaults = dict(config.items(args.url))
|
|
parser.add_argument("-h", "--help", action="help", help="show this help message and exit")
|
|
parser.add_argument('--orig', help='origin of the request for signatures')
|
|
parser.add_argument('--slug', action='append', default=[])
|
|
parser.add_argument('--key', help='HMAC key for signatures')
|
|
parser.add_argument('--pg-dsn', help='Psycopg2 DB DSN')
|
|
parser.add_argument('--schema', help='schema name')
|
|
args = parser.parse_args()
|
|
for key in ('orig', 'key', 'pg_dsn', 'schema', 'batch_size'):
|
|
if getattr(args, key, None):
|
|
defaults[key] = getattr(args, key)
|
|
|
|
logger = logging.getLogger('wcs-olap')
|
|
failure = False
|
|
for url in urls:
|
|
if config.has_section('wcs-olap'):
|
|
defaults.update(config.items('wcs-olap'))
|
|
if config.has_section(url):
|
|
defaults.update((config.items(url)))
|
|
try:
|
|
key = defaults['key']
|
|
orig = defaults['orig']
|
|
schema = defaults['schema']
|
|
pg_dsn = defaults['pg_dsn']
|
|
slugs = defaults.get('slugs', '').strip().split() or getattr(args, 'slug', [])
|
|
batch_size = int(defaults.get('batch_size', 500))
|
|
except KeyError, e:
|
|
failure = True
|
|
logger.error('configuration incomplete for %s: %s', url, e)
|
|
else:
|
|
try:
|
|
api = wcs_api.WcsApi(url=url, orig=orig, key=key, slugs=slugs,
|
|
verify=defaults.get('verify', 'True') == 'True',
|
|
batch_size=batch_size)
|
|
logger.info('starting synchronizing w.c.s. at %r with PostgreSQL at %s', url,
|
|
pg_dsn)
|
|
feeder = WcsOlapFeeder(api=api, schema=schema, pg_dsn=pg_dsn, logger=logger,
|
|
config=defaults, do_feed=feed, fake=fake)
|
|
feeder.feed()
|
|
logger.info('finished')
|
|
feed_result = False
|
|
except:
|
|
if args.no_log_errors:
|
|
raise
|
|
feed_result = True
|
|
logger.exception('failed to synchronize with %s', url)
|
|
failure = failure or feed_result
|
|
defaults = {}
|
|
if failure:
|
|
sys.exit(1)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|