entrouvert
/
wcs-olap
16
0
Fork 0
Code Issues Pull Requests Releases Activity

wcs_api: return w.c.s. API errors (fixes #17924)

This commit is contained in:
Benjamin Dauvergne 2019-01-18 14:11:59 +01:00
parent 82993d5dd3
commit de8605c48f
6 changed files with 146 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -54,7 +54,7 @@ setup(name="wcs-olap",
maintainer_email="bdauvergne@entrouvert.com",
packages=find_packages(),
include_package_data=True,
install_requires=['requests', 'psycopg2', 'isodate', 'six'],
install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'],
entry_points={
'console_scripts': ['wcs-olap=wcs_olap.cmd:main'],
},

33
tests/conftest.py
View File

@ -200,3 +200,36 @@ ALLOWED_HOSTS = ['%s']
yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID)
os.kill(WCS_PID, 9)
shutil.rmtree(str(WCS_DIR))
@pytest.fixture
def olap_cmd(wcs, tmpdir, postgres_db):
config_ini = tmpdir / 'config.ini'
model_dir = tmpdir / 'model_dir'
model_dir.mkdir()
with config_ini.open('w') as fd:
fd.write(u'''
[wcs-olap]
cubes_model_dirs = {model_dir}
pg_dsn = {dsn}
[{wcs.url}]
orig = olap
key = olap
schema = olap
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
from wcs_olap import cmd
import sys
def f(no_log_errors=True):
old_argv = sys.argv
try:
sys.argv = ['', str(config_ini)]
if no_log_errors:
sys.argv.insert(1, '--no-log-errors')
cmd.main2()
finally:
sys.argv = old_argv
f.model_dir = model_dir
return f

57
tests/test_wcs.py
View File

@ -1,28 +1,14 @@
import json
import pytest
import requests
import pathlib2
import mock
def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog):
config_ini = tmpdir / 'config.ini'
model_dir = tmpdir / 'model_dir'
model_dir.mkdir()
with config_ini.open('w') as fd:
fd.write(u'''
[wcs-olap]
cubes_model_dirs = {model_dir}
pg_dsn = {dsn}
[{wcs.url}]
orig = olap
key = olap
schema = olap
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
from wcs_olap import cmd
import sys
sys.argv = ['', '--no-log-errors', str(config_ini)]
cmd.main2()
def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
olap_cmd()
expected_schema = [
('agent', 'id'),
@ -95,8 +81,35 @@ schema = olap
assert list(c.fetchall()) == expected_schema
# verify JSON schema
with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
with (olap_cmd.model_dir / 'olap.model').open() as fd, \
(pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
json_schema = json.load(fd)
expected_json_schema = json.load(fd2)
expected_json_schema['pg_dsn'] = postgres_db.dsn
assert json_schema == expected_json_schema
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
with mock.patch('requests.get', side_effect=requests.RequestException('wat!')):
with pytest.raises(SystemExit):
olap_cmd(no_log_errors=False)
assert 'wat!' in caplog.text
def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog):
with mock.patch('requests.get') as mocked_get:
mocked_get.return_value.ok = False
mocked_get.return_value.status_code = 401
mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}'
with pytest.raises(SystemExit):
olap_cmd(no_log_errors=False)
assert 'invalid signature' in caplog.text
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
with mock.patch('requests.get') as mocked_get:
mocked_get.return_value.ok = True
mocked_get.return_value.json.side_effect = ValueError('invalid JSON')
with pytest.raises(SystemExit):
olap_cmd(no_log_errors=False)
assert 'Invalid JSON content' in caplog.text

1
tox.ini
View File

@ -22,6 +22,7 @@ deps =
psycopg2-binary
vobject
gadjo
mock
django>=1.11,<1.12
commands =
./get_wcs.sh

41
wcs_olap/feeder.py
View File

@ -8,6 +8,7 @@ import hashlib
from utils import Whatever
import psycopg2
from cached_property import cached_property
from wcs_olap.wcs_api import WcsApiError
@ -49,12 +50,6 @@ class WcsOlapFeeder(object):
self.fake = fake
self.logger = logger or Whatever()
self.schema = schema
self.connection = psycopg2.connect(dsn=pg_dsn)
self.connection.autocommit = True
self.cur = self.connection.cursor()
self.formdefs = api.formdefs
self.roles = api.roles
self.categories = api.categories
self.do_feed = do_feed
self.ctx = Context()
self.ctx.push({
@ -240,9 +235,29 @@ class WcsOlapFeeder(object):
self.base_cube = self.model['cubes'][0]
self.agents_mapping = {}
self.formdata_json_index = []
self.has_jsonb = self.detect_jsonb()
if self.has_jsonb:
cube['json_field'] = 'json_data'
# keep at end of __init__ to prevent leak if __init__ raises
self.connection = psycopg2.connect(dsn=pg_dsn)
self.connection.autocommit = True
self.cur = self.connection.cursor()
try:
self.has_jsonb = self.detect_jsonb()
if self.has_jsonb:
cube['json_field'] = 'json_data'
except Exception:
self.connection.close()
raise
@cached_property
def formdefs(self):
return self.api.formdefs
@cached_property
def roles(self):
return self.api.roles
@cached_property
def categories(self):
return self.api.categories
def detect_jsonb(self):
self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'")
@ -437,17 +452,13 @@ CREATE TABLE public.dates AS (SELECT
formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed)
formdef_feeder.feed()
except WcsApiError as e:
# ignore authorization errors
if (len(e.args) > 2 and
getattr(e.args[2], 'response', None) and
e.args[2].response.status_code == 403):
continue
self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e)
self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e)
if 'cubes_model_dirs' in self.config:
model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema)
with open(model_path, 'w') as f:
json.dump(self.model, f, indent=2, sort_keys=True)
except Exception:
# keep temporary schema alive for debugging
raise
else:
if self.do_feed:

56
wcs_olap/wcs_api.py
View File

@ -1,3 +1,4 @@
import six
import requests
import urlparse
import urllib
@ -11,8 +12,43 @@ from . import signature
logger = logging.getLogger(__name__)
def exception_to_text(e):
try:
return six.text_type(e)
except Exception:
pass
try:
return six.text_type(e.decode('utf8'))
except Exception:
pass
try:
return six.text_type(repr(e))
except Exception:
pass
try:
args = e.args
try:
content = six.text_type(repr(args)) if args != [] else ''
except Exception:
content = '<exception-while-rendering-args>'
except AttributeError:
content = ''
return u'%s(%s)' % (e.__class__.__name__, content)
class WcsApiError(Exception):
pass
def __init__(self, message, **kwargs):
super(WcsApiError, self).__init__(message)
self.kwargs = kwargs
def __str__(self):
kwargs = self.kwargs.copy()
if 'exception' in kwargs:
kwargs['exception'] = exception_to_text(kwargs['exception'])
return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items()))
class BaseObject(object):
@ -205,16 +241,24 @@ class WcsApi(object):
signed_url = signature.sign_url(presigned_url, self.key)
try:
response = requests.get(signed_url, verify=self.verify)
response.raise_for_status()
except requests.RequestException, e:
raise WcsApiError('GET request failed', signed_url, e)
except requests.RequestException as e:
raise WcsApiError('GET request failed', url=signed_url, exception=e)
else:
if not response.ok:
try:
text = response.text
except UnicodeError:
text = '<undecodable>' + repr(response.content)
raise WcsApiError('GET response is not 200',
url=signed_url,
status_code=response.status_code,
content=text)
try:
content = response.json()
self.cache[presigned_url] = content
return content
except ValueError, e:
raise WcsApiError('Invalid JSON content', signed_url, e)
except ValueError as e:
raise WcsApiError('Invalid JSON content', url=signed_url, exception=e)
@property
def roles(self):