wcs_api: return w.c.s. API errors (fixes #17924)
This commit is contained in:
parent
82993d5dd3
commit
de8605c48f
2
setup.py
2
setup.py
|
@ -54,7 +54,7 @@ setup(name="wcs-olap",
|
|||
maintainer_email="bdauvergne@entrouvert.com",
|
||||
packages=find_packages(),
|
||||
include_package_data=True,
|
||||
install_requires=['requests', 'psycopg2', 'isodate', 'six'],
|
||||
install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'],
|
||||
entry_points={
|
||||
'console_scripts': ['wcs-olap=wcs_olap.cmd:main'],
|
||||
},
|
||||
|
|
|
@ -200,3 +200,36 @@ ALLOWED_HOSTS = ['%s']
|
|||
yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID)
|
||||
os.kill(WCS_PID, 9)
|
||||
shutil.rmtree(str(WCS_DIR))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def olap_cmd(wcs, tmpdir, postgres_db):
|
||||
config_ini = tmpdir / 'config.ini'
|
||||
model_dir = tmpdir / 'model_dir'
|
||||
model_dir.mkdir()
|
||||
with config_ini.open('w') as fd:
|
||||
fd.write(u'''
|
||||
[wcs-olap]
|
||||
cubes_model_dirs = {model_dir}
|
||||
pg_dsn = {dsn}
|
||||
|
||||
[{wcs.url}]
|
||||
orig = olap
|
||||
key = olap
|
||||
schema = olap
|
||||
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
|
||||
|
||||
from wcs_olap import cmd
|
||||
import sys
|
||||
|
||||
def f(no_log_errors=True):
|
||||
old_argv = sys.argv
|
||||
try:
|
||||
sys.argv = ['', str(config_ini)]
|
||||
if no_log_errors:
|
||||
sys.argv.insert(1, '--no-log-errors')
|
||||
cmd.main2()
|
||||
finally:
|
||||
sys.argv = old_argv
|
||||
f.model_dir = model_dir
|
||||
return f
|
||||
|
|
|
@ -1,28 +1,14 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import requests
|
||||
import pathlib2
|
||||
import mock
|
||||
|
||||
|
||||
def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog):
|
||||
config_ini = tmpdir / 'config.ini'
|
||||
model_dir = tmpdir / 'model_dir'
|
||||
model_dir.mkdir()
|
||||
with config_ini.open('w') as fd:
|
||||
fd.write(u'''
|
||||
[wcs-olap]
|
||||
cubes_model_dirs = {model_dir}
|
||||
pg_dsn = {dsn}
|
||||
|
||||
[{wcs.url}]
|
||||
orig = olap
|
||||
key = olap
|
||||
schema = olap
|
||||
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
|
||||
|
||||
from wcs_olap import cmd
|
||||
import sys
|
||||
|
||||
sys.argv = ['', '--no-log-errors', str(config_ini)]
|
||||
cmd.main2()
|
||||
def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||
olap_cmd()
|
||||
|
||||
expected_schema = [
|
||||
('agent', 'id'),
|
||||
|
@ -95,8 +81,35 @@ schema = olap
|
|||
assert list(c.fetchall()) == expected_schema
|
||||
|
||||
# verify JSON schema
|
||||
with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
|
||||
with (olap_cmd.model_dir / 'olap.model').open() as fd, \
|
||||
(pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
|
||||
json_schema = json.load(fd)
|
||||
expected_json_schema = json.load(fd2)
|
||||
expected_json_schema['pg_dsn'] = postgres_db.dsn
|
||||
assert json_schema == expected_json_schema
|
||||
|
||||
|
||||
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||
with mock.patch('requests.get', side_effect=requests.RequestException('wat!')):
|
||||
with pytest.raises(SystemExit):
|
||||
olap_cmd(no_log_errors=False)
|
||||
assert 'wat!' in caplog.text
|
||||
|
||||
|
||||
def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||
with mock.patch('requests.get') as mocked_get:
|
||||
mocked_get.return_value.ok = False
|
||||
mocked_get.return_value.status_code = 401
|
||||
mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}'
|
||||
with pytest.raises(SystemExit):
|
||||
olap_cmd(no_log_errors=False)
|
||||
assert 'invalid signature' in caplog.text
|
||||
|
||||
|
||||
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||
with mock.patch('requests.get') as mocked_get:
|
||||
mocked_get.return_value.ok = True
|
||||
mocked_get.return_value.json.side_effect = ValueError('invalid JSON')
|
||||
with pytest.raises(SystemExit):
|
||||
olap_cmd(no_log_errors=False)
|
||||
assert 'Invalid JSON content' in caplog.text
|
||||
|
|
1
tox.ini
1
tox.ini
|
@ -22,6 +22,7 @@ deps =
|
|||
psycopg2-binary
|
||||
vobject
|
||||
gadjo
|
||||
mock
|
||||
django>=1.11,<1.12
|
||||
commands =
|
||||
./get_wcs.sh
|
||||
|
|
|
@ -8,6 +8,7 @@ import hashlib
|
|||
from utils import Whatever
|
||||
import psycopg2
|
||||
|
||||
from cached_property import cached_property
|
||||
from wcs_olap.wcs_api import WcsApiError
|
||||
|
||||
|
||||
|
@ -49,12 +50,6 @@ class WcsOlapFeeder(object):
|
|||
self.fake = fake
|
||||
self.logger = logger or Whatever()
|
||||
self.schema = schema
|
||||
self.connection = psycopg2.connect(dsn=pg_dsn)
|
||||
self.connection.autocommit = True
|
||||
self.cur = self.connection.cursor()
|
||||
self.formdefs = api.formdefs
|
||||
self.roles = api.roles
|
||||
self.categories = api.categories
|
||||
self.do_feed = do_feed
|
||||
self.ctx = Context()
|
||||
self.ctx.push({
|
||||
|
@ -240,9 +235,29 @@ class WcsOlapFeeder(object):
|
|||
self.base_cube = self.model['cubes'][0]
|
||||
self.agents_mapping = {}
|
||||
self.formdata_json_index = []
|
||||
self.has_jsonb = self.detect_jsonb()
|
||||
if self.has_jsonb:
|
||||
cube['json_field'] = 'json_data'
|
||||
# keep at end of __init__ to prevent leak if __init__ raises
|
||||
self.connection = psycopg2.connect(dsn=pg_dsn)
|
||||
self.connection.autocommit = True
|
||||
self.cur = self.connection.cursor()
|
||||
try:
|
||||
self.has_jsonb = self.detect_jsonb()
|
||||
if self.has_jsonb:
|
||||
cube['json_field'] = 'json_data'
|
||||
except Exception:
|
||||
self.connection.close()
|
||||
raise
|
||||
|
||||
@cached_property
|
||||
def formdefs(self):
|
||||
return self.api.formdefs
|
||||
|
||||
@cached_property
|
||||
def roles(self):
|
||||
return self.api.roles
|
||||
|
||||
@cached_property
|
||||
def categories(self):
|
||||
return self.api.categories
|
||||
|
||||
def detect_jsonb(self):
|
||||
self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'")
|
||||
|
@ -437,17 +452,13 @@ CREATE TABLE public.dates AS (SELECT
|
|||
formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed)
|
||||
formdef_feeder.feed()
|
||||
except WcsApiError as e:
|
||||
# ignore authorization errors
|
||||
if (len(e.args) > 2 and
|
||||
getattr(e.args[2], 'response', None) and
|
||||
e.args[2].response.status_code == 403):
|
||||
continue
|
||||
self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e)
|
||||
self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e)
|
||||
if 'cubes_model_dirs' in self.config:
|
||||
model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema)
|
||||
with open(model_path, 'w') as f:
|
||||
json.dump(self.model, f, indent=2, sort_keys=True)
|
||||
except Exception:
|
||||
# keep temporary schema alive for debugging
|
||||
raise
|
||||
else:
|
||||
if self.do_feed:
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import six
|
||||
import requests
|
||||
import urlparse
|
||||
import urllib
|
||||
|
@ -11,8 +12,43 @@ from . import signature
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def exception_to_text(e):
|
||||
try:
|
||||
return six.text_type(e)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
return six.text_type(e.decode('utf8'))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
return six.text_type(repr(e))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
args = e.args
|
||||
try:
|
||||
content = six.text_type(repr(args)) if args != [] else ''
|
||||
except Exception:
|
||||
content = '<exception-while-rendering-args>'
|
||||
except AttributeError:
|
||||
content = ''
|
||||
return u'%s(%s)' % (e.__class__.__name__, content)
|
||||
|
||||
|
||||
class WcsApiError(Exception):
|
||||
pass
|
||||
def __init__(self, message, **kwargs):
|
||||
super(WcsApiError, self).__init__(message)
|
||||
self.kwargs = kwargs
|
||||
|
||||
def __str__(self):
|
||||
kwargs = self.kwargs.copy()
|
||||
if 'exception' in kwargs:
|
||||
kwargs['exception'] = exception_to_text(kwargs['exception'])
|
||||
return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items()))
|
||||
|
||||
|
||||
class BaseObject(object):
|
||||
|
@ -205,16 +241,24 @@ class WcsApi(object):
|
|||
signed_url = signature.sign_url(presigned_url, self.key)
|
||||
try:
|
||||
response = requests.get(signed_url, verify=self.verify)
|
||||
response.raise_for_status()
|
||||
except requests.RequestException, e:
|
||||
raise WcsApiError('GET request failed', signed_url, e)
|
||||
except requests.RequestException as e:
|
||||
raise WcsApiError('GET request failed', url=signed_url, exception=e)
|
||||
else:
|
||||
if not response.ok:
|
||||
try:
|
||||
text = response.text
|
||||
except UnicodeError:
|
||||
text = '<undecodable>' + repr(response.content)
|
||||
raise WcsApiError('GET response is not 200',
|
||||
url=signed_url,
|
||||
status_code=response.status_code,
|
||||
content=text)
|
||||
try:
|
||||
content = response.json()
|
||||
self.cache[presigned_url] = content
|
||||
return content
|
||||
except ValueError, e:
|
||||
raise WcsApiError('Invalid JSON content', signed_url, e)
|
||||
except ValueError as e:
|
||||
raise WcsApiError('Invalid JSON content', url=signed_url, exception=e)
|
||||
|
||||
@property
|
||||
def roles(self):
|
||||
|
|
Loading…
Reference in New Issue