wcs_api: return w.c.s. API errors (fixes #17924)
This commit is contained in:
parent
82993d5dd3
commit
de8605c48f
2
setup.py
2
setup.py
|
@ -54,7 +54,7 @@ setup(name="wcs-olap",
|
||||||
maintainer_email="bdauvergne@entrouvert.com",
|
maintainer_email="bdauvergne@entrouvert.com",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
install_requires=['requests', 'psycopg2', 'isodate', 'six'],
|
install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'],
|
||||||
entry_points={
|
entry_points={
|
||||||
'console_scripts': ['wcs-olap=wcs_olap.cmd:main'],
|
'console_scripts': ['wcs-olap=wcs_olap.cmd:main'],
|
||||||
},
|
},
|
||||||
|
|
|
@ -200,3 +200,36 @@ ALLOWED_HOSTS = ['%s']
|
||||||
yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID)
|
yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID)
|
||||||
os.kill(WCS_PID, 9)
|
os.kill(WCS_PID, 9)
|
||||||
shutil.rmtree(str(WCS_DIR))
|
shutil.rmtree(str(WCS_DIR))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def olap_cmd(wcs, tmpdir, postgres_db):
|
||||||
|
config_ini = tmpdir / 'config.ini'
|
||||||
|
model_dir = tmpdir / 'model_dir'
|
||||||
|
model_dir.mkdir()
|
||||||
|
with config_ini.open('w') as fd:
|
||||||
|
fd.write(u'''
|
||||||
|
[wcs-olap]
|
||||||
|
cubes_model_dirs = {model_dir}
|
||||||
|
pg_dsn = {dsn}
|
||||||
|
|
||||||
|
[{wcs.url}]
|
||||||
|
orig = olap
|
||||||
|
key = olap
|
||||||
|
schema = olap
|
||||||
|
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
|
||||||
|
|
||||||
|
from wcs_olap import cmd
|
||||||
|
import sys
|
||||||
|
|
||||||
|
def f(no_log_errors=True):
|
||||||
|
old_argv = sys.argv
|
||||||
|
try:
|
||||||
|
sys.argv = ['', str(config_ini)]
|
||||||
|
if no_log_errors:
|
||||||
|
sys.argv.insert(1, '--no-log-errors')
|
||||||
|
cmd.main2()
|
||||||
|
finally:
|
||||||
|
sys.argv = old_argv
|
||||||
|
f.model_dir = model_dir
|
||||||
|
return f
|
||||||
|
|
|
@ -1,28 +1,14 @@
|
||||||
import json
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import requests
|
||||||
import pathlib2
|
import pathlib2
|
||||||
|
import mock
|
||||||
|
|
||||||
|
|
||||||
def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog):
|
def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||||
config_ini = tmpdir / 'config.ini'
|
olap_cmd()
|
||||||
model_dir = tmpdir / 'model_dir'
|
|
||||||
model_dir.mkdir()
|
|
||||||
with config_ini.open('w') as fd:
|
|
||||||
fd.write(u'''
|
|
||||||
[wcs-olap]
|
|
||||||
cubes_model_dirs = {model_dir}
|
|
||||||
pg_dsn = {dsn}
|
|
||||||
|
|
||||||
[{wcs.url}]
|
|
||||||
orig = olap
|
|
||||||
key = olap
|
|
||||||
schema = olap
|
|
||||||
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
|
|
||||||
|
|
||||||
from wcs_olap import cmd
|
|
||||||
import sys
|
|
||||||
|
|
||||||
sys.argv = ['', '--no-log-errors', str(config_ini)]
|
|
||||||
cmd.main2()
|
|
||||||
|
|
||||||
expected_schema = [
|
expected_schema = [
|
||||||
('agent', 'id'),
|
('agent', 'id'),
|
||||||
|
@ -95,8 +81,35 @@ schema = olap
|
||||||
assert list(c.fetchall()) == expected_schema
|
assert list(c.fetchall()) == expected_schema
|
||||||
|
|
||||||
# verify JSON schema
|
# verify JSON schema
|
||||||
with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
|
with (olap_cmd.model_dir / 'olap.model').open() as fd, \
|
||||||
|
(pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
|
||||||
json_schema = json.load(fd)
|
json_schema = json.load(fd)
|
||||||
expected_json_schema = json.load(fd2)
|
expected_json_schema = json.load(fd2)
|
||||||
expected_json_schema['pg_dsn'] = postgres_db.dsn
|
expected_json_schema['pg_dsn'] = postgres_db.dsn
|
||||||
assert json_schema == expected_json_schema
|
assert json_schema == expected_json_schema
|
||||||
|
|
||||||
|
|
||||||
|
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||||
|
with mock.patch('requests.get', side_effect=requests.RequestException('wat!')):
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
olap_cmd(no_log_errors=False)
|
||||||
|
assert 'wat!' in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||||
|
with mock.patch('requests.get') as mocked_get:
|
||||||
|
mocked_get.return_value.ok = False
|
||||||
|
mocked_get.return_value.status_code = 401
|
||||||
|
mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}'
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
olap_cmd(no_log_errors=False)
|
||||||
|
assert 'invalid signature' in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
|
||||||
|
with mock.patch('requests.get') as mocked_get:
|
||||||
|
mocked_get.return_value.ok = True
|
||||||
|
mocked_get.return_value.json.side_effect = ValueError('invalid JSON')
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
olap_cmd(no_log_errors=False)
|
||||||
|
assert 'Invalid JSON content' in caplog.text
|
||||||
|
|
1
tox.ini
1
tox.ini
|
@ -22,6 +22,7 @@ deps =
|
||||||
psycopg2-binary
|
psycopg2-binary
|
||||||
vobject
|
vobject
|
||||||
gadjo
|
gadjo
|
||||||
|
mock
|
||||||
django>=1.11,<1.12
|
django>=1.11,<1.12
|
||||||
commands =
|
commands =
|
||||||
./get_wcs.sh
|
./get_wcs.sh
|
||||||
|
|
|
@ -8,6 +8,7 @@ import hashlib
|
||||||
from utils import Whatever
|
from utils import Whatever
|
||||||
import psycopg2
|
import psycopg2
|
||||||
|
|
||||||
|
from cached_property import cached_property
|
||||||
from wcs_olap.wcs_api import WcsApiError
|
from wcs_olap.wcs_api import WcsApiError
|
||||||
|
|
||||||
|
|
||||||
|
@ -49,12 +50,6 @@ class WcsOlapFeeder(object):
|
||||||
self.fake = fake
|
self.fake = fake
|
||||||
self.logger = logger or Whatever()
|
self.logger = logger or Whatever()
|
||||||
self.schema = schema
|
self.schema = schema
|
||||||
self.connection = psycopg2.connect(dsn=pg_dsn)
|
|
||||||
self.connection.autocommit = True
|
|
||||||
self.cur = self.connection.cursor()
|
|
||||||
self.formdefs = api.formdefs
|
|
||||||
self.roles = api.roles
|
|
||||||
self.categories = api.categories
|
|
||||||
self.do_feed = do_feed
|
self.do_feed = do_feed
|
||||||
self.ctx = Context()
|
self.ctx = Context()
|
||||||
self.ctx.push({
|
self.ctx.push({
|
||||||
|
@ -240,9 +235,29 @@ class WcsOlapFeeder(object):
|
||||||
self.base_cube = self.model['cubes'][0]
|
self.base_cube = self.model['cubes'][0]
|
||||||
self.agents_mapping = {}
|
self.agents_mapping = {}
|
||||||
self.formdata_json_index = []
|
self.formdata_json_index = []
|
||||||
self.has_jsonb = self.detect_jsonb()
|
# keep at end of __init__ to prevent leak if __init__ raises
|
||||||
if self.has_jsonb:
|
self.connection = psycopg2.connect(dsn=pg_dsn)
|
||||||
cube['json_field'] = 'json_data'
|
self.connection.autocommit = True
|
||||||
|
self.cur = self.connection.cursor()
|
||||||
|
try:
|
||||||
|
self.has_jsonb = self.detect_jsonb()
|
||||||
|
if self.has_jsonb:
|
||||||
|
cube['json_field'] = 'json_data'
|
||||||
|
except Exception:
|
||||||
|
self.connection.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def formdefs(self):
|
||||||
|
return self.api.formdefs
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def roles(self):
|
||||||
|
return self.api.roles
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def categories(self):
|
||||||
|
return self.api.categories
|
||||||
|
|
||||||
def detect_jsonb(self):
|
def detect_jsonb(self):
|
||||||
self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'")
|
self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'")
|
||||||
|
@ -437,17 +452,13 @@ CREATE TABLE public.dates AS (SELECT
|
||||||
formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed)
|
formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed)
|
||||||
formdef_feeder.feed()
|
formdef_feeder.feed()
|
||||||
except WcsApiError as e:
|
except WcsApiError as e:
|
||||||
# ignore authorization errors
|
self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e)
|
||||||
if (len(e.args) > 2 and
|
|
||||||
getattr(e.args[2], 'response', None) and
|
|
||||||
e.args[2].response.status_code == 403):
|
|
||||||
continue
|
|
||||||
self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e)
|
|
||||||
if 'cubes_model_dirs' in self.config:
|
if 'cubes_model_dirs' in self.config:
|
||||||
model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema)
|
model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema)
|
||||||
with open(model_path, 'w') as f:
|
with open(model_path, 'w') as f:
|
||||||
json.dump(self.model, f, indent=2, sort_keys=True)
|
json.dump(self.model, f, indent=2, sort_keys=True)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
# keep temporary schema alive for debugging
|
||||||
raise
|
raise
|
||||||
else:
|
else:
|
||||||
if self.do_feed:
|
if self.do_feed:
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import six
|
||||||
import requests
|
import requests
|
||||||
import urlparse
|
import urlparse
|
||||||
import urllib
|
import urllib
|
||||||
|
@ -11,8 +12,43 @@ from . import signature
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def exception_to_text(e):
|
||||||
|
try:
|
||||||
|
return six.text_type(e)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
return six.text_type(e.decode('utf8'))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
return six.text_type(repr(e))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = e.args
|
||||||
|
try:
|
||||||
|
content = six.text_type(repr(args)) if args != [] else ''
|
||||||
|
except Exception:
|
||||||
|
content = '<exception-while-rendering-args>'
|
||||||
|
except AttributeError:
|
||||||
|
content = ''
|
||||||
|
return u'%s(%s)' % (e.__class__.__name__, content)
|
||||||
|
|
||||||
|
|
||||||
class WcsApiError(Exception):
|
class WcsApiError(Exception):
|
||||||
pass
|
def __init__(self, message, **kwargs):
|
||||||
|
super(WcsApiError, self).__init__(message)
|
||||||
|
self.kwargs = kwargs
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
kwargs = self.kwargs.copy()
|
||||||
|
if 'exception' in kwargs:
|
||||||
|
kwargs['exception'] = exception_to_text(kwargs['exception'])
|
||||||
|
return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items()))
|
||||||
|
|
||||||
|
|
||||||
class BaseObject(object):
|
class BaseObject(object):
|
||||||
|
@ -205,16 +241,24 @@ class WcsApi(object):
|
||||||
signed_url = signature.sign_url(presigned_url, self.key)
|
signed_url = signature.sign_url(presigned_url, self.key)
|
||||||
try:
|
try:
|
||||||
response = requests.get(signed_url, verify=self.verify)
|
response = requests.get(signed_url, verify=self.verify)
|
||||||
response.raise_for_status()
|
except requests.RequestException as e:
|
||||||
except requests.RequestException, e:
|
raise WcsApiError('GET request failed', url=signed_url, exception=e)
|
||||||
raise WcsApiError('GET request failed', signed_url, e)
|
|
||||||
else:
|
else:
|
||||||
|
if not response.ok:
|
||||||
|
try:
|
||||||
|
text = response.text
|
||||||
|
except UnicodeError:
|
||||||
|
text = '<undecodable>' + repr(response.content)
|
||||||
|
raise WcsApiError('GET response is not 200',
|
||||||
|
url=signed_url,
|
||||||
|
status_code=response.status_code,
|
||||||
|
content=text)
|
||||||
try:
|
try:
|
||||||
content = response.json()
|
content = response.json()
|
||||||
self.cache[presigned_url] = content
|
self.cache[presigned_url] = content
|
||||||
return content
|
return content
|
||||||
except ValueError, e:
|
except ValueError as e:
|
||||||
raise WcsApiError('Invalid JSON content', signed_url, e)
|
raise WcsApiError('Invalid JSON content', url=signed_url, exception=e)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def roles(self):
|
def roles(self):
|
||||||
|
|
Loading…
Reference in New Issue