489 lines
19 KiB
Python
489 lines
19 KiB
Python
# passerelle - uniform access to multiple data sources and services
|
|
# Copyright (C) 2020 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import json
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
from requests.exceptions import ConnectionError
|
|
|
|
import tests.utils
|
|
from passerelle.apps.opendatasoft.models import OpenDataSoft, Query
|
|
from passerelle.utils import import_site
|
|
from tests.test_manager import login
|
|
|
|
pytestmark = pytest.mark.django_db
|
|
|
|
|
|
FAKED_CONTENT_Q_SEARCH = json.dumps(
|
|
{
|
|
'nhits': 76,
|
|
'parameters': {
|
|
'dataset': 'referentiel-adresse-test',
|
|
'format': 'json',
|
|
'q': "rue de l'aubepine",
|
|
'rows': 3,
|
|
'timezone': 'UTC',
|
|
},
|
|
'records': [
|
|
{
|
|
'datasetid': 'referentiel-adresse-test',
|
|
'fields': {
|
|
'adresse_complete': "33 RUE DE L'AUBEPINE STRASBOURG",
|
|
'date_exprt': '2019-10-23',
|
|
'geo_point': [48.6060963542, 7.76978279836],
|
|
'nom_commun': 'Strasbourg',
|
|
'nom_rue': "RUE DE L'AUBEPINE",
|
|
'num_com': 482,
|
|
'numero': '33',
|
|
'source': 'Ville et Eurométropole de Strasbourg',
|
|
},
|
|
'geometry': {'coordinates': [7.76978279836, 48.6060963542], 'type': 'Point'},
|
|
'record_timestamp': '2019-12-02T14:15:08.376000+00:00',
|
|
'recordid': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
|
|
},
|
|
{
|
|
'datasetid': 'referentiel-adresse-test',
|
|
'fields': {
|
|
'adresse_complete': "19 RUE DE L'AUBEPINE LIPSHEIM",
|
|
'date_exprt': '2019-10-23',
|
|
'geo_point': [48.4920620548, 7.66177412454],
|
|
'nom_commun': 'Lipsheim',
|
|
'nom_rue': "RUE DE L'AUBEPINE",
|
|
'num_com': 268,
|
|
'numero': '19',
|
|
'source': 'Ville et Eurométropole de Strasbourg',
|
|
},
|
|
'geometry': {'coordinates': [7.66177412454, 48.4920620548], 'type': 'Point'},
|
|
'record_timestamp': '2019-12-02T14:15:08.376000+00:00',
|
|
'recordid': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
},
|
|
{
|
|
'datasetid': 'referentiel-adresse-test',
|
|
'fields': {
|
|
'adresse_complete': "29 RUE DE L'AUBEPINE STRASBOURG",
|
|
'date_exprt': '2019-10-23',
|
|
'geo_point': [48.6056497224, 7.76988497729],
|
|
'nom_commun': 'Strasbourg',
|
|
'nom_rue': "RUE DE L'AUBEPINE",
|
|
'num_com': 482,
|
|
'numero': '29',
|
|
'source': 'Ville et Eurométropole de Strasbourg',
|
|
},
|
|
'geometry': {'coordinates': [7.76988497729, 48.6056497224], 'type': 'Point'},
|
|
'record_timestamp': '2019-12-02T14:15:08.376000+00:00',
|
|
'recordid': '0984a5e1745701f71c91af73ce764e1f7132e0ff',
|
|
},
|
|
],
|
|
}
|
|
)
|
|
|
|
FAKED_CONTENT_ID_SEARCH = json.dumps(
|
|
{
|
|
'nhits': 1,
|
|
'parameters': {
|
|
'dataset': 'referentiel-adresse-test',
|
|
'format': 'json',
|
|
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
'rows': 1,
|
|
'timezone': 'UTC',
|
|
},
|
|
'records': [
|
|
{
|
|
'datasetid': 'referentiel-adresse-test',
|
|
'fields': {
|
|
'adresse_complete': "19 RUE DE L'AUBEPINE LIPSHEIM",
|
|
'date_exprt': '2019-10-23',
|
|
'geo_point': [48.4920620548, 7.66177412454],
|
|
'nom_commun': 'Lipsheim',
|
|
'nom_rue': "RUE DE L'AUBEPINE",
|
|
'num_com': 268,
|
|
'numero': '19',
|
|
'source': 'Ville et Eurométropole de Strasbourg',
|
|
},
|
|
'geometry': {'coordinates': [7.66177412454, 48.4920620548], 'type': 'Point'},
|
|
'record_timestamp': '2019-12-02T14:15:08.376000+00:00',
|
|
'recordid': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def connector():
|
|
return tests.utils.setup_access_rights(
|
|
OpenDataSoft.objects.create(
|
|
slug='my_connector',
|
|
service_url='http://www.example.net',
|
|
api_key='my_secret',
|
|
)
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def query(connector):
|
|
return Query.objects.create(
|
|
resource=connector,
|
|
name='Référenciel adresses de test',
|
|
slug='my_query',
|
|
description='Rechercher une adresse',
|
|
dataset='referentiel-adresse-test',
|
|
text_template='{{numero}} {{nom_rue}} {{nom_commun}}',
|
|
filter_expression='''
|
|
refine.source=Ville et Eurométropole de Strasbourg
|
|
exclude.numero=42
|
|
exclude.numero=43
|
|
''',
|
|
sort='-nom_rue',
|
|
limit=3,
|
|
)
|
|
|
|
|
|
def test_views(db, admin_user, app, connector):
|
|
app = login(app)
|
|
resp = app.get('/opendatasoft/my_connector/', status=200)
|
|
resp = resp.click('New Query')
|
|
resp.form['name'] = 'my query'
|
|
resp.form['slug'] = 'my-query'
|
|
resp.form['dataset'] = 'my-dataset'
|
|
resp = resp.form.submit()
|
|
resp = resp.follow()
|
|
assert resp.html.find('div', {'id': 'panel-queries'}).ul.li.a.text == 'my query'
|
|
|
|
|
|
def test_export_import(query):
|
|
assert OpenDataSoft.objects.count() == 1
|
|
assert Query.objects.count() == 1
|
|
serialization = {'resources': [query.resource.export_json()]}
|
|
OpenDataSoft.objects.all().delete()
|
|
assert OpenDataSoft.objects.count() == 0
|
|
assert Query.objects.count() == 0
|
|
import_site(serialization)
|
|
assert OpenDataSoft.objects.count() == 1
|
|
assert Query.objects.count() == 1
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_search_empty_contents(mocked_get, app, connector):
|
|
endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
|
assert endpoint == '/opendatasoft/my_connector/search'
|
|
|
|
# error returned by opendadasoft server
|
|
json_response = json.dumps({'error': "The query is invalid : Field 00 doesn't exist"})
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=json_response, status_code=200)
|
|
resp = app.get(endpoint, status=200)
|
|
assert resp.json['err']
|
|
assert resp.json['err_desc'] == "The query is invalid : Field 00 doesn't exist"
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_search(mocked_get, app, connector):
|
|
endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
|
assert endpoint == '/opendatasoft/my_connector/search'
|
|
params = {
|
|
'dataset': 'referentiel-adresse-test',
|
|
'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
|
|
'sort': '-nom_rue',
|
|
'limit': 3,
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'sort': '-nom_rue',
|
|
'rows': '3',
|
|
}
|
|
assert not resp.json['err']
|
|
assert len(resp.json['data']) == 3
|
|
# check order is kept
|
|
assert [x['id'] for x in resp.json['data']] == [
|
|
'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
|
|
'7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
'0984a5e1745701f71c91af73ce764e1f7132e0ff',
|
|
]
|
|
# check text results
|
|
assert [x['text'] for x in resp.json['data']] == [
|
|
"33 RUE DE L'AUBEPINE Strasbourg",
|
|
"19 RUE DE L'AUBEPINE Lipsheim",
|
|
"29 RUE DE L'AUBEPINE Strasbourg",
|
|
]
|
|
# check additional attributes
|
|
assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_search_using_q(mocked_get, app, connector):
|
|
endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
|
assert endpoint == '/opendatasoft/my_connector/search'
|
|
params = {
|
|
'dataset': 'referentiel-adresse-test',
|
|
'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
|
|
'sort': '-nom_rue',
|
|
'limit': '3',
|
|
'q': "rue de l'aubepine",
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'rows': '3',
|
|
'q': 'rue de aubepine',
|
|
}
|
|
assert not resp.json['err']
|
|
assert len(resp.json['data']) == 3
|
|
# check order is kept
|
|
assert [x['id'] for x in resp.json['data']] == [
|
|
'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
|
|
'7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
'0984a5e1745701f71c91af73ce764e1f7132e0ff',
|
|
]
|
|
# check text results
|
|
assert [x['text'] for x in resp.json['data']] == [
|
|
"33 RUE DE L'AUBEPINE Strasbourg",
|
|
"19 RUE DE L'AUBEPINE Lipsheim",
|
|
"29 RUE DE L'AUBEPINE Strasbourg",
|
|
]
|
|
# check additional attributes
|
|
assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
|
|
|
|
# check operators are removed
|
|
params['q'] = 'please, do NOT send boolean operators like And, OR and nOt'
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params']['q'] == 'please do send boolean operators like'
|
|
|
|
params['q'] = 'field operators are almost ignored too:, -, ==, >, <, >=, <=, [start_date TO end_date]'
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert (
|
|
mocked_get.call_args[1]['params']['q']
|
|
== 'field operators are almost ignored too start_date TO end_date'
|
|
)
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_search_using_id(mocked_get, app, connector):
|
|
endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
|
assert endpoint == '/opendatasoft/my_connector/search'
|
|
params = {
|
|
'dataset': 'referentiel-adresse-test',
|
|
'text_template': '{{numero}} {{nom_rue}} {{nom_commun}}',
|
|
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
assert len(resp.json['data']) == 1
|
|
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_query_q(mocked_get, app, query):
|
|
endpoint = '/opendatasoft/my_connector/q/my_query/'
|
|
params = {
|
|
'limit': 3,
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'refine.source': ['Ville et Eurométropole de Strasbourg'],
|
|
'exclude.numero': ['42', '43'],
|
|
'sort': '-nom_rue',
|
|
'rows': 3,
|
|
}
|
|
assert not resp.json['err']
|
|
assert len(resp.json['data']) == 3
|
|
# check order is kept
|
|
assert [x['id'] for x in resp.json['data']] == [
|
|
'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
|
|
'7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
'0984a5e1745701f71c91af73ce764e1f7132e0ff',
|
|
]
|
|
# check text results
|
|
assert [x['text'] for x in resp.json['data']] == [
|
|
"33 RUE DE L'AUBEPINE Strasbourg",
|
|
"19 RUE DE L'AUBEPINE Lipsheim",
|
|
"29 RUE DE L'AUBEPINE Strasbourg",
|
|
]
|
|
# check additional attributes
|
|
assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_query_q_using_q(mocked_get, app, query):
|
|
endpoint = '/opendatasoft/my_connector/q/my_query/'
|
|
params = {
|
|
'q': "rue de l'aubepine",
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'refine.source': ['Ville et Eurométropole de Strasbourg'],
|
|
'exclude.numero': ['42', '43'],
|
|
'rows': 3,
|
|
'q': 'rue de aubepine',
|
|
}
|
|
assert not resp.json['err']
|
|
assert len(resp.json['data']) == 3
|
|
# check order is kept
|
|
assert [x['id'] for x in resp.json['data']] == [
|
|
'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
|
|
'7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
'0984a5e1745701f71c91af73ce764e1f7132e0ff',
|
|
]
|
|
# check text results
|
|
assert [x['text'] for x in resp.json['data']] == [
|
|
"33 RUE DE L'AUBEPINE Strasbourg",
|
|
"19 RUE DE L'AUBEPINE Lipsheim",
|
|
"29 RUE DE L'AUBEPINE Strasbourg",
|
|
]
|
|
# check additional attributes
|
|
assert [x['numero'] for x in resp.json['data']] == ['33', '19', '29']
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_query_q_using_id(mocked_get, app, query):
|
|
endpoint = '/opendatasoft/my_connector/q/my_query/'
|
|
params = {
|
|
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert mocked_get.call_args[1]['params'] == {
|
|
'apikey': 'my_secret',
|
|
'dataset': 'referentiel-adresse-test',
|
|
'refine.source': ['Ville et Eurométropole de Strasbourg'],
|
|
'exclude.numero': ['42', '43'],
|
|
'rows': 3,
|
|
'q': 'recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
assert len(resp.json['data']) == 1
|
|
assert resp.json['data'][0]['text'] == "19 RUE DE L'AUBEPINE Lipsheim"
|
|
|
|
|
|
def test_opendatasoft_query_unicity(admin_user, app, connector, query):
|
|
connector2 = OpenDataSoft.objects.create(
|
|
slug='my_connector2',
|
|
api_key='my_secret',
|
|
)
|
|
Query.objects.create(
|
|
resource=connector2,
|
|
name='Foo Bar',
|
|
slug='foo-bar',
|
|
)
|
|
|
|
app = login(app)
|
|
resp = app.get('/manage/opendatasoft/%s/query/new/' % connector.slug)
|
|
resp.form['slug'] = query.slug
|
|
resp.form['name'] = 'Foo Bar'
|
|
resp.form['dataset'] = 'my-dataset'
|
|
resp = resp.form.submit()
|
|
assert resp.status_code == 200
|
|
assert Query.objects.filter(resource=connector).count() == 1
|
|
assert 'A query with this slug already exists' in resp.text
|
|
resp.form['slug'] = 'foo-bar'
|
|
resp.form['name'] = query.name
|
|
resp.form['dataset'] = 'my-dataset'
|
|
resp = resp.form.submit()
|
|
assert Query.objects.filter(resource=connector).count() == 1
|
|
assert resp.status_code == 200
|
|
assert 'A query with this name already exists' in resp.text
|
|
resp.form['slug'] = 'foo-bar'
|
|
resp.form['name'] = 'Foo Bar'
|
|
resp.form['dataset'] = 'my-dataset'
|
|
resp = resp.form.submit()
|
|
assert resp.status_code == 302
|
|
assert Query.objects.filter(resource=connector).count() == 2
|
|
new_query = Query.objects.latest('pk')
|
|
assert new_query.resource == connector
|
|
|
|
resp = app.get('/manage/opendatasoft/%s/query/%s/' % (connector.slug, new_query.pk))
|
|
resp.form['slug'] = query.slug
|
|
resp.form['name'] = 'Foo Bar'
|
|
resp = resp.form.submit()
|
|
assert resp.status_code == 200
|
|
assert 'A query with this slug already exists' in resp.text
|
|
resp.form['slug'] = 'foo-bar'
|
|
resp.form['name'] = query.name
|
|
resp = resp.form.submit()
|
|
assert resp.status_code == 200
|
|
assert 'A query with this name already exists' in resp.text
|
|
resp.form['slug'] = 'foo-bar'
|
|
resp.form['name'] = 'Foo Bar'
|
|
resp = resp.form.submit()
|
|
assert resp.status_code == 302
|
|
|
|
|
|
@mock.patch('passerelle.utils.Request.get')
|
|
def test_query_q_having_original_fields(mocked_get, app, query):
|
|
endpoint = '/opendatasoft/my_connector/q/my_query/'
|
|
params = {
|
|
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
|
|
}
|
|
content = json.loads(FAKED_CONTENT_ID_SEARCH)
|
|
content['records'][0]['fields']['id'] = 'original id'
|
|
content['records'][0]['fields']['text'] = 'original text'
|
|
query.text_template = '{{id}} - {{original_id}} - {{original_text}}'
|
|
query.save()
|
|
mocked_get.return_value = tests.utils.FakedResponse(content=json.dumps(content), status_code=200)
|
|
resp = app.get(endpoint, params=params, status=200)
|
|
assert resp.json['data'][0]['original_id'] == 'original id'
|
|
assert resp.json['data'][0]['original_text'] == 'original text'
|
|
assert (
|
|
resp.json['data'][0]['text']
|
|
== '7cafcd5c692773e8b863587b2d38d6be82e023d8 - original id - original text'
|
|
)
|
|
|
|
|
|
def test_call_search_errors(app, connector):
|
|
endpoint = tests.utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
|
|
assert endpoint == '/opendatasoft/my_connector/search'
|
|
url = connector.service_url + '/api/records/1.0/search/'
|
|
|
|
# Connection error
|
|
exception = ConnectionError('Remote end closed connection without response')
|
|
with tests.utils.mock_url(url=url, exception=exception):
|
|
resp = app.get(endpoint)
|
|
assert resp.json['err']
|
|
assert resp.json['err_desc'] == 'OpenDataSoft error: Remote end closed connection without response'
|
|
|
|
# API error, provides HTTP status code
|
|
with tests.utils.mock_url(url=url, response='{"error": "Unknown dataset: foo"}', status_code=404):
|
|
resp = app.get(endpoint)
|
|
assert resp.json['err']
|
|
assert resp.json['err_desc'] == 'Unknown dataset: foo'
|
|
|
|
# HTTP error
|
|
with tests.utils.mock_url(url=url, response='not a json content', reason='Not Found', status_code=404):
|
|
resp = app.get(endpoint)
|
|
assert resp.json['err']
|
|
assert 'OpenDataSoft error: 404 Client Error: Not Found' in resp.json['err_desc']
|
|
|
|
# bad JSON response
|
|
with tests.utils.mock_url(url=url, response='not a json content', status_code=200):
|
|
resp = app.get(endpoint)
|
|
assert resp.json['err']
|
|
assert resp.json['err_desc'] == 'OpenDataSoft error: bad JSON response'
|