opendatasoft: add opendatasoft connector (#40979)

This commit is contained in:
Nicolas Roche 2020-03-30 18:38:04 +02:00
parent bdfffaeb41
commit c09c92888d
9 changed files with 558 additions and 0 deletions

View File

View File

@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.18 on 2020-05-15 17:28
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
import passerelle.utils.templates
class Migration(migrations.Migration):
initial = True
dependencies = [
('base', '0020_auto_20200515_1923'),
]
operations = [
migrations.CreateModel(
name='OpenDataSoft',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('title', models.CharField(max_length=50, verbose_name='Title')),
('slug', models.SlugField(unique=True, verbose_name='Identifier')),
('description', models.TextField(verbose_name='Description')),
('service_url', models.CharField(help_text='OpenData Web Service URL', max_length=256, verbose_name='Service URL')),
('api_key', models.CharField(blank=True, help_text='API key used as credentials', max_length=128, verbose_name='API key')),
('users', models.ManyToManyField(blank=True, related_name='_opendatasoft_users_+', related_query_name='+', to='base.ApiUser')),
],
options={
'verbose_name': 'OpenDataSoft Web Service',
},
),
migrations.CreateModel(
name='Query',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=128, verbose_name='Name')),
('slug', models.SlugField(max_length=128, verbose_name='Slug')),
('description', models.TextField(blank=True, verbose_name='Description')),
('dataset', models.CharField(help_text='dataset to query', max_length=128, verbose_name='Dataset')),
('text_template', models.TextField(blank=True, help_text="Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}", validators=[passerelle.utils.templates.validate_template], verbose_name='Text template')),
('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queries', to='opendatasoft.OpenDataSoft', verbose_name='Resource')),
],
options={
'verbose_name': 'Query',
'ordering': ['name'],
'abstract': False,
},
),
migrations.AlterUniqueTogether(
name='query',
unique_together=set([('resource', 'slug'), ('resource', 'name')]),
),
]

View File

@ -0,0 +1,144 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2020 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.db import models
from django.shortcuts import get_object_or_404
from django.template import Context, Template
from django.core.urlresolvers import reverse
from django.utils.encoding import force_text
from django.utils.six.moves.urllib import parse as urlparse
from django.utils.translation import ugettext_lazy as _
from passerelle.utils.templates import validate_template
from passerelle.base.models import BaseResource, BaseQuery
from passerelle.utils.api import endpoint
class OpenDataSoft(BaseResource):
service_url = models.CharField(
_('Service URL'),
max_length=256, blank=False,
help_text=_('OpenDataSoft webservice URL'),
)
api_key = models.CharField(
_('API key'),
max_length=128, blank=True,
help_text=_('API key used as credentials'),
)
category = _('Data Sources')
documentation_url = 'https://doc-publik.entrouvert.com/admin-fonctionnel/parametrage-avance/connecteur-opendadasoft/'
class Meta:
verbose_name = _('OpenDataSoft Web Service')
@endpoint(
perm='can_access',
description=_('Search'),
parameters={
'dataset': {'description': _('Dataset')},
'text_template': {'description': _('Text template')},
'id': {'description': _('Record identifier')},
'q': {'description': _('Full text query')},
'limit': {'description': _('Maximum items')},
})
def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs):
scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
path = urlparse.urljoin(path, 'api/records/1.0/search/')
url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
if id is not None:
query = 'recordid:%s' % id
else:
query = q
params = {
'dataset': dataset,
'q': query,
}
if self.api_key:
params.update({'apikey': self.api_key})
if limit:
params.update({'rows': limit})
result_response = self.requests.get(url, params=params)
result = []
for record in result_response.json().get('records'):
data = {}
data['id'] = record.get('recordid')
context = {}
for key, value in record.get('fields').items():
context[key] = force_text(value)
template = Template(text_template)
data['text'] = template.render(Context(context)).strip()
result.append(data)
return {'data': result}
@endpoint(name='q',
description=_('Query'),
pattern=r'^(?P<query_slug>[\w:_-]+)/$',
perm='can_access',
show=False)
def q(self, request, query_slug, **kwargs):
query = get_object_or_404(Query, resource=self, slug=query_slug)
return query.q(request, **kwargs)
def create_query_url(self):
return reverse('opendatasoft-query-new', kwargs={'slug': self.slug})
class Query(BaseQuery):
resource = models.ForeignKey(
to=OpenDataSoft,
related_name='queries',
verbose_name=_('Resource'))
dataset = models.CharField(
_('Dataset'),
max_length=128, blank=False,
help_text=_('dataset to query'),
)
text_template = models.TextField(
verbose_name=_('Text template'),
help_text=_(
"Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}"
),
validators=[validate_template],
blank=True
)
delete_view = 'opendatasoft-query-delete'
edit_view = 'opendatasoft-query-edit'
def q(self, request, **kwargs):
return self.resource.search(
request, dataset=self.dataset, text_template=self.text_template, **kwargs)
def as_endpoint(self):
endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)
search_endpoint = self.resource.search.endpoint_info
endpoint.func = search_endpoint.func
endpoint.show_undocumented_params = False
# Copy generic params descriptions from original endpoint
# if they are not overloaded by the query
for param in search_endpoint.parameters:
if param in ('dataset', 'text_template') and getattr(self, param):
continue
endpoint.parameters[param] = search_endpoint.parameters[param]
return endpoint

View File

@ -0,0 +1,28 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2020 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf.urls import url
from . import views
management_urlpatterns = [
url(r'^(?P<slug>[\w,-]+)/query/new/$',
views.QueryNew.as_view(), name='opendatasoft-query-new'),
url(r'^(?P<slug>[\w,-]+)/query/(?P<pk>\d+)/$',
views.QueryEdit.as_view(), name='opendatasoft-query-edit'),
url(r'^(?P<slug>[\w,-]+)/query/(?P<pk>\d+)/delete/$',
views.QueryDelete.as_view(), name='opendatasoft-query-delete'),
]

View File

@ -0,0 +1,50 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2020 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django import forms
from django.views.generic import UpdateView, CreateView, DeleteView
from passerelle.base.mixins import ResourceChildViewMixin
from . import models
class QueryForm(forms.ModelForm):
class Meta:
model = models.Query
fields = '__all__'
exclude = ['resource']
class QueryNew(ResourceChildViewMixin, CreateView):
model = models.Query
form_class = QueryForm
template_name = "passerelle/manage/resource_child_form.html"
def form_valid(self, form):
form.instance.resource = self.resource
return super(QueryNew, self).form_valid(form)
class QueryEdit(ResourceChildViewMixin, UpdateView):
model = models.Query
form_class = QueryForm
template_name = "passerelle/manage/resource_child_form.html"
class QueryDelete(ResourceChildViewMixin, DeleteView):
model = models.Query
template_name = "passerelle/manage/resource_child_confirm_delete.html"

View File

@ -147,6 +147,7 @@ INSTALLED_APPS = (
'passerelle.apps.mdel_ddpacs',
'passerelle.apps.mobyt',
'passerelle.apps.okina',
'passerelle.apps.opendatasoft',
'passerelle.apps.opengis',
'passerelle.apps.orange',
'passerelle.apps.ovh',

View File

@ -181,6 +181,11 @@ li.connector.cryptor a::before {
content: "\f023"; /* lock */
}
li.connector.opendatasoft a::before {
content: "\f1c0"; /* database */
}
li.connector.status-down span.connector-name::after {
font-family: FontAwesome;
content: "\f00d"; /* times */

275
tests/test_opendatasoft.py Normal file
View File

@ -0,0 +1,275 @@
# -*- coding: utf-8 -*-
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2020 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import mock
import json
import pytest
import utils
from passerelle.apps.opendatasoft.models import OpenDataSoft, Query
from test_manager import login, admin_user
FAKED_CONTENT_Q_SEARCH = json.dumps({
"nhits": 76,
"parameters": {
"dataset": "referentiel-adresse-test",
"format": "json",
"q": "rue de l'aubepine",
"rows": 3,
"timezone": "UTC"
},
"records": [
{
"datasetid": "referentiel-adresse-test",
"fields": {
"adresse_complete": "33 RUE DE L'AUBEPINE STRASBOURG",
"date_exprt": "2019-10-23",
"geo_point": [
48.6060963542,
7.76978279836
],
"nom_commun": "Strasbourg",
"nom_rue": "RUE DE L'AUBEPINE",
"num_com": 482,
"numero": "33",
"source": u"Ville et Eurométropole de Strasbourg"
},
"geometry": {
"coordinates": [
7.76978279836,
48.6060963542
],
"type": "Point"
},
"record_timestamp": "2019-12-02T14:15:08.376000+00:00",
"recordid": "e00cf6161e52a4c8fe510b2b74d4952036cb3473"
},
{
"datasetid": "referentiel-adresse-test",
"fields": {
"adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
"date_exprt": "2019-10-23",
"geo_point": [
48.4920620548,
7.66177412454
],
"nom_commun": "Lipsheim",
"nom_rue": "RUE DE L'AUBEPINE",
"num_com": 268,
"numero": "19",
"source": u"Ville et Eurométropole de Strasbourg"
},
"geometry": {
"coordinates": [
7.66177412454,
48.4920620548
],
"type": "Point"
},
"record_timestamp": "2019-12-02T14:15:08.376000+00:00",
"recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
},
{
"datasetid": "referentiel-adresse-test",
"fields": {
"adresse_complete": "29 RUE DE L'AUBEPINE STRASBOURG",
"date_exprt": "2019-10-23",
"geo_point": [
48.6056497224,
7.76988497729
],
"nom_commun": "Strasbourg",
"nom_rue": "RUE DE L'AUBEPINE",
"num_com": 482,
"numero": "29",
"source": u"Ville et Eurométropole de Strasbourg"
},
"geometry": {
"coordinates": [
7.76988497729,
48.6056497224
],
"type": "Point"
},
"record_timestamp": "2019-12-02T14:15:08.376000+00:00",
"recordid": "0984a5e1745701f71c91af73ce764e1f7132e0ff"
}
]
})
FAKED_CONTENT_ID_SEARCH = json.dumps({
"nhits": 1,
"parameters": {
"dataset": "referentiel-adresse-test",
"format": "json",
"q": "recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8",
"rows": 1,
"timezone": "UTC"
},
"records": [
{
"datasetid": "referentiel-adresse-test",
"fields": {
"adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
"date_exprt": "2019-10-23",
"geo_point": [
48.4920620548,
7.66177412454
],
"nom_commun": "Lipsheim",
"nom_rue": "RUE DE L'AUBEPINE",
"num_com": 268,
"numero": "19",
u"source": "Ville et Eurométropole de Strasbourg"
},
"geometry": {
"coordinates": [
7.66177412454,
48.4920620548
],
"type": "Point"
},
"record_timestamp": "2019-12-02T14:15:08.376000+00:00",
"recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
}
]
})
@pytest.fixture
def connector(db):
return utils.setup_access_rights(OpenDataSoft.objects.create(
slug='my_connector',
api_key='my_secret',
))
@pytest.fixture
def query(connector):
return Query.objects.create(
resource=connector,
name='Référenciel adresses de test',
slug='my_query',
description='Rechercher une adresse',
dataset='referentiel-adresse-test',
text_template='{{numero}} {{nom_rue|safe}} {{nom_commun}}',
)
def test_views(db, admin_user, app, connector):
app = login(app)
resp = app.get('/opendatasoft/my_connector/', status=200)
resp = resp.click('New Query')
resp.form['name'] = 'my query'
resp.form['slug'] = 'my-query'
resp.form['dataset'] = 'my-dataset'
resp = resp.form.submit()
resp = resp.follow()
assert resp.html.find('div', {'id': 'queries'}).ul.li.a.text == 'my query'
@mock.patch('passerelle.utils.Request.get')
def test_search_using_q(mocked_get, app, connector):
endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
assert endpoint == '/opendatasoft/my_connector/search'
params = {
'dataset': 'referentiel-adresse-test',
'text_template': '{{numero}} {{nom_rue|safe}} {{nom_commun}}',
'q': "rue de l'aubepine",
'rows': 3,
}
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
resp = app.get(endpoint, params=params, status=200)
assert not resp.json['err']
assert len(resp.json['data']) == 3
# order is keept
assert resp.json['data'][0] == {
'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
'text': "33 RUE DE L'AUBEPINE Strasbourg"
}
assert resp.json['data'][1] == {
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
'text': "19 RUE DE L'AUBEPINE Lipsheim"
}
assert resp.json['data'][2] == {
'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',
'text': "29 RUE DE L'AUBEPINE Strasbourg"
}
@mock.patch('passerelle.utils.Request.get')
def test_search_using_id(mocked_get, app, connector):
endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
assert endpoint == '/opendatasoft/my_connector/search'
params = {
'dataset': 'referentiel-adresse-test',
'text_template': '{{numero}} {{nom_rue|safe}} {{nom_commun}}',
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
}
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
resp = app.get(endpoint, params=params, status=200)
assert resp.json == {
'err': 0,
'data': [{
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
'text': "19 RUE DE L'AUBEPINE Lipsheim"
}]}
@mock.patch('passerelle.utils.Request.get')
def test_query_q_using_q(mocked_get, app, query):
endpoint = '/opendatasoft/my_connector/q/my_query/'
params = {
'q': "rue de l'aubepine",
'rows': 3,
}
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
resp = app.get(endpoint, params=params, status=200)
assert not resp.json['err']
assert len(resp.json['data']) == 3
# order is keept
assert resp.json['data'][0] == {
'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
'text': "33 RUE DE L'AUBEPINE Strasbourg"
}
assert resp.json['data'][1] == {
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
'text': "19 RUE DE L'AUBEPINE Lipsheim"
}
assert resp.json['data'][2] == {
'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',
'text': "29 RUE DE L'AUBEPINE Strasbourg"
}
@mock.patch('passerelle.utils.Request.get')
def test_query_q_using_id(mocked_get, app, query):
endpoint = '/opendatasoft/my_connector/q/my_query/'
params = {
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
}
mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
resp = app.get(endpoint, params=params, status=200)
assert resp.json == {
'err': 0,
'data': [{
'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
'text': "19 RUE DE L'AUBEPINE Lipsheim"
}]}