entrouvert
/
combo
16
1
Fork 0
Code Issues Pull Requests 12 Releases Wiki Activity

search: handle several search engines (#23534)

This commit is contained in:
Thomas NOËL 2018-07-20 15:42:39 +02:00
parent c01cc27af2
commit d4f0d28e6e
11 changed files with 168 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
combo/apps/search/README
View File

@ -4,6 +4,8 @@ Configure search services in settings.COMBO_SEARCH_SERVICES:
'user': {
'label': 'Search a user',
'url': 'https://.../api/user/?q=%(q)s',
# 'cache_duration': 60, # in seconds, default is 0
# 'signature': True, # boolean, default is False
},
}

2
combo/apps/search/management/commands/update_index.py
View File

@ -30,7 +30,7 @@ class Command(UpdateIndexCommand):
dest='skip_external_links_collection')
def handle(self, **options):
if SearchCell.objects.filter(_search_service='_text').count() == 0:
if not any(SearchCell.get_cells_by_search_service('_text')):
# do not index site if there's no matching search cell
return
if not options.get('skip_external_links_collection', False):

22
combo/apps/search/migrations/0002_auto_20180720_1511.py Normal file
View File

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import jsonfield.fields
class Migration(migrations.Migration):
dependencies = [
('search', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='searchcell',
name='_search_services',
field=jsonfield.fields.JSONField(default=dict,
verbose_name='Search Services',
blank=True),
),
]

35
combo/apps/search/migrations/0003_create_search_services.py Normal file
View File

@ -0,0 +1,35 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
def create_search_services(apps, schema_editor):
SearchCell = apps.get_model('search', 'SearchCell')
for searchcell in SearchCell.objects.all():
if searchcell._search_service:
searchcell._search_services = {'data': [searchcell._search_service]}
else:
searchcell._search_services = {'data': []}
searchcell.save()
def back_to_search_service(apps, schema_editor):
SearchCell = apps.get_model('search', 'SearchCell')
for searchcell in SearchCell.objects.all():
if searchcell._search_services.get('data'):
searchcell._search_service = searchcell._search_services.get('data')[0]
else:
searchcell._search_service = ''
searchcell.save()
class Migration(migrations.Migration):
dependencies = [
('search', '0002_auto_20180720_1511'),
]
operations = [
migrations.RunPython(create_search_services, back_to_search_service),
]

18
combo/apps/search/migrations/0004_remove_searchcell__search_service.py Normal file
View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('search', '0003_create_search_services'),
]
operations = [
migrations.RemoveField(
model_name='searchcell',
name='_search_service',
),
]

94
combo/apps/search/models.py
View File

@ -15,44 +15,35 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django import template
from django.http import HttpResponse
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.forms import models as model_forms, Select
from django.utils.http import quote
from jsonfield import JSONField
from combo.utils import requests
from combo.data.models import CellBase
from combo.data.library import register_cell_class
from combo.utils import NothingInCacheException, get_templated_url
from combo.utils import get_templated_url
@register_cell_class
class SearchCell(CellBase):
template_name = 'combo/search-cell.html'
_search_service = models.CharField(verbose_name=_('Search Service'), max_length=64)
_search_services = JSONField(_('Search Services'), default=dict, blank=True)
class Meta:
verbose_name = _('Search')
def is_visible(self, user=None):
if not self.search_service:
if not self.search_services:
return False
return super(SearchCell, self).is_visible(user=user)
def get_default_form_class(self):
search_services = [(None, _('Not configured'))]
search_services.append(('_text', _('Page Contents')))
search_services.extend([(code, service['label'])
for code, service in settings.COMBO_SEARCH_SERVICES.items()])
widgets = {'_search_service': Select(choices=search_services)}
return model_forms.modelform_factory(self.__class__,
fields=['_search_service'],
widgets=widgets)
@property
def varname(self):
if self.slug:
@ -61,10 +52,27 @@ class SearchCell(CellBase):
return ''
@property
def search_service(self):
if self._search_service == '_text':
return {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')}
return settings.COMBO_SEARCH_SERVICES.get(self._search_service) or {}
def search_services(self):
services = []
for service_slug in self._search_services.get('data') or []:
if service_slug == '_text':
service = {'url': reverse('api-search') + '?q=%(q)s', 'label': _('Page Contents')}
else:
service = settings.COMBO_SEARCH_SERVICES.get(service_slug)
if service and service.get('url'):
service['slug'] = service_slug
services.append(service)
return services
@property
def has_multiple_search_services(self):
return len(self._search_services.get('data') or []) > 1
@classmethod
def get_cells_by_search_service(cls, search_service):
for cell in cls.objects.all():
if search_service in (cell._search_services.get('data') or []):
yield cell
def modify_global_context(self, context, request):
# if self.varname is in the query string (of the page),
@ -94,25 +102,39 @@ class SearchCell(CellBase):
return extra_context
@classmethod
def ajax_results_view(cls, request, cell_pk):
def ajax_results_view(cls, request, cell_pk, service_slug):
cell = cls.objects.get(pk=cell_pk)
if not cell.is_visible(request.user) or not cell.page.is_visible(request.user):
raise PermissionDenied
query = request.GET.get('q')
if query and cell.search_service.get('url'):
url = cell.search_service.get('url')
url = get_templated_url(url)
url = url % {'q': quote(query.encode('utf-8'))}
if url.startswith('/'):
url = request.build_absolute_uri(url)
results = requests.get(url, cache_duration=0).json()
else:
results = {'err': 0, 'data': []}
def render_response(service={}, results={'err': 0, 'data': []}):
template_names = ['combo/search-cell-results.html']
if cell.slug:
template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug)
tmpl = template.loader.select_template(template_names)
context = {'cell': cell, 'results': results, 'search_service': service}
return HttpResponse(tmpl.render(context, request), content_type='text/html')
template_names = ['combo/search-cell-results.html']
if cell.slug:
template_names.insert(0, 'combo/cells/%s/search-cell-results.html' % cell.slug)
tmpl = template.loader.select_template(template_names)
context= {'cell': cell, 'results': results}
return HttpResponse(tmpl.render(context, request), content_type='text/html')
for service in cell.search_services:
if service.get('slug') == service_slug:
break
else:
return render_response()
query = request.GET.get('q')
if not query:
return render_response(service)
url = get_templated_url(service['url'], context={'q': query, 'search_service': service})
url = url % {'q': quote(query.encode('utf-8'))} # if url contains %(q)s
if url.startswith('/'):
url = request.build_absolute_uri(url)
if not url:
return render_response(service)
kwargs = {}
kwargs['cache_duration'] = service.get('cache_duration', 0)
kwargs['remote_service'] = 'auto' if service.get('signature') else None
results = requests.get(url, **kwargs).json()
return render_response(service, results)

3
combo/apps/search/templates/combo/search-cell-results.html
View File

@ -1,3 +1,5 @@
{% if results.data %}
{% if cell.has_multiple_search_services %}<p class="search-service-label">{{ search_service.label }}</p>{% endif %}
<div class="links-list">
<ul>
{% for item in results.data %}
@ -7,3 +9,4 @@
{% endfor %}
</ul>
</div>
{% endif %}

24
combo/apps/search/templates/combo/search-cell.html
View File

@ -2,30 +2,36 @@
{% block cell-content %}
<form id="combo-search-form-{{ cell.pk }}" class="combo-search-form">
<input type="text" name="q" autocomplete="off" id="combo-search-input-{{ cell.pk }}" class="combo-search-input"
data-autocomplete-json="{% url 'combo-search-ajax-results' cell_pk=cell.pk %}{% if initial_query_string %}?{{ initial_query_string }}{% endif %}" />{# initial_query_string pass some context to ajax call #}
<input type="text" name="q" autocomplete="off" id="combo-search-input-{{ cell.pk }}" class="combo-search-input" />
<button class="submit-button">{% trans "Search" %}</button>
</form>
<div id="combo-search-results-{{ cell.pk }}" class="combo-search-results"></div>
{% for search_service in cell.search_services %}
<div id="combo-search-results-{{ cell.pk }}-{{ forloop.counter }}" class="combo-search-results combo-search-results-{{ search_service.slug }}"></div>
{% endfor %}
<script>
$(function() {
var combo_search_timeout_{{ cell.pk }};
var combo_search_form_{{ cell.pk }} = $('#combo-search-form-{{ cell.pk }}');
var combo_search_input_{{ cell.pk }} = $('#combo-search-input-{{ cell.pk }}');
var combo_search_results_{{ cell.pk }} = $('#combo-search-results-{{ cell.pk }}');
var xhr = null;
{% for search_service in cell.search_services %}
var combo_search_results_{{ cell.pk }}_{{ forloop.counter }} = $('#combo-search-results-{{ cell.pk }}-{{ forloop.counter }}');
var xhr_{{ forloop.counter }} = null;
var url_{{ forloop.counter }} = '{% url 'combo-search-ajax-results' cell_pk=cell.pk service_slug=search_service.slug %}{% if initial_query_string %}?{{ intial_query_string }}{% endif %}';
{% endfor %}
function combo_search_update_{{ cell.pk }}() {
if (xhr) xhr.abort();
xhr = $.get(combo_search_input_{{ cell.pk }}.data('autocomplete-json'),
{% for search_service in cell.search_services %}
if (xhr_{{ forloop.counter }}) xhr_{{ forloop.counter }}.abort();
xhr_{{ forloop.counter }} = $.get(url_{{ forloop.counter }},
{'q': combo_search_input_{{ cell.pk }}.val()},
function (response) {
xhr = null;
combo_search_results_{{ cell.pk }}.html(response);
xhr_{{ forloop.counter }} = null;
combo_search_results_{{ cell.pk }}_{{ forloop.counter }}.html(response);
}
);
{% endfor %}
};
combo_search_input_{{ cell.pk }}.on('paste keyup', function() {

2
combo/apps/search/urls.py
View File

@ -19,6 +19,6 @@ from django.conf.urls import url
from .models import SearchCell
urlpatterns = [
url(r'^ajax/search/(?P<cell_pk>\w+)/$', SearchCell.ajax_results_view,
url(r'^ajax/search/(?P<cell_pk>\w+)/(?P<service_slug>[\w-]+)/$', SearchCell.ajax_results_view,
name='combo-search-ajax-results'),
]

2
combo/public/views.py
View File

@ -491,7 +491,7 @@ def menu_badges(request):
def api_search(request):
for cell in SearchCell.objects.filter(_search_service='_text'):
for cell in SearchCell.get_cells_by_search_service('_text'):
if not cell.is_visible(request.user):
continue
break

24
tests/test_search.py
View File

@ -48,7 +48,7 @@ def test_search_cell(app):
page.save()
cell = SearchCell(page=page, placeholder='content', order=0)
cell._search_service = 'search1'
cell._search_services = {'data': ['search1']}
cell.save()
resp = cell.render({})
@ -66,35 +66,35 @@ def test_search_cell(app):
mock_json = mock.Mock()
mock_json.json.return_value = response
requests_get.return_value = mock_json
resp = app.get('/ajax/search/%s/?q=foo' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo'
assert '<li>' not in resp.text
resp = app.get('/ajax/search/%s/?q=foo%%23bar' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search1/?q=foo%%23bar' % cell.pk, status=200)
assert requests_get.call_args[0][0] == 'http://www.example.net/search/?q=foo%23bar'
assert '<li>' not in resp.text
response['data'] = [{'url': 'http://test', 'text': 'barbarbar'}]
resp = app.get('/ajax/search/%s/?q=foo' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
assert resp.text.count('<li>') == 1
assert '<li><a href="http://test">barbarbar</a>' in resp.text
response['data'] = [{'url': 'http://test', 'text': 'barbarbar',
'description': 'this is <b>html</b>'}]
resp = app.get('/ajax/search/%s/?q=foo' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search1/?q=foo' % cell.pk, status=200)
assert resp.text.count('<li>') == 1
assert '<li><a href="http://test">barbarbar</a>' in resp.text
assert 'this is <b>html</b>' in resp.text
with override_settings(TEMPLATE_VARS=TEMPLATE_VARS):
cell._search_service = 'search_tmpl'
cell._search_services = {'data': ['search_tmpl']}
cell.save()
with mock.patch('combo.apps.search.models.requests.get') as requests_get:
response = {'err': 0, 'data': []}
mock_json = mock.Mock()
mock_json.json.return_value = response
requests_get.return_value = mock_json
resp = app.get('/ajax/search/%s/?q=foo' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search_tmpl/?q=foo' % cell.pk, status=200)
assert requests_get.call_args[0][0] == 'http://search.example.net/?q=foo'
# TEMPLATE_VARS are accessible in template
@ -103,7 +103,7 @@ def test_search_cell(app):
templates_settings = [settings.TEMPLATES[0].copy()]
templates_settings[0]['DIRS'] = ['%s/templates-1' % os.path.abspath(os.path.dirname(__file__))]
with override_settings(TEMPLATES=templates_settings):
resp = app.get('/ajax/search/%s/?q=bar' % cell.pk, status=200)
resp = app.get('/ajax/search/%s/search_tmpl/?q=bar' % cell.pk, status=200)
assert requests_get.call_args[0][0] == 'http://search.example.net/?q=bar'
assert 'searchfoo results.data=[]' in resp.text
assert 'search_url=http://search.example.net/' in resp.text
@ -113,7 +113,7 @@ def test_search_global_context(app):
page = Page(title='Search', slug='search_page', template_name='standard')
page.save()
cell = SearchCell(page=page, placeholder='content', order=0)
cell._search_service = 'search1'
cell._search_services = {'data': ['search1']}
cell.save()
assert cell.varname == ''
@ -143,7 +143,7 @@ def test_search_cell_visibility(app):
cell = SearchCell(page=page, order=0)
assert not cell.is_visible()
cell._search_service = '_text'
cell._search_services = {'data': ['_text']}
assert cell.is_visible()
def test_search_contents():
@ -218,7 +218,7 @@ def test_search_api(app):
resp = app.get('/api/search/?q=foobar', status=404)
cell = SearchCell(page=page, _search_service='_text', order=0)
cell = SearchCell(page=page, _search_services={'data': ['_text']}, order=0)
cell.save()
resp = app.get('/api/search/?q=foobar', status=200)
@ -242,7 +242,7 @@ def test_update_index_command(app):
page = Page(title='example page', slug='example-page')
page.save()
cell = SearchCell(page=page, _search_service='_text', order=0)
cell = SearchCell(page=page, _search_services={'data': ['_text']}, order=0)
cell.save()
call_command('update_index')