From 6d9d03b5c095133913fff93bc315f661744deb64 Mon Sep 17 00:00:00 2001 From: Matt Molyneaux Date: Thu, 26 Apr 2018 23:01:08 +0100 Subject: [PATCH 1/2] Tests to demonstrate the bug --- tests/test_watson/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_watson/tests.py b/tests/test_watson/tests.py index 8c283f0..dd82787 100644 --- a/tests/test_watson/tests.py +++ b/tests/test_watson/tests.py @@ -23,6 +23,7 @@ from django.conf import settings from django.contrib.auth.models import User from django import template from django.utils.encoding import force_text +from django.db.models import Case, When, Value, IntegerField from watson import search as watson from watson.models import SearchEntry @@ -527,6 +528,26 @@ class SearchTest(SearchTestBase): ) ).get().title, "title model1 instance11") + def testReferencingWatsonRankInAnnotations(self): + """We should be able to reference watson_rank from annotate expressions""" + entries = watson.search("model1").annotate( + relevant=Case( + When(watson_rank__gt=1.0, then=Value(1)), + default_value=Value(0), + output_field=IntegerField() + ) + ) + + # watson_rank does not return the same value across backends, so we + # can't hard code what those will be. In some cases (e.g. the regex + # backend) all ranking is hard coded to 1.0. That doesn't matter - we + # just want to make sure that Django is able to construct a valid query + for entry in entries: + if entry.watson_rank > 1.0: + self.assertTrue(entry.relevant) + else: + self.assertFalse(entry.relevant) + class LiveFilterSearchTest(SearchTest): From 4f68db052347b344e0f9b4e498388b34df8e37f2 Mon Sep 17 00:00:00 2001 From: Matt Molyneaux Date: Fri, 27 Apr 2018 10:05:35 +0100 Subject: [PATCH 2/2] Backend fixes for using `watson_rank` in expressions Referencing columns defined using `QuerySet.extra` does not make them visible to expressions such as `Value`, `F`, or `When`, and Django docs state that the `extra` method will get no further bug fixes. --- watson/backends.py | 67 +++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/watson/backends.py b/watson/backends.py index 12374a7..2efb01a 100644 --- a/watson/backends.py +++ b/watson/backends.py @@ -8,7 +8,8 @@ import re from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.db import transaction, connections, router -from django.db.models import Q +from django.db.models import Q, FloatField +from django.db.models.expressions import RawSQL, Value from django.utils.encoding import force_text from django.utils import six @@ -65,11 +66,7 @@ class SearchBackend(six.with_metaclass(abc.ABCMeta)): def do_search_ranking(self, engine_slug, queryset, search_text): """Ranks the given queryset according to the relevance of the given search text.""" - return queryset.extra( - select={ - "watson_rank": "1", - }, - ) + return queryset.annotate(watson_rank=Value(1.0, output_field=FloatField())) @abc.abstractmethod def do_search(self, engine_slug, queryset, search_text): @@ -78,11 +75,7 @@ class SearchBackend(six.with_metaclass(abc.ABCMeta)): def do_filter_ranking(self, engine_slug, queryset, search_text): """Ranks the given queryset according to the relevance of the given search text.""" - return queryset.extra( - select={ - "watson_rank": "1", - }, - ) + return queryset.annotate(watson_rank=Value(1.0, output_field=FloatField())) @abc.abstractmethod def do_filter(self, engine_slug, queryset, search_text): @@ -274,15 +267,11 @@ class PostgresSearchBackend(SearchBackend): def do_search_ranking(self, engine_slug, queryset, search_text): """Performs full text ranking.""" - return queryset.extra( - select={ - "watson_rank": "ts_rank_cd(watson_searchentry.search_tsv, to_tsquery('{search_config}', %s))".format( - search_config=self.search_config - ), - }, - select_params=(self.escape_postgres_query(search_text),), - order_by=("-watson_rank",), - ) + return queryset.annotate( + watson_rank=RawSQL("ts_rank_cd(watson_searchentry.search_tsv, to_tsquery('{config}', %s))".format( + config=self.search_config, + ), (self.escape_postgres_query(search_text),)) + ).order_by("-watson_rank") def do_filter(self, engine_slug, queryset, search_text): """Performs the full text filter.""" @@ -318,15 +307,11 @@ class PostgresSearchBackend(SearchBackend): def do_filter_ranking(self, engine_slug, queryset, search_text): """Performs the full text ranking.""" - return queryset.extra( - select={ - "watson_rank": "ts_rank_cd(watson_searchentry.search_tsv, to_tsquery('{search_config}', %s))".format( - search_config=self.search_config - ), - }, - select_params=(self.escape_postgres_query(search_text),), - order_by=("-watson_rank",), - ) + return queryset.annotate( + watson_rank=RawSQL("ts_rank_cd(watson_searchentry.search_tsv, to_tsquery('{config}', %s))".format( + config=self.search_config, + ), (self.escape_postgres_query(search_text),)) + ).order_by("-watson_rank") def do_string_cast(self, connection, column_name): return "{column_name}::text".format( @@ -411,17 +396,13 @@ class MySQLSearchBackend(SearchBackend): def do_search_ranking(self, engine_slug, queryset, search_text): """Performs full text ranking.""" search_text = self._format_query(search_text) - return queryset.extra( - select={ - "watson_rank": """ + return queryset.annotate( + watson_rank=RawSQL(""" ((MATCH (title) AGAINST (%s IN BOOLEAN MODE)) * 3) + ((MATCH (description) AGAINST (%s IN BOOLEAN MODE)) * 2) + ((MATCH (content) AGAINST (%s IN BOOLEAN MODE)) * 1) - """, - }, - select_params=(search_text, search_text, search_text,), - order_by=("-watson_rank",), - ) + """, (search_text, search_text, search_text,)) + ).order_by("-watson_rank") def do_filter(self, engine_slug, queryset, search_text): """Performs the full text filter.""" @@ -452,17 +433,13 @@ class MySQLSearchBackend(SearchBackend): def do_filter_ranking(self, engine_slug, queryset, search_text): """Performs the full text ranking.""" search_text = self._format_query(search_text) - return queryset.extra( - select={ - "watson_rank": """ + return queryset.annotate( + watson_rank=RawSQL(""" ((MATCH (watson_searchentry.title) AGAINST (%s IN BOOLEAN MODE)) * 3) + ((MATCH (watson_searchentry.description) AGAINST (%s IN BOOLEAN MODE)) * 2) + ((MATCH (watson_searchentry.content) AGAINST (%s IN BOOLEAN MODE)) * 1) - """, - }, - select_params=(search_text, search_text, search_text,), - order_by=("-watson_rank",), - ) + """, (search_text, search_text, search_text,)) + ).order_by("-watson_rank") class AdaptiveSearchBackend(SearchBackend):