Added flake8, fixed syntax, flattened package structure

This commit is contained in:
Rustem Sayargaliev 2016-11-07 21:02:12 +01:00
parent 03c929c571
commit fb1a3991f1
36 changed files with 301 additions and 208 deletions

View File

@ -7,14 +7,24 @@ cache:
- $HOME/.cache/pip - $HOME/.cache/pip
matrix: matrix:
fast_finish: true fast_finish: true
env:
matrix:
- WORKER=python
- WORKER=flake8
services: services:
- postgresql - postgresql
- mysql - mysql
install: install:
- pip install tox - pip install tox
- pip install flake8
before_script: before_script:
- mysql -e 'create database test_project' - |
- psql -c 'create database test_project;' -U postgres if [[ $WORKER == python ]]; then
script: tox mysql -e 'create database test_project';
psql -c 'create database test_project;' -U postgres;
fi
script:
- if [[ $WORKER == python ]]; then tox; fi
- if [[ $WORKER == flake8 ]]; then flake8 --jobs=2 . ; fi
notifications: notifications:
email: false email: false

View File

@ -1,4 +1,4 @@
include src/watson/templates/watson/*.html include watson/templates/watson/*.html
include src/watson/locale/*/LC_MESSAGES/django.* include watson/locale/*/LC_MESSAGES/django.*
include LICENSE include LICENSE
include README.markdown include README.markdown

View File

@ -3,15 +3,15 @@ from distutils.core import setup
from watson import __version__ from watson import __version__
setup( setup(
name = "django-watson", name="django-watson",
version = '.'.join(str(x) for x in __version__), version='.'.join(str(x) for x in __version__),
description = "Full-text multi-table search application for Django. Easy to install and use, with good performance.", description="Full-text multi-table search application for Django. Easy to install and use, with good performance.",
long_description = open(os.path.join(os.path.dirname(__file__), "README.markdown")).read(), long_description=open(os.path.join(os.path.dirname(__file__), "README.markdown")).read(),
author = "Dave Hall", author="Dave Hall",
author_email = "dave@etianen.com", author_email="dave@etianen.com",
url = "http://github.com/etianen/django-watson", url="http://github.com/etianen/django-watson",
zip_safe = False, zip_safe=False,
packages = [ packages=[
"watson", "watson",
"watson.management", "watson.management",
"watson.management.commands", "watson.management.commands",
@ -19,10 +19,7 @@ setup(
"watson.south_migrations", "watson.south_migrations",
"watson.templatetags", "watson.templatetags",
], ],
package_dir = { package_data={
"": "src",
},
package_data = {
"watson": [ "watson": [
"locale/*/LC_MESSAGES/django.*", "locale/*/LC_MESSAGES/django.*",
"templates/watson/*.html", "templates/watson/*.html",

View File

@ -1,7 +0,0 @@
"""
Multi-table search application for Django, using native database search engines.
Developed by Dave Hall.
<http://www.etianen.com/>
"""

View File

@ -1,5 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys, os, os.path import os
import sys
from optparse import OptionParser from optparse import OptionParser
AVAILABLE_DATABASES = { AVAILABLE_DATABASES = {
@ -97,9 +98,9 @@ def main():
'BACKEND': 'django.template.backends.django.DjangoTemplates', 'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': ['templates'], 'DIRS': ['templates'],
'APP_DIRS': True, 'APP_DIRS': True,
}], }],
) )
# Run Django setup (1.7+). # Run Django setup (1.7+).
import django import django
try: try:
@ -110,9 +111,9 @@ def main():
from django.test.utils import get_runner from django.test.utils import get_runner
TestRunner = get_runner(settings) TestRunner = get_runner(settings)
test_runner = TestRunner( test_runner = TestRunner(
verbosity = int(options.verbosity), verbosity=int(options.verbosity),
interactive = options.interactive, interactive=options.interactive,
failfast = options.failfast, failfast=options.failfast,
) )
# Run the tests. # Run the tests.
failures = test_runner.run_tests(["test_watson"]) failures = test_runner.run_tests(["test_watson"])

View File

@ -4,47 +4,46 @@ from django.utils.encoding import force_text, python_2_unicode_compatible
@python_2_unicode_compatible @python_2_unicode_compatible
class TestModelBase(models.Model): class TestModelBase(models.Model):
title = models.CharField( title = models.CharField(
max_length = 200, max_length=200,
) )
content = models.TextField( content = models.TextField(
blank = True, blank=True,
) )
description = models.TextField( description = models.TextField(
blank = True, blank=True,
) )
is_published = models.BooleanField( is_published = models.BooleanField(
default = True, default=True,
) )
def __str__(self): def __str__(self):
return force_text(self.title) return force_text(self.title)
class Meta: class Meta:
abstract = True abstract = True
class WatsonTestModel1(TestModelBase): class WatsonTestModel1(TestModelBase):
pass pass
str_pk_gen = 0; str_pk_gen = 0
def get_str_pk(): def get_str_pk():
global str_pk_gen global str_pk_gen
str_pk_gen += 1; str_pk_gen += 1
return str(str_pk_gen) return str(str_pk_gen)
class WatsonTestModel2(TestModelBase): class WatsonTestModel2(TestModelBase):
id = models.CharField( id = models.CharField(
primary_key = True, primary_key=True,
max_length = 100, max_length=100,
default = get_str_pk default=get_str_pk
) )

View File

@ -29,7 +29,7 @@ from watson.models import SearchEntry
from watson.backends import escape_query from watson.backends import escape_query
from test_watson.models import WatsonTestModel1, WatsonTestModel2 from test_watson.models import WatsonTestModel1, WatsonTestModel2
from test_watson import admin # Force early registration of all admin models. from test_watson import admin # Force early registration of all admin models. # noQA
class RegistrationTest(TestCase): class RegistrationTest(TestCase):
@ -105,28 +105,32 @@ class SearchTestBase(TestCase):
# Register the test models. # Register the test models.
watson.register(self.model1) watson.register(self.model1)
watson.register(self.model2, exclude=("id",)) watson.register(self.model2, exclude=("id",))
complex_registration_search_engine.register(WatsonTestModel1, exclude=("content", "description",), store=("is_published",)) complex_registration_search_engine.register(
complex_registration_search_engine.register(WatsonTestModel2, fields=("title",)) WatsonTestModel1, exclude=("content", "description",), store=("is_published",)
)
complex_registration_search_engine.register(
WatsonTestModel2, fields=("title",)
)
# Create some test models. # Create some test models.
self.test11 = WatsonTestModel1.objects.create( self.test11 = WatsonTestModel1.objects.create(
title = "title model1 instance11", title="title model1 instance11",
content = "content model1 instance11", content="content model1 instance11",
description = "description model1 instance11", description="description model1 instance11",
) )
self.test12 = WatsonTestModel1.objects.create( self.test12 = WatsonTestModel1.objects.create(
title = "title model1 instance12", title="title model1 instance12",
content = "content model1 instance12", content="content model1 instance12",
description = "description model1 instance12", description="description model1 instance12",
) )
self.test21 = WatsonTestModel2.objects.create( self.test21 = WatsonTestModel2.objects.create(
title = "title model2 instance21", title="title model2 instance21",
content = "content model2 instance21", content="content model2 instance21",
description = "description model2 instance21", description="description model2 instance21",
) )
self.test22 = WatsonTestModel2.objects.create( self.test22 = WatsonTestModel2.objects.create(
title = "title model2 instance22", title="title model2 instance22",
content = "content model2 instance22", content="content model2 instance22",
description = "description model2 instance22", description="description model2 instance22",
) )
def tearDown(self): def tearDown(self):
@ -366,8 +370,10 @@ class SearchTest(SearchTestBase):
self.assertEqual(watson.search("abcd@efgh").count(), 1) self.assertEqual(watson.search("abcd@efgh").count(), 1)
x.delete() x.delete()
@skipUnless(
@skipUnless(watson.get_backend().supports_prefix_matching, "Search backend does not support prefix matching.") watson.get_backend().supports_prefix_matching,
"Search backend does not support prefix matching."
)
def testMultiTablePrefixSearch(self): def testMultiTablePrefixSearch(self):
self.assertEqual(watson.search("DESCR").count(), 4) self.assertEqual(watson.search("DESCR").count(), 4)
@ -411,67 +417,83 @@ class SearchTest(SearchTestBase):
def testLimitedModelQuerySet(self): def testLimitedModelQuerySet(self):
# Test a search that should get all models. # Test a search that should get all models.
self.assertEqual(watson.search("TITLE", models=(WatsonTestModel1.objects.filter(title__icontains="TITLE"), WatsonTestModel2.objects.filter(title__icontains="TITLE"),)).count(), 4) self.assertEqual(watson.search(
"TITLE",
models=(
WatsonTestModel1.objects.filter(title__icontains="TITLE"),
WatsonTestModel2.objects.filter(title__icontains="TITLE"),
)
).count(), 4)
# Test a search that should get two models. # Test a search that should get two models.
self.assertEqual(watson.search("MODEL1", models=(WatsonTestModel1.objects.filter( self.assertEqual(
title__icontains = "MODEL1", watson.search(
description__icontains = "MODEL1", "MODEL1",
),)).count(), 2) models=(WatsonTestModel1.objects.filter(
title__icontains="MODEL1",
description__icontains="MODEL1",
),)
).count(), 2)
self.assertEqual(watson.search("MODEL2", models=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("MODEL2", models=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL2", title__icontains="MODEL2",
description__icontains = "MODEL2", description__icontains="MODEL2",
),)).count(), 2) ),)).count(), 2)
# Test a search that should get one model. # Test a search that should get one model.
self.assertEqual(watson.search("INSTANCE11", models=(WatsonTestModel1.objects.filter( self.assertEqual(watson.search("INSTANCE11", models=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL1", title__icontains="MODEL1",
),)).count(), 1) ),)).count(), 1)
self.assertEqual(watson.search("INSTANCE21", models=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("INSTANCE21", models=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL2", title__icontains="MODEL2",
),)).count(), 1) ),)).count(), 1)
# Test a search that should get no models. # Test a search that should get no models.
self.assertEqual(watson.search("INSTANCE11", models=(WatsonTestModel1.objects.filter( self.assertEqual(watson.search("INSTANCE11", models=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL2", title__icontains="MODEL2",
),)).count(), 0) ),)).count(), 0)
self.assertEqual(watson.search("INSTANCE21", models=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("INSTANCE21", models=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL1", title__icontains="MODEL1",
),)).count(), 0) ),)).count(), 0)
def testExcludedModelQuerySet(self): def testExcludedModelQuerySet(self):
# Test a search that should get all models. # Test a search that should get all models.
self.assertEqual(watson.search("TITLE", exclude=(WatsonTestModel1.objects.filter(title__icontains="FOOO"), WatsonTestModel2.objects.filter(title__icontains="FOOO"),)).count(), 4) self.assertEqual(
watson.search(
"TITLE",
exclude=(
WatsonTestModel1.objects.filter(title__icontains="FOOO"),
WatsonTestModel2.objects.filter(title__icontains="FOOO"),)
).count(), 4)
# Test a search that should get two models. # Test a search that should get two models.
self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel1.objects.filter( self.assertEqual(watson.search("MODEL1", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "INSTANCE21", title__icontains="INSTANCE21",
description__icontains = "INSTANCE22", description__icontains="INSTANCE22",
),)).count(), 2) ),)).count(), 2)
self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("MODEL2", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "INSTANCE11", title__icontains="INSTANCE11",
description__icontains = "INSTANCE12", description__icontains="INSTANCE12",
),)).count(), 2) ),)).count(), 2)
# Test a search that should get one model. # Test a search that should get one model.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter( self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL2", title__icontains="MODEL2",
),)).count(), 1) ),)).count(), 1)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL1", title__icontains="MODEL1",
),)).count(), 1) ),)).count(), 1)
# Test a search that should get no models. # Test a search that should get no models.
self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter( self.assertEqual(watson.search("INSTANCE11", exclude=(WatsonTestModel1.objects.filter(
title__icontains = "MODEL1", title__icontains="MODEL1",
),)).count(), 0) ),)).count(), 0)
self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter( self.assertEqual(watson.search("INSTANCE21", exclude=(WatsonTestModel2.objects.filter(
title__icontains = "MODEL2", title__icontains="MODEL2",
),)).count(), 0) ),)).count(), 0)
def testKitchenSink(self): def testKitchenSink(self):
"""For sanity, let's just test everything together in one giant search of doom!""" """For sanity, let's just test everything together in one giant search of doom!"""
self.assertEqual(watson.search( self.assertEqual(watson.search(
"INSTANCE11", "INSTANCE11",
models = ( models=(
WatsonTestModel1.objects.filter(title__icontains="INSTANCE11"), WatsonTestModel1.objects.filter(title__icontains="INSTANCE11"),
WatsonTestModel2.objects.filter(title__icontains="TITLE"), WatsonTestModel2.objects.filter(title__icontains="TITLE"),
), ),
exclude = ( exclude=(
WatsonTestModel1.objects.filter(title__icontains="MODEL2"), WatsonTestModel1.objects.filter(title__icontains="MODEL2"),
WatsonTestModel2.objects.filter(title__icontains="MODEL1"), WatsonTestModel2.objects.filter(title__icontains="MODEL1"),
) )
@ -500,7 +522,10 @@ class LiveFilterSearchTest(SearchTest):
self.test11.is_published = False self.test11.is_published = False
self.test11.save() self.test11.save()
# This should still return 4, since we're overriding the publication. # This should still return 4, since we're overriding the publication.
self.assertEqual(watson.search("tItle Content Description", models=(WatsonTestModel2, WatsonTestModel1._base_manager.all(),)).count(), 4) self.assertEqual(watson.search(
"tItle Content Description",
models=(WatsonTestModel2, WatsonTestModel1._base_manager.all(),)
).count(), 4)
class RankingTest(SearchTestBase): class RankingTest(SearchTestBase):
@ -522,7 +547,10 @@ class RankingTest(SearchTestBase):
self.assertRaises(AttributeError, lambda: watson.search("TITLE", ranking=False)[0].watson_rank) self.assertRaises(AttributeError, lambda: watson.search("TITLE", ranking=False)[0].watson_rank)
def testRankingParamAbsentOnFilter(self): def testRankingParamAbsentOnFilter(self):
self.assertRaises(AttributeError, lambda: watson.filter(WatsonTestModel1, "TITLE", ranking=False)[0].watson_rank) self.assertRaises(
AttributeError,
lambda: watson.filter(WatsonTestModel1, "TITLE", ranking=False)[0].watson_rank
)
@skipUnless(watson.get_backend().supports_ranking, "search backend does not support ranking") @skipUnless(watson.get_backend().supports_ranking, "search backend does not support ranking")
def testRankingWithSearch(self): def testRankingWithSearch(self):
@ -545,7 +573,10 @@ class ComplexRegistrationTest(SearchTestBase):
self.assertEqual(complex_registration_search_engine.search("instance11")[0].meta["is_published"], True) self.assertEqual(complex_registration_search_engine.search("instance11")[0].meta["is_published"], True)
def testMetaNotStored(self): def testMetaNotStored(self):
self.assertRaises(KeyError, lambda: complex_registration_search_engine.search("instance21")[0].meta["is_published"]) self.assertRaises(
KeyError,
lambda: complex_registration_search_engine.search("instance21")[0].meta["is_published"]
)
def testFieldsExcludedOnSearch(self): def testFieldsExcludedOnSearch(self):
self.assertEqual(complex_registration_search_engine.search("TITLE").count(), 4) self.assertEqual(complex_registration_search_engine.search("TITLE").count(), 4)
@ -562,13 +593,13 @@ class ComplexRegistrationTest(SearchTestBase):
class AdminIntegrationTest(SearchTestBase): class AdminIntegrationTest(SearchTestBase):
def setUp(self): def setUp(self):
super(AdminIntegrationTest, self).setUp() super(AdminIntegrationTest, self).setUp()
self.user = User( self.user = User(
username = "foo", username="foo",
is_staff = True, is_staff=True,
is_superuser = True, is_superuser=True,
) )
self.user.set_password("bar") self.user.set_password("bar")
self.user.save() self.user.save()
@ -577,8 +608,8 @@ class AdminIntegrationTest(SearchTestBase):
def testAdminIntegration(self): def testAdminIntegration(self):
# Log the user in. # Log the user in.
self.client.login( self.client.login(
username = "foo", username="foo",
password = "bar", password="bar",
) )
# Test a search with no query. # Test a search with no query.
response = self.client.get("/admin/test_watson/watsontestmodel1/") response = self.client.get("/admin/test_watson/watsontestmodel1/")

View File

@ -5,7 +5,7 @@ from django.contrib import admin
urlpatterns = [ urlpatterns = [
url("^simple/", include("watson.urls")), url("^simple/", include("watson.urls")),
url("^custom/", include("watson.urls"), kwargs={ url("^custom/", include("watson.urls"), kwargs={
"query_param": "fooo", "query_param": "fooo",
"empty_query_redirect": "/simple/", "empty_query_redirect": "/simple/",
@ -15,6 +15,6 @@ urlpatterns = [
}, },
"paginate_by": 10, "paginate_by": 10,
}), }),
url("^admin/", include(admin.site.urls)), url("^admin/", include(admin.site.urls)),
] ]

10
tox.ini
View File

@ -12,6 +12,10 @@ deps =
postgres: psycopg2 postgres: psycopg2
mysql: mysqlclient mysql: mysqlclient
commands = commands =
sqlite: coverage run src/tests/runtests.py sqlite: coverage run tests/runtests.py
postgres: coverage run src/tests/runtests.py -d psql postgres: coverage run tests/runtests.py -d psql
mysql: coverage run src/tests/runtests.py -d mysql mysql: coverage run tests/runtests.py -d mysql
[flake8]
max-line-length=120
exclude=build,venv,migrations,south_migrations,.tox

View File

@ -68,9 +68,9 @@ class SearchAdmin(admin.ModelAdmin):
if not self.search_engine.is_registered(self.model) and self.search_fields: if not self.search_engine.is_registered(self.model) and self.search_fields:
self.search_engine.register( self.search_engine.register(
self.model, self.model,
fields = self.search_fields, fields=self.search_fields,
adapter_cls = self.search_adapter_cls, adapter_cls=self.search_adapter_cls,
get_live_queryset = lambda self_: None, # Ensure complete queryset is used in admin. get_live_queryset=lambda self_: None, # Ensure complete queryset is used in admin.
) )
def get_changelist(self, request, **kwargs): def get_changelist(self, request, **kwargs):

View File

@ -118,7 +118,7 @@ class RegexSearchMixin(six.with_metaclass(abc.ABCMeta)):
""", """ """, """
({db_table}.{content_type_id} = %s) ({db_table}.{content_type_id} = %s)
"""] """]
word_kwargs= { word_kwargs = {
"db_table": db_table, "db_table": db_table,
"model_db_table": model_db_table, "model_db_table": model_db_table,
"engine_slug": connection.ops.quote_name("engine_slug"), "engine_slug": connection.ops.quote_name("engine_slug"),
@ -147,9 +147,13 @@ class RegexSearchMixin(six.with_metaclass(abc.ABCMeta)):
# Add in all words. # Add in all words.
for word in search_text.split(): for word in search_text.split():
regex = regex_from_word(word) regex = regex_from_word(word)
word_query.append(""" word_query.append(
({db_table}.{title} {iregex_operator} OR {db_table}.{description} {iregex_operator} OR {db_table}.{content} {iregex_operator}) """
""") ({db_table}.{title} {iregex_operator}
OR {db_table}.{description} {iregex_operator}
OR {db_table}.{content} {iregex_operator})
"""
)
word_args.extend((regex, regex, regex)) word_args.extend((regex, regex, regex))
# Compile the query. # Compile the query.
full_word_query = " AND ".join(word_query).format(**word_kwargs) full_word_query = " AND ".join(word_query).format(**word_kwargs)
@ -354,25 +358,36 @@ class MySQLSearchBackend(SearchBackend):
def is_installed(self): def is_installed(self):
"""Checks whether django-watson is installed.""" """Checks whether django-watson is installed."""
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute("SHOW INDEX FROM watson_searchentry WHERE Key_name = 'watson_searchentry_fulltext'"); cursor.execute("SHOW INDEX FROM watson_searchentry WHERE Key_name = 'watson_searchentry_fulltext'")
return bool(cursor.fetchall()) return bool(cursor.fetchall())
def do_install(self): def do_install(self):
"""Executes the MySQL specific SQL code to install django-watson.""" """Executes the MySQL specific SQL code to install django-watson."""
cursor = connection.cursor() cursor = connection.cursor()
# Drop all foreign keys on the watson_searchentry table. # Drop all foreign keys on the watson_searchentry table.
cursor.execute("SELECT CONSTRAINT_NAME FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE() AND TABLE_NAME = 'watson_searchentry' AND CONSTRAINT_TYPE = 'FOREIGN KEY'") cursor.execute(
"SELECT CONSTRAINT_NAME FROM information_schema.TABLE_CONSTRAINTS "
"WHERE CONSTRAINT_SCHEMA = DATABASE() "
"AND TABLE_NAME = 'watson_searchentry' "
"AND CONSTRAINT_TYPE = 'FOREIGN KEY'"
)
for constraint_name, in cursor.fetchall(): for constraint_name, in cursor.fetchall():
cursor.execute("ALTER TABLE watson_searchentry DROP FOREIGN KEY {constraint_name}".format( cursor.execute(
constraint_name=constraint_name, "ALTER TABLE watson_searchentry DROP FOREIGN KEY {constraint_name}".format(
)) constraint_name=constraint_name,
)
)
# Change the storage engine to MyISAM. # Change the storage engine to MyISAM.
cursor.execute("ALTER TABLE watson_searchentry ENGINE = MyISAM") cursor.execute("ALTER TABLE watson_searchentry ENGINE = MyISAM")
# Add the full text indexes. # Add the full text indexes.
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_fulltext ON watson_searchentry (title, description, content)") cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_fulltext "
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_title ON watson_searchentry (title)") "ON watson_searchentry (title, description, content)")
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_description ON watson_searchentry (description)") cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_title "
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_content ON watson_searchentry (content)") "ON watson_searchentry (title)")
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_description "
"ON watson_searchentry (description)")
cursor.execute("CREATE FULLTEXT INDEX watson_searchentry_content "
"ON watson_searchentry (content)")
def do_uninstall(self): def do_uninstall(self):
"""Executes the SQL needed to uninstall django-watson.""" """Executes the SQL needed to uninstall django-watson."""
@ -427,7 +442,8 @@ class MySQLSearchBackend(SearchBackend):
tables=("watson_searchentry",), tables=("watson_searchentry",),
where=( where=(
"watson_searchentry.engine_slug = %s", "watson_searchentry.engine_slug = %s",
"MATCH (watson_searchentry.title, watson_searchentry.description, watson_searchentry.content) AGAINST (%s IN BOOLEAN MODE)", "MATCH (watson_searchentry.title, watson_searchentry.description, watson_searchentry.content) "
"AGAINST (%s IN BOOLEAN MODE)",
"watson_searchentry.{ref_name} = {table_name}.{pk_name}".format( "watson_searchentry.{ref_name} = {table_name}.{pk_name}".format(
ref_name=ref_name, ref_name=ref_name,
table_name=connection.ops.quote_name(model._meta.db_table), table_name=connection.ops.quote_name(model._meta.db_table),

View File

@ -2,8 +2,6 @@
from __future__ import unicode_literals, print_function from __future__ import unicode_literals, print_function
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError from django.core.management.base import BaseCommand, CommandError
from django.apps import apps from django.apps import apps
from django.contrib import admin from django.contrib import admin
@ -21,56 +19,68 @@ from watson.models import SearchEntry
# Sets up registration for django-watson's admin integration. # Sets up registration for django-watson's admin integration.
admin.autodiscover() admin.autodiscover()
def get_engine(engine_slug_): def get_engine(engine_slug_):
'''returns search engine with a given name''' """returns search engine with a given name"""
try: try:
return [x[1] for x in SearchEngine.get_created_engines() if x[0] == engine_slug_][0] return [x[1] for x in SearchEngine.get_created_engines() if x[0] == engine_slug_][0]
except IndexError: except IndexError:
raise CommandError("Search Engine \"%s\" is not registered!" % force_text(engine_slug_)) raise CommandError("Search Engine \"%s\" is not registered!" % force_text(engine_slug_))
def rebuild_index_for_model(model_, engine_slug_, verbosity_): def rebuild_index_for_model(model_, engine_slug_, verbosity_):
'''rebuilds index for a model''' """rebuilds index for a model"""
search_engine_ = get_engine(engine_slug_) search_engine_ = get_engine(engine_slug_)
local_refreshed_model_count = [0] # HACK: Allows assignment to outer scope. local_refreshed_model_count = [0] # HACK: Allows assignment to outer scope.
def iter_search_entries(): def iter_search_entries():
for obj in model_._default_manager.all().iterator(): for obj in model_._default_manager.all().iterator():
for search_entry in search_engine_._update_obj_index_iter(obj): for search_entry in search_engine_._update_obj_index_iter(obj):
yield search_entry yield search_entry
local_refreshed_model_count[0] += 1 local_refreshed_model_count[0] += 1
if verbosity_ >= 3: if verbosity_ >= 3:
print("Refreshed search entry for {model} {obj} in {engine_slug!r} search engine.".format( print(
model = force_text(model_._meta.verbose_name), "Refreshed search entry for {model} {obj} "
obj = force_text(obj), "in {engine_slug!r} search engine.".format(
engine_slug = force_text(engine_slug_), model=force_text(model_._meta.verbose_name),
)) obj=force_text(obj),
engine_slug=force_text(engine_slug_),
)
)
if verbosity_ == 2: if verbosity_ == 2:
print("Refreshed {local_refreshed_model_count} {model} search entry(s) in {engine_slug!r} search engine.".format( print(
model = force_text(model_._meta.verbose_name), "Refreshed {local_refreshed_model_count} {model} search entry(s) "
local_refreshed_model_count = local_refreshed_model_count[0], "in {engine_slug!r} search engine.".format(
engine_slug = force_text(engine_slug_), model=force_text(model_._meta.verbose_name),
)) local_refreshed_model_count=local_refreshed_model_count[0],
engine_slug=force_text(engine_slug_),
)
)
_bulk_save_search_entries(iter_search_entries()) _bulk_save_search_entries(iter_search_entries())
return local_refreshed_model_count[0] return local_refreshed_model_count[0]
class Command(BaseCommand): class Command(BaseCommand):
args = "[[--engine=search_engine] <app.model|model> <app.model|model> ... ]" args = "[[--engine=search_engine] <app.model|model> <app.model|model> ... ]"
help = "Rebuilds the database indices needed by django-watson. You can (re-)build index for selected models by specifying them" help = "Rebuilds the database indices needed by django-watson. " \
"You can (re-)build index for selected models by specifying them"
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument("apps", nargs="*", action="store", default=[]) parser.add_argument("apps", nargs="*", action="store", default=[])
parser.add_argument('--engine', parser.add_argument(
'--engine',
action="store", action="store",
help='Search engine models are registered with' help='Search engine models are registered with'
) )
@transaction.atomic() @transaction.atomic()
def handle(self, *args, **options): def handle(self, *args, **options):
"""Runs the management command.""" """Runs the management command."""
activate(settings.LANGUAGE_CODE) activate(settings.LANGUAGE_CODE)
verbosity = int(options.get("verbosity", 1)) verbosity = int(options.get("verbosity", 1))
# see if we're asked to use a specific search engine # see if we're asked to use a specific search engine
if options.get('engine'): if options.get('engine'):
engine_slug = options['engine'] engine_slug = options['engine']
@ -78,13 +88,13 @@ class Command(BaseCommand):
else: else:
engine_slug = "default" engine_slug = "default"
engine_selected = False engine_selected = False
# work-around for legacy optparser hack in BaseCommand. In Django=1.10 the # work-around for legacy optparser hack in BaseCommand. In Django=1.10 the
# args are collected in options['apps'], but in earlier versions they are # args are collected in options['apps'], but in earlier versions they are
# kept in args. # kept in args.
if len(options['apps']): if len(options['apps']):
args = options['apps'] args = options['apps']
# get the search engine we'll be checking registered models for, may be "default" # get the search engine we'll be checking registered models for, may be "default"
search_engine = get_engine(engine_slug) search_engine = get_engine(engine_slug)
models = [] models = []
@ -101,7 +111,10 @@ class Command(BaseCommand):
else: else:
model = None model = None
if model is None or not search_engine.is_registered(model): if model is None or not search_engine.is_registered(model):
raise CommandError("Model \"%s\" is not registered with django-watson search engine \"%s\"!" % (force_text(model_name), force_text(engine_slug))) raise CommandError(
"Model \"%s\" is not registered with django-watson search engine \"%s\"!"
% (force_text(model_name), force_text(engine_slug))
)
models.append(model) models.append(model)
refreshed_model_count = 0 refreshed_model_count = 0
@ -128,24 +141,31 @@ class Command(BaseCommand):
for model in registered_models: for model in registered_models:
refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity) refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity)
# Clean out any search entries that exist for stale content types. Only do it during full rebuild # Clean out any search entries that exist for stale content types.
# Only do it during full rebuild
valid_content_types = [ContentType.objects.get_for_model(model) for model in registered_models] valid_content_types = [ContentType.objects.get_for_model(model) for model in registered_models]
stale_entries = SearchEntry.objects.filter( stale_entries = SearchEntry.objects.filter(
engine_slug = engine_slug, engine_slug=engine_slug,
).exclude( ).exclude(
content_type__in = valid_content_types content_type__in=valid_content_types
) )
stale_entry_count = stale_entries.count() stale_entry_count = stale_entries.count()
if stale_entry_count > 0: if stale_entry_count > 0:
stale_entries.delete() stale_entries.delete()
if verbosity >= 1: if verbosity >= 1:
print("Deleted {stale_entry_count} stale search entry(s) in {engine_slug!r} search engine.".format( print(
stale_entry_count = stale_entry_count, "Deleted {stale_entry_count} stale search entry(s) "
engine_slug = force_text(engine_slug), "in {engine_slug!r} search engine.".format(
)) stale_entry_count=stale_entry_count,
engine_slug=force_text(engine_slug),
)
)
if verbosity == 1: if verbosity == 1:
print("Refreshed {refreshed_model_count} search entry(s) in {engine_slug!r} search engine.".format( print(
refreshed_model_count = refreshed_model_count, "Refreshed {refreshed_model_count} search entry(s) "
engine_slug = force_text(engine_slug), "in {engine_slug!r} search engine.".format(
)) refreshed_model_count=refreshed_model_count,
engine_slug=force_text(engine_slug),
)
)

View File

@ -3,6 +3,7 @@
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from watson import search as watson from watson import search as watson
class Command(BaseCommand): class Command(BaseCommand):
help = "List all registed models by django-watson." help = "List all registed models by django-watson."
@ -12,5 +13,3 @@ class Command(BaseCommand):
self.stdout.write("The following models are registed for the django-watson search engine:\n") self.stdout.write("The following models are registed for the django-watson search engine:\n")
for mdl in watson.get_registered_models(): for mdl in watson.get_registered_models():
self.stdout.write("- %s\n" % mdl.__name__) self.stdout.write("- %s\n" % mdl.__name__)

View File

@ -2,8 +2,6 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import json
from django.db import models from django.db import models
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.utils.functional import cached_property from django.utils.functional import cached_property
@ -13,6 +11,7 @@ try:
except ImportError: except ImportError:
from django.contrib.contenttypes.generic import GenericForeignKey from django.contrib.contenttypes.generic import GenericForeignKey
def has_int_pk(model): def has_int_pk(model):
"""Tests whether the given model has an integer primary key.""" """Tests whether the given model has an integer primary key."""
pk = model._meta.pk pk = model._meta.pk
@ -34,9 +33,9 @@ class SearchEntry(models.Model):
"""An entry in the search index.""" """An entry in the search index."""
engine_slug = models.CharField( engine_slug = models.CharField(
max_length = 200, max_length=200,
db_index = True, db_index=True,
default = "default", default="default",
) )
content_type = models.ForeignKey( content_type = models.ForeignKey(
@ -46,28 +45,28 @@ class SearchEntry(models.Model):
object_id = models.TextField() object_id = models.TextField()
object_id_int = models.IntegerField( object_id_int = models.IntegerField(
blank = True, blank=True,
null = True, null=True,
db_index = True, db_index=True,
) )
object = GenericForeignKey() object = GenericForeignKey()
title = models.CharField( title = models.CharField(
max_length = 1000, max_length=1000,
) )
description = models.TextField( description = models.TextField(
blank = True, blank=True,
) )
content = models.TextField( content = models.TextField(
blank = True, blank=True,
) )
url = models.CharField( url = models.CharField(
max_length = 1000, max_length=1000,
blank = True, blank=True,
) )
meta_encoded = models.TextField() meta_encoded = models.TextField()

View File

@ -2,7 +2,8 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import sys, json import json
import sys
from itertools import chain, islice from itertools import chain, islice
from threading import local from threading import local
from functools import wraps from functools import wraps
@ -63,11 +64,14 @@ class SearchAdapter(object):
try: try:
value = getattr(self, prefix) value = getattr(self, prefix)
except AttributeError: except AttributeError:
raise SearchAdapterError("Could not find a property called {name!r} on either {obj!r} or {search_adapter!r}".format( raise SearchAdapterError(
name = prefix, "Could not find a property called {name!r}"
obj = obj, " on either {obj!r} or {search_adapter!r}".format(
search_adapter = self, name=prefix,
)) obj=obj,
search_adapter=self,
)
)
else: else:
# Run the attribute on the search adapter, if it's callable. # Run the attribute on the search adapter, if it's callable.
if not isinstance(value, (QuerySet, models.Manager)): if not isinstance(value, (QuerySet, models.Manager)):
@ -97,7 +101,8 @@ class SearchAdapter(object):
def get_title(self, obj): def get_title(self, obj):
""" """
Returns the title of this search result. This is given high priority in search result ranking. Returns the title of this search result.
This is given high priority in search result ranking.
You can access the title of the search entry as `entry.title` in your search results. You can access the title of the search entry as `entry.title` in your search results.
@ -107,11 +112,12 @@ class SearchAdapter(object):
def get_description(self, obj): def get_description(self, obj):
""" """
Returns the description of this search result. This is given medium priority in search result ranking. Returns the description of this search result.
This is given medium priority in search result ranking.
You can access the description of the search entry as `entry.description` in your search results. Since You can access the description of the search entry as `entry.description`
this should contains a short description of the search entry, it's excellent for providing a summary in your search results. Since this should contains a short description of the search entry,
in your search results. it's excellent for providing a summary in your search results.
The default implementation returns `""`. The default implementation returns `""`.
""" """
@ -119,15 +125,20 @@ class SearchAdapter(object):
def get_content(self, obj): def get_content(self, obj):
""" """
Returns the content of this search result. This is given low priority in search result ranking. Returns the content of this search result.
This is given low priority in search result ranking.
You can access the content of the search entry as `entry.content` in your search results, although You can access the content of the search entry as `entry.content` in your search results,
this field generally contains a big mess of search data so is less suitable for frontend display. although this field generally contains a big mess of search data so is less suitable
for frontend display.
The default implementation returns all the registered fields in your model joined together. The default implementation returns all the registered fields in your model joined together.
""" """
# Get the field names to look up. # Get the field names to look up.
field_names = self.fields or (field.name for field in self.model._meta.fields if isinstance(field, (models.CharField, models.TextField))) field_names = self.fields or (
field.name for field in self.model._meta.fields if
isinstance(field, (models.CharField, models.TextField))
)
# Exclude named fields. # Exclude named fields.
field_names = (field_name for field_name in field_names if field_name not in self.exclude) field_names = (field_name for field_name in field_names if field_name not in self.exclude)
# Create the text. # Create the text.
@ -244,7 +255,11 @@ class SearchContextManager(local):
# Save all the models. # Save all the models.
tasks, is_invalid = self._stack.pop() tasks, is_invalid = self._stack.pop()
if not is_invalid: if not is_invalid:
_bulk_save_search_entries(list(chain.from_iterable(engine._update_obj_index_iter(obj) for engine, obj in tasks))) _bulk_save_search_entries(
list(chain.from_iterable(engine._update_obj_index_iter(obj)
for engine, obj in tasks)
)
)
# Context management. # Context management.
@ -345,9 +360,11 @@ class SearchEngine(object):
"""Initializes the search engine.""" """Initializes the search engine."""
# Check the slug is unique for this project. # Check the slug is unique for this project.
if engine_slug in SearchEngine._created_engines: if engine_slug in SearchEngine._created_engines:
raise SearchEngineError("A search engine has already been created with the slug {engine_slug!r}".format( raise SearchEngineError(
engine_slug = engine_slug, "A search engine has already been created with the slug {engine_slug!r}".format(
)) engine_slug=engine_slug,
)
)
# Initialize thie engine. # Initialize thie engine.
self._registered_models = {} self._registered_models = {}
self._engine_slug = engine_slug self._engine_slug = engine_slug
@ -374,13 +391,16 @@ class SearchEngine(object):
field_overrides["get_live_queryset"] = lambda self_: live_queryset.all() field_overrides["get_live_queryset"] = lambda self_: live_queryset.all()
# Check for existing registration. # Check for existing registration.
if self.is_registered(model): if self.is_registered(model):
raise RegistrationError("{model!r} is already registered with this search engine".format( raise RegistrationError(
model = model, "{model!r} is already registered with this search engine".format(
)) model=model,
))
# Perform any customization. # Perform any customization.
if field_overrides: if field_overrides:
# Conversion to str is needed because Python 2 doesn't accept unicode for class name # Conversion to str is needed because Python 2 doesn't accept unicode for class name
adapter_cls = type(str("Custom") + adapter_cls.__name__, (adapter_cls,), field_overrides) adapter_cls = type(
str("Custom") + adapter_cls.__name__, (adapter_cls,), field_overrides
)
# Perform the registration. # Perform the registration.
adapter_obj = adapter_cls(model) adapter_obj = adapter_cls(model)
self._registered_models[model] = adapter_obj self._registered_models[model] = adapter_obj
@ -401,7 +421,7 @@ class SearchEngine(object):
# Check for registration. # Check for registration.
if not self.is_registered(model): if not self.is_registered(model):
raise RegistrationError("{model!r} is not registered with this search engine".format( raise RegistrationError("{model!r} is not registered with this search engine".format(
model = model, model=model,
)) ))
# Perform the unregistration. # Perform the unregistration.
del self._registered_models[model] del self._registered_models[model]
@ -418,7 +438,7 @@ class SearchEngine(object):
if self.is_registered(model): if self.is_registered(model):
return self._registered_models[model] return self._registered_models[model]
raise RegistrationError("{model!r} is not registered with this search engine".format( raise RegistrationError("{model!r} is not registered with this search engine".format(
model = model, model=model,
)) ))
def _get_entries_for_obj(self, obj): def _get_entries_for_obj(self, obj):
@ -430,20 +450,20 @@ class SearchEngine(object):
object_id = force_text(obj.pk) object_id = force_text(obj.pk)
# Get the basic list of search entries. # Get the basic list of search entries.
search_entries = SearchEntry.objects.filter( search_entries = SearchEntry.objects.filter(
content_type = content_type, content_type=content_type,
engine_slug = self._engine_slug, engine_slug=self._engine_slug,
) )
if has_int_pk(model): if has_int_pk(model):
# Do a fast indexed lookup. # Do a fast indexed lookup.
object_id_int = int(obj.pk) object_id_int = int(obj.pk)
search_entries = search_entries.filter( search_entries = search_entries.filter(
object_id_int = object_id_int, object_id_int=object_id_int,
) )
else: else:
# Alas, have to do a slow unindexed lookup. # Alas, have to do a slow unindexed lookup.
object_id_int = None object_id_int = None
search_entries = search_entries.filter( search_entries = search_entries.filter(
object_id = object_id, object_id=object_id,
) )
return object_id_int, search_entries return object_id_int, search_entries
@ -514,31 +534,35 @@ class SearchEngine(object):
queryset = sub_queryset.values_list("pk", flat=True) queryset = sub_queryset.values_list("pk", flat=True)
if has_int_pk(model): if has_int_pk(model):
filter &= Q( filter &= Q(
object_id_int__in = queryset, object_id_int__in=queryset,
) )
else: else:
live_ids = list(queryset) live_ids = list(queryset)
if live_ids: if live_ids:
filter &= Q( filter &= Q(
object_id__in = live_ids, object_id__in=live_ids,
) )
else: else:
# HACK: There is a bug in Django (https://code.djangoproject.com/ticket/15145) that messes up __in queries when the iterable is empty. # HACK: There is a bug in Django
# This bit of nonsense ensures that this aspect of the query will be impossible to fulfill. # (https://code.djangoproject.com/ticket/15145)
# that messes up __in queries when the iterable is empty.
# This bit of nonsense ensures that this aspect of the query
# will be impossible to fulfill.
filter &= Q( filter &= Q(
content_type = ContentType.objects.get_for_model(model).id + 1, content_type=ContentType.objects.get_for_model(model).id + 1,
) )
# Add the model to the filter. # Add the model to the filter.
content_type = ContentType.objects.get_for_model(model) content_type = ContentType.objects.get_for_model(model)
filter &= Q( filter &= Q(
content_type = content_type, content_type=content_type,
) )
# Combine with the other filters. # Combine with the other filters.
filters |= filter filters |= filter
return filters return filters
def _get_included_models(self, models): def _get_included_models(self, models):
"""Returns an iterable of models and querysets that should be included in the search query.""" """Returns an iterable of models and querysets that should be included
in the search query."""
for model in models or self.get_registered_models(): for model in models or self.get_registered_models():
if isinstance(model, QuerySet): if isinstance(model, QuerySet):
yield model yield model
@ -559,7 +583,7 @@ class SearchEngine(object):
return SearchEntry.objects.none() return SearchEntry.objects.none()
# Get the initial queryset. # Get the initial queryset.
queryset = SearchEntry.objects.filter( queryset = SearchEntry.objects.filter(
engine_slug = self._engine_slug, engine_slug=self._engine_slug,
) )
# Process the allowed models. # Process the allowed models.
queryset = queryset.filter( queryset = queryset.filter(
@ -620,10 +644,11 @@ def get_backend(backend_name=None):
try: try:
backend_cls = getattr(backend_module, backend_cls_name) backend_cls = getattr(backend_module, backend_cls_name)
except AttributeError: except AttributeError:
raise ImproperlyConfigured("Could not find a class named {backend_module_name!r} in {backend_cls_name!r}".format( raise ImproperlyConfigured(
backend_module_name = backend_module_name, "Could not find a class named {backend_module_name!r} in {backend_cls_name!r}".format(
backend_cls_name = backend_cls_name, backend_module_name=backend_module_name,
)) backend_cls_name=backend_cls_name,
))
# Initialize the backend. # Initialize the backend.
backend = backend_cls() backend = backend_cls()
_backends_cache[backend_name] = backend _backends_cache[backend_name] = backend

View File

@ -9,7 +9,6 @@ from watson.views import search, search_json
urlpatterns = [ urlpatterns = [
url("^$", search, name="search"), url("^$", search, name="search"),
url("^json/$", search_json, name="search_json"), url("^json/$", search_json, name="search_json"),
] ]