From 5bb313971f1af2ce8b16e76ca3c8cf9caaeae716 Mon Sep 17 00:00:00 2001 From: Dustin Broderick Date: Tue, 28 Nov 2017 13:40:54 -0700 Subject: [PATCH 1/2] Adds 'slim' option to buildwatson --- watson/management/commands/buildwatson.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/watson/management/commands/buildwatson.py b/watson/management/commands/buildwatson.py index 59703d2..00ed2fa 100644 --- a/watson/management/commands/buildwatson.py +++ b/watson/management/commands/buildwatson.py @@ -28,7 +28,7 @@ def get_engine(engine_slug_): raise CommandError("Search Engine \"%s\" is not registered!" % force_text(engine_slug_)) -def rebuild_index_for_model(model_, engine_slug_, verbosity_): +def rebuild_index_for_model(model_, engine_slug_, verbosity_, slim_=False): """rebuilds index for a model""" search_engine_ = get_engine(engine_slug_) @@ -74,6 +74,14 @@ class Command(BaseCommand): action="store", help='Search engine models are registered with' ) + parser.add_argument( + '--slim', + action='store_true', + default=False, + help="Only include objects which satisfy the filter specified during \ + model registration. WARNING: buildwatson must be rerun if the filter \ + changes or the index will be incomplete." + ) @transaction.atomic() def handle(self, *args, **options): @@ -89,6 +97,9 @@ class Command(BaseCommand): engine_slug = "default" engine_selected = False + # Do we do a partial index? + slim = options.get("slim") + # work-around for legacy optparser hack in BaseCommand. In Django=1.10 the # args are collected in options['apps'], but in earlier versions they are # kept in args. @@ -123,7 +134,7 @@ class Command(BaseCommand): if verbosity >= 3: print("Using search engine \"%s\"" % engine_slug) for model in models: - refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity) + refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity, slim_=slim) else: # full rebuild (for one or all search engines) if engine_selected: @@ -139,7 +150,7 @@ class Command(BaseCommand): registered_models = search_engine.get_registered_models() # Rebuild the index for all registered models. for model in registered_models: - refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity) + refreshed_model_count += rebuild_index_for_model(model, engine_slug, verbosity, slim_=slim) # Clean out any search entries that exist for stale content types. # Only do it during full rebuild From 0a9ccc4e2eb09a3e8bde38e0f194672ff28b111c Mon Sep 17 00:00:00 2001 From: Dustin Broderick Date: Tue, 28 Nov 2017 13:42:36 -0700 Subject: [PATCH 2/2] Adds functional 'slim' indexing --- watson/management/commands/buildwatson.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/watson/management/commands/buildwatson.py b/watson/management/commands/buildwatson.py index 00ed2fa..7fc1eb0 100644 --- a/watson/management/commands/buildwatson.py +++ b/watson/management/commands/buildwatson.py @@ -36,7 +36,13 @@ def rebuild_index_for_model(model_, engine_slug_, verbosity_, slim_=False): local_refreshed_model_count = [0] # HACK: Allows assignment to outer scope. def iter_search_entries(): - for obj in model_._default_manager.all().iterator(): + # Only index specified objects if slim_ is True + if slim_ and search_engine_._registered_models[model_].get_live_queryset(): + obj_list = search_engine_._registered_models[model_].get_live_queryset() + else: + obj_list = model_._default_manager.all() + + for obj in obj_list.iterator(): for search_entry in search_engine_._update_obj_index_iter(obj): yield search_entry local_refreshed_model_count[0] += 1 @@ -82,7 +88,7 @@ class Command(BaseCommand): model registration. WARNING: buildwatson must be rerun if the filter \ changes or the index will be incomplete." ) - + @transaction.atomic() def handle(self, *args, **options): """Runs the management command."""