sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

tests/admin_pages/test_tests.py

@@ -1613,6 +1613,12 @@ def test_tests_webservice_response(pub):
 
     assert 'must start with http://' in resp.text
 
+    resp = app.get('/backoffice/forms/1/tests/%s/webservice-responses/' % testdef.id)
+    resp = resp.click('Import from other test')
+    resp = resp.form.submit()
+
+    assert resp.pyquery('div.error').text() == 'required field'
+
     testdef2 = TestDef.create_from_formdata(formdef, formdata)
     testdef.name = 'Second test'
     testdef2.store()

wcs/admin/tests.py

@@ -962,8 +962,8 @@ class WebserviceResponseDirectory(Directory):
         form.add(
             SingleSelectWidget,
             'testdef_id',
-            required=False,
-            options=testdef_options,
+            required=True,
+            options=[(None, '---', None)] + testdef_options,
             **{'data-autocomplete': 'true'},
         )
 

wcs/sql.py

@@ -1787,7 +1787,7 @@ WITH
             --          distance search is done using pg_trgm, https://www.postgresql.org/docs/current/pgtrgm.html
             -- otherwise: token as is and likely no search result later
             SELECT word,
-                coalesce((select perfect.token::tsquery FROM wcs_search_tokens AS perfect WHERE perfect.token = plainto_tsquery(word)::text),
+                coalesce((select perfect.token::tsquery FROM wcs_search_tokens AS perfect WHERE perfect.token = (tsvector_to_array(to_tsvector(word)))[1]),
                          tsquery_agg_or(partial.token::tsquery),
                          plainto_tsquery(word)) AS tokens
             FROM tokenized

wcs/workflows.py

@@ -418,7 +418,7 @@ class AttachmentEvolutionPart(EvolutionPart):
             return os.path.join(get_publisher().app_dir, self.filename)
 
     def get_file_pointer(self):
-        if self.filename.startswith('uuid-'):
+        if not self.filename or self.filename.startswith('uuid-'):
             return None
         return open(self.get_file_path(), 'rb')  # pylint: disable=consider-using-with