sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

tests/api/test_formdef.py

@@ -386,10 +386,6 @@ def test_backoffice_submission_formdef_list(pub, local_user, access, auth):
 
 @pytest.mark.parametrize('auth', ['signature', 'http-basic'])
 def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth):
-    from wcs import sql
-    # purge garbage from other tests
-    sql.purge_obsolete_search_tokens()
-
     pub.role_class.wipe()
     role = pub.role_class(name='Foo bar')
     role.store()
@@ -433,17 +429,6 @@ def test_backoffice_submission_formdef_list_search(pub, local_user, access, auth
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=test')
     assert len(resp.json['data']) == 2
 
-    # dump content of tokens, something is wrong
-    _, cur = sql.get_connection_and_cursor()
-    cur.execute("SELECT * FROM wcs_search_tokens")
-    print(cur.fetchall())
-
-    cur.execute("SELECT version()")
-    print(cur.fetchall())
-
-    cur.execute("select wcs_tsquery('tes')")
-    print(cur.fetchall())
-
     resp = get_url('/api/formdefs/?backoffice-submission=on&q=tes')
     assert len(resp.json['data']) == 2
 

wcs/sql.py

@@ -1791,7 +1791,7 @@ WITH
                          tsquery_agg_or(plainto_tsquery(partial.token)),
                          plainto_tsquery(word)) AS tokens
             FROM tokenized
-            LEFT JOIN wcs_search_tokens AS partial ON partial.token <-> plainto_tsquery('simple', word)::text < 0.7 AND word not similar to '%[0-9]{2,}%'
+            LEFT JOIN wcs_search_tokens AS partial ON partial.token % plainto_tsquery('simple', word)::text AND word not similar to '%[0-9]{2,}%'
             GROUP BY word)
 SELECT tsquery_agg_and(tokens) FROM super_tokenized;
 $function$;"""