wcs_search_tokens: first part, create and maintain table (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

@@ -1679,0 +1728,4 @@
+$function$;""")
+
+    # Second part : insert and update triggers
+    cur.execute("CREATE OR REPLACE TRIGGER wcs_all_forms_fts_trg_ins AFTER INSERT ON wcs_all_forms FOR EACH ROW WHEN (NEW.fts IS NOT NULL) EXECUTE PROCEDURE wcs_search_tokens_trigger_fn();")

@@ -703,2 +708,4 @@
             CronJob(cls.clean_loggederrors, hours=[3], minutes=[0], name='clean_loggederrors')
         )
+        cls.register_cronjob(
+            CronJob(cls.clean_search_tokens, weekdays=[0], hours=[1], minutes=[0], name='clean_tokens')

@@ -97,2 +97,4 @@
 
 
+def _table_exists(cur, table_name):
+    cur.execute('SELECT 1 FROM pg_class WHERE relname = %s;', (table_name,))

@@ -1679,3 +1696,4 @@
         cur.close()
 
 
+def init_search_tokens(conn=None, cur=None):

@@ -1682,0 +1730,4 @@
+        tokenized as (select unnest(regexp_split_to_array($1, '\s+')) w),
+        super_tokenized as (
+            select w,
+                coalesce((select plainto_tsquery(perfect.token) from wcs_search_tokens perfect where perfect.token = plainto_tsquery(w)::text),

@@ -5121,3 +5281,3 @@
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (106, 'add context column to logged_errors table')
+SQL_LEVEL = (107, 'improved fts method')

@@ -1682,0 +1740,4 @@
+    # And last: functions to use this brand new table
+    # These two aggregates make the search query far simpler to write
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_or  (tsquery) (sfunc=tsquery_or,  stype=tsquery);')
+    cur.execute('CREATE OR REPLACE AGGREGATE tsquery_agg_and (tsquery) (sfunc=tsquery_and, stype=tsquery);')

@@ -1682,0 +1758,4 @@
+                         tsquery_agg_or(plainto_tsquery(partial.token) order by partial.token <-> w desc),
+                         plainto_tsquery(w)) tokens
+            FROM tokenized
+            LEFT JOIN wcs_search_tokens AS partial ON partial.token % w AND w not similar to '%[0-9]{2,}%'

@@ -1682,0 +1798,4 @@
+    ):
+        # Second part: insert and update triggers for wcs_all_forms
+        cur.execute(
+            """CREATE TRIGGER wcs_all_forms_fts_trg_ins