sql: delay re-index operations on first cron (#22383)

tests/test_sql.py

@@ -8,6 +8,8 @@ import string
 import sys
 import time
 
+from django.core.management import call_command
+
 from quixote import cleanup
 
 from wcs import formdef, publisher, fields
@@ -1102,6 +1104,15 @@ def test_migration_12_users_fts():
     assert column_exists_in_table(cur, 'users', 'fts')
     assert migration_level(cur) >= 12
 
+    # no fts, migration only prepare re-index
+    assert len(sql.SqlUser.get_ids_from_query('pierre')) == 0
+
+    assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
+    assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
+    call_command('cron')  # first cron = reindex
+    assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
+    assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
+
     # make sure the fts is filled after the migration
     assert len(sql.SqlUser.get_ids_from_query('pierre')) == 1
 
@@ -1127,6 +1138,15 @@ def test_migration_21_users_ascii_name():
     assert column_exists_in_table(cur, 'users', 'ascii_name')
     assert migration_level(cur) >= 21
 
+    # no fts, migration only prepare re-index
+    assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 0
+
+    assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
+    assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
+    call_command('cron')  # first cron = reindex
+    assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
+    assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
+
     # make sure the ascii_name is filled after the migration
     assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 1
 

wcs/publisher.py

@@ -298,6 +298,10 @@ class WcsPublisher(StubWcsPublisher):
         import sql
         sql.migrate()
 
+    def reindex_sql(self):
+        import sql
+        sql.reindex()
+
     def cleanup(self):
         if self.is_using_postgresql():
             import sql

wcs/qommon/cron.py

@@ -35,6 +35,11 @@ def cron_worker(publisher, now):
         publisher.set_config()
     except:
         return
+
+    # reindex user and formdata if needed (should only be run once)
+    if publisher.is_using_postgresql():
+        publisher.reindex_sql()
+
     for job in publisher.cronjobs:
         if job.days and now[2] not in job.days:
             continue

wcs/sql.py

@@ -2123,6 +2123,31 @@ def get_sql_level(conn, cur):
     sql_level = int(cur.fetchone()[0])
     return sql_level
 
+@guard_postgres
+def is_reindex_needed(index, conn, cur):
+    do_meta_table(conn, cur, insert_current_sql_level=False)
+    key_name = 'reindex_%s' % index
+    cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
+    row = cur.fetchone()
+    if row is None:
+        cur.execute('''INSERT INTO wcs_meta (id, key, value)
+                       VALUES (DEFAULT, %s, %s)''', (key_name, 'no'))
+        return False
+    return row[0] == 'needed'
+
+@guard_postgres
+def set_reindex(index, value, conn, cur):
+    do_meta_table(conn, cur, insert_current_sql_level=False)
+    key_name = 'reindex_%s' % index
+    cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
+    row = cur.fetchone()
+    if row is None:
+        cur.execute('''INSERT INTO wcs_meta (id, key, value)
+                       VALUES (DEFAULT, %s, %s)''', (key_name, value))
+    else:
+        cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', (
+                    value, key_name))
+
 def migrate_views(conn, cur):
     drop_views(None, conn, cur)
     from wcs.formdef import FormDef
@@ -2172,18 +2197,13 @@ def migrate():
         # 12: (second part), store fts in existing rows
         # 21: (second part), store ascii_name of users
         # 23: (first part), use misc.simplify() over full text queries
-        for user_id in SqlUser.keys():
-            SqlUser.get(user_id).store()
+        set_reindex('user', 'needed', conn=conn, cur=cur)
     if sql_level < 23:
         # 17: store last_update_time in tables
         # 18: add user name to full-text search index
         # 21: (third part), add user ascii_names to full-text index
         # 23: (second part) use misc.simplify() over full text queries
-        # load and store all formdatas
-        from wcs.formdef import FormDef
-        for formdef in FormDef.select():
-            for formdata in formdef.data_class().select():
-                formdata.store()
+        set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 24:
         from wcs.formdef import FormDef
         # 24: add index on evolution(formdata_id)
@@ -2197,3 +2217,24 @@ def migrate():
 
     conn.commit()
     cur.close()
+
+
+@guard_postgres
+def reindex():
+    conn, cur = get_connection_and_cursor()
+
+    if is_reindex_needed('user', conn=conn, cur=cur):
+        for user in SqlUser.select(iterator=True):
+            user.store()
+        set_reindex('user', 'done', conn=conn, cur=cur)
+
+    if is_reindex_needed('formdata', conn=conn, cur=cur):
+        # load and store all formdatas
+        from wcs.formdef import FormDef
+        for formdef in FormDef.select():
+            for formdata in formdef.data_class().select(iterator=True):
+                formdata.store()
+        set_reindex('formdata', 'done', conn=conn, cur=cur)
+
+    conn.commit()
+    cur.close()