sql: add blockdef fields to fulltext indexation (#53284)

tests/test_sql.py

@@ -9,6 +9,7 @@ from quixote import cleanup
 
 import wcs.qommon.storage as st
 from wcs import fields, sql
+from wcs.blocks import BlockDef
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon import force_str
@@ -26,6 +27,14 @@ def setup_module(module):
 
     pub = create_temporary_pub(sql_mode=True)
 
+    block = BlockDef()
+    block.name = 'fooblock'
+    block.fields = [
+        fields.StringField(id='1', label='string', type='string'),
+        fields.ItemField(id='2', label='item', items=('boat', 'plane', 'kick scooter')),
+    ]
+    block.store()
+
     formdef = FormDef()
     formdef.name = 'tests'
     formdef.fields = [
@@ -36,6 +45,7 @@ def setup_module(module):
         fields.ItemField(id='4', label='item', items=('apple', 'pear', 'peach', 'apricot')),
         fields.DateField(id='5', label='date'),
         fields.ItemsField(id='6', label='items', items=('apple', 'pear', 'peach', 'apricot')),
+        fields.BlockField(id='7', label='block', type='block:fooblock'),
     ]
     formdef.store()
 
@@ -162,6 +172,22 @@ def test_sql_field_items():
     check_sql_field('6', ['pomme', 'poire', 'pêche'], display=True)
 
 
+def test_sql_block_field_text():
+    check_sql_field('7', {'data': [{'1': 'foo'}, {'1': 'bar'}]})
+
+
+def test_sql_block_field_item():
+    check_sql_field(
+        '7',
+        {
+            'data': [
+                {'2': 'boat', '2_display': 'Yacht'},
+                {'2': 'plane', '2_display': 'Cessna'},
+            ]
+        },
+    )
+
+
 def test_sql_geoloc():
     test_formdef = FormDef()
     test_formdef.name = 'geoloc'
@@ -369,12 +395,29 @@ def test_sql_get_ids_from_query():
     formdata.store()
     id3 = formdata.id
 
+    formdata = data_class()
+    formdata.data = {
+        '7': {
+            'data': [
+                {'1': 'some other example having foo', '2': 'boat', '2_display': 'Yatch'},
+                {'1': 'bar', '2': 'plane', '2_display': 'Cessna'},
+            ]
+        }
+    }
+    formdata.store()
+    id4 = formdata.id
+
     ids = data_class.get_ids_from_query('text')
     assert set(ids) == set([id1, id3])
 
     ids = data_class.get_ids_from_query('classical')
     assert set(ids) == set([id2])
 
+    ids = data_class.get_ids_from_query('FOO')
+    assert set(ids) == set([id4])
+    ids = data_class.get_ids_from_query('cessna')
+    assert set(ids) == set([id4])
+
 
 def test_sql_rollback_on_error():
     data_class = formdef.data_class(mode='sql')

wcs/sql.py

@@ -2096,14 +2096,25 @@ class SqlDataMixin(SqlMixin):
         fts_strings.append(self._formdef.name)
         if self.tracking_code:
             fts_strings.append(self.tracking_code)
-        for field in self._formdef.get_all_fields():
-            if not self.data.get(field.id):
+
+        def get_all_fields():
+            for field in self._formdef.get_all_fields():
+                if field.key == 'block' and self.data.get(field.id):
+                    for data in self.data[field.id].get('data'):
+                        for subfield in field.block.fields:
+                            yield subfield, data
+                else:
+                    data = self.data
+                    yield field, self.data
+
+        for field, data in get_all_fields():
+            if not data.get(field.id):
                 continue
             value = None
             if field.key in ('string', 'text', 'email'):
-                value = self.data.get(field.id)
+                value = data.get(field.id)
             elif field.key in ('item', 'items'):
-                value = self.data.get('%s_display' % field.id)
+                value = data.get('%s_display' % field.id)
             if value:
                 if isinstance(value, str):
                     fts_strings.append(value)
@@ -3378,7 +3389,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (50, 'switch role uuid column to varchar')
+SQL_LEVEL = (51, 'add index on formdata blockdef fields')
 
 
 def migrate_global_views(conn, cur):
@@ -3503,7 +3514,7 @@ def migrate():
         # 21: (second part), store ascii_name of users
         # 23: (first part), use misc.simplify() over full text queries
         set_reindex('user', 'needed', conn=conn, cur=cur)
-    if sql_level < 41:
+    if sql_level < 51:
         # 17: store last_update_time in tables
         # 18: add user name to full-text search index
         # 21: (third part), add user ascii_names to full-text index
@@ -3513,6 +3524,7 @@ def migrate():
         # 31: add user_label to formdata
         # 38: extract submission_agent_id to its own column
         # 41: update full text normalization
+        # 51: add index on formdata blockdef fields
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 46:
         from wcs.carddef import CardDef