sql: test purge of search tokens (#86527)

introduce a new mechanism to implement FTS with fuzzy-match.
This is made possible by adding and maintaining a table of the
FTS tokens, wcs_search_tokens, fed with searchable_formdefs
and wcs_all_forms.
When a query is issued, its tokens are matched against the
tokens with a fuzzy match when no direct match is found, and
the query is then rebuilt.

tests/test_datasource_users.py

@@ -1,3 +1,5 @@
+import xml.etree.ElementTree as ET
+
 import pytest
 
 from wcs import data_sources
@@ -57,11 +59,11 @@ def test_datasource_users(pub):
 
     assert data_sources.get_items({'type': datasource.slug}) == [
         (
-            '1',
+            str(users[0].id),
             'John Doe 0',
-            '1',
+            str(users[0].id),
             {
-                'id': 1,
+                'id': users[0].id,
                 'text': 'John Doe 0',
                 'user_name_identifier_0': 'abc0',
                 'user_nameid': 'abc0',
@@ -72,11 +74,11 @@ def test_datasource_users(pub):
             },
         ),
         (
-            '2',
+            str(users[1].id),
             'John Doe 1',
-            '2',
+            str(users[1].id),
             {
-                'id': 2,
+                'id': users[1].id,
                 'text': 'John Doe 1',
                 'user_name_identifier_0': 'abc1',
                 'user_nameid': 'abc1',
@@ -89,11 +91,11 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_items(datasource.extended_data_source) == [
         (
-            '1',
+            str(users[0].id),
             'John Doe 0',
-            '1',
+            str(users[0].id),
             {
-                'id': 1,
+                'id': users[0].id,
                 'text': 'John Doe 0',
                 'user_name_identifier_0': 'abc0',
                 'user_nameid': 'abc0',
@@ -104,11 +106,11 @@ def test_datasource_users(pub):
             },
         ),
         (
-            '2',
+            str(users[1].id),
             'John Doe 1',
-            '2',
+            str(users[1].id),
             {
-                'id': 2,
+                'id': users[1].id,
                 'text': 'John Doe 1',
                 'user_name_identifier_0': 'abc1',
                 'user_nameid': 'abc1',
@@ -121,7 +123,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -131,7 +133,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -143,7 +145,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -153,7 +155,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -169,7 +171,7 @@ def test_datasource_users(pub):
     datasource.store()
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -179,7 +181,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -191,7 +193,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -201,7 +203,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -223,7 +225,7 @@ def test_datasource_users(pub):
     users[0].store()
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -235,7 +237,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -254,7 +256,7 @@ def test_datasource_users(pub):
     datasource.store()
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -266,7 +268,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -291,7 +293,7 @@ def test_datasource_users(pub):
     assert not datasource.include_disabled_users
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -303,7 +305,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -319,7 +321,7 @@ def test_datasource_users(pub):
     datasource.store()
     assert data_sources.get_structured_items({'type': datasource.slug}) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -329,7 +331,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -341,7 +343,7 @@ def test_datasource_users(pub):
     ]
     assert data_sources.get_structured_items(datasource.extended_data_source) == [
         {
-            'id': 1,
+            'id': users[0].id,
             'text': 'John Doe 0',
             'user_name_identifier_0': 'abc0',
             'user_nameid': 'abc0',
@@ -351,7 +353,7 @@ def test_datasource_users(pub):
             'user_email': None,
         },
         {
-            'id': 2,
+            'id': users[1].id,
             'text': 'John Doe 1',
             'user_name_identifier_0': 'abc1',
             'user_nameid': 'abc1',
@@ -365,7 +367,7 @@ def test_datasource_users(pub):
 
     # by uuid
     assert datasource.get_structured_value('abc0') == {
-        'id': 1,
+        'id': users[0].id,
         'text': 'John Doe 0',
         'user_name_identifier_0': 'abc0',
         'user_nameid': 'abc0',
@@ -376,7 +378,7 @@ def test_datasource_users(pub):
     }
     assert datasource.get_display_value('abc0') == 'John Doe 0'
     assert datasource.get_structured_value('abc1') == {
-        'id': 2,
+        'id': users[1].id,
         'text': 'John Doe 1',
         'user_name_identifier_0': 'abc1',
         'user_nameid': 'abc1',
@@ -388,8 +390,8 @@ def test_datasource_users(pub):
     assert datasource.get_display_value('abc1') == 'John Doe 1'
 
     # by id
-    assert datasource.get_structured_value('1') == {
-        'id': 1,
+    assert datasource.get_structured_value(str(users[0].id)) == {
+        'id': users[0].id,
         'text': 'John Doe 0',
         'user_name_identifier_0': 'abc0',
         'user_nameid': 'abc0',
@@ -398,9 +400,9 @@ def test_datasource_users(pub):
         'user_display_name': 'John Doe 0',
         'user_email': None,
     }
-    assert datasource.get_display_value('1') == 'John Doe 0'
-    assert datasource.get_structured_value('2') == {
-        'id': 2,
+    assert datasource.get_display_value(str(users[0].id)) == 'John Doe 0'
+    assert datasource.get_structured_value(str(users[1].id)) == {
+        'id': users[1].id,
         'text': 'John Doe 1',
         'user_name_identifier_0': 'abc1',
         'user_nameid': 'abc1',
@@ -409,11 +411,11 @@ def test_datasource_users(pub):
         'user_display_name': 'John Doe 1',
         'user_email': None,
     }
-    assert datasource.get_display_value('2') == 'John Doe 1'
+    assert datasource.get_display_value(str(users[1].id)) == 'John Doe 1'
 
     # by numeric id
-    assert datasource.get_structured_value(1) == {
-        'id': 1,
+    assert datasource.get_structured_value(users[0].id) == {
+        'id': users[0].id,
         'text': 'John Doe 0',
         'user_name_identifier_0': 'abc0',
         'user_nameid': 'abc0',
@@ -422,9 +424,9 @@ def test_datasource_users(pub):
         'user_display_name': 'John Doe 0',
         'user_email': None,
     }
-    assert datasource.get_display_value(1) == 'John Doe 0'
-    assert datasource.get_structured_value(2) == {
-        'id': 2,
+    assert datasource.get_display_value(users[0].id) == 'John Doe 0'
+    assert datasource.get_structured_value(users[1].id) == {
+        'id': users[1].id,
         'text': 'John Doe 1',
         'user_name_identifier_0': 'abc1',
         'user_nameid': 'abc1',
@@ -433,7 +435,7 @@ def test_datasource_users(pub):
         'user_display_name': 'John Doe 1',
         'user_email': None,
     }
-    assert datasource.get_display_value(2) == 'John Doe 1'
+    assert datasource.get_display_value(users[1].id) == 'John Doe 1'
 
     datasource.users_included_roles = [role1.id]
     datasource.users_excluded_roles = [role2.id]
@@ -445,7 +447,7 @@ def test_datasource_users(pub):
     assert datasource.get_structured_value('abc0') is None
     assert datasource.get_display_value('abc0') is None
     assert datasource.get_structured_value('abc1') == {
-        'id': 2,
+        'id': users[1].id,
         'text': 'John Doe 1',
         'user_name_identifier_0': 'abc1',
         'user_nameid': 'abc1',
@@ -457,10 +459,10 @@ def test_datasource_users(pub):
     assert datasource.get_display_value('abc1') == 'John Doe 1'
 
     # by id
-    assert datasource.get_structured_value('1') is None
-    assert datasource.get_display_value('1') is None
-    assert datasource.get_structured_value('2') == {
-        'id': 2,
+    assert datasource.get_structured_value(str(users[0].id)) is None
+    assert datasource.get_display_value(str(users[0].id)) is None
+    assert datasource.get_structured_value(str(users[1].id)) == {
+        'id': users[1].id,
         'text': 'John Doe 1',
         'user_name_identifier_0': 'abc1',
         'user_nameid': 'abc1',
@@ -469,7 +471,7 @@ def test_datasource_users(pub):
         'user_display_name': 'John Doe 1',
         'user_email': None,
     }
-    assert datasource.get_display_value('2') == 'John Doe 1'
+    assert datasource.get_display_value(str(users[1].id)) == 'John Doe 1'
 
     datasource.include_disabled_users = False
     datasource.store()
@@ -483,10 +485,10 @@ def test_datasource_users(pub):
     assert datasource.get_display_value('abc1') is None
 
     # by id
-    assert datasource.get_structured_value('1') is None
-    assert datasource.get_display_value('1') is None
-    assert datasource.get_structured_value('2') is None
-    assert datasource.get_display_value('2') is None
+    assert datasource.get_structured_value(str(users[0].id)) is None
+    assert datasource.get_display_value(str(users[0].id)) is None
+    assert datasource.get_structured_value(str(users[1].id)) is None
+    assert datasource.get_display_value(str(users[1].id)) is None
 
 
 def test_datasource_users_user_formdef(pub):
@@ -509,9 +511,9 @@ def test_datasource_users_user_formdef(pub):
 
     assert data_sources.get_items({'type': datasource.slug}) == [
         (
-            '3',
+            str(user.id),
             'John Doe',
-            '3',
+            str(user.id),
             {
                 'user_display_name': 'John Doe',
                 'user_email': None,
@@ -520,8 +522,67 @@ def test_datasource_users_user_formdef(pub):
                 'user_var_plop': 'Bar',
                 'user_admin_access': False,
                 'user_backoffice_access': False,
-                'id': 3,
+                'id': user.id,
                 'text': 'John Doe',
             },
         )
     ]
+
+
+def test_legacy_format_import(pub):
+    data_source_xml = """<datasource id="255">
+  <name>Agents de la ville</name>
+  <slug>agents_de_la_ville</slug>
+  <data_source>
+    <type>wcs:users</type>
+    <value />
+  </data_source><users_included_roles>
+    <item>8201764fc2c24b92bd691fd231a4cf76</item>
+  </users_included_roles>
+</datasource>"""
+    ds = NamedDataSource.import_from_xml_tree(ET.fromstring(data_source_xml))
+    assert ds.users_included_roles == ['8201764fc2c24b92bd691fd231a4cf76']
+
+
+def test_new_format_import(pub):
+    data_source_xml = """<datasource id="255">
+  <name>Agents de la ville</name>
+  <slug>agents_de_la_ville</slug>
+  <data_source>
+    <type>wcs:users</type>
+    <value />
+  </data_source><users_included_roles>
+    <role role-id="8201764fc2c24b92bd691fd231a4cf76" role-slug="agent">Agents</role>
+  </users_included_roles>
+</datasource>"""
+    ds = NamedDataSource.import_from_xml_tree(ET.fromstring(data_source_xml))
+    assert ds.users_included_roles == []  # role doesn't exist
+
+    # import with id match
+    pub.role_class.wipe()
+    role1 = pub.role_class(name='role')
+    role1.id = '8201764fc2c24b92bd691fd231a4cf76'
+    role1.store()
+
+    ds = NamedDataSource.import_from_xml_tree(ET.fromstring(data_source_xml), include_id=True)
+    assert ds.users_included_roles == [role1.id]
+
+    # import with slug match
+    pub.role_class.wipe()
+    role1 = pub.role_class(name='Agents')
+    role1.slug = 'agent'
+    role1.store()
+
+    ds = NamedDataSource.import_from_xml_tree(ET.fromstring(data_source_xml), include_id=False)
+    assert ds.users_included_roles == [role1.id]
+
+    # import with name match
+    pub.role_class.wipe()
+    role1 = pub.role_class(name='Agents')
+    role1.slug = 'agent'
+    role1.store()
+
+    ds = NamedDataSource.import_from_xml_tree(
+        ET.fromstring(data_source_xml.replace('role-slug="agent"', '')), include_id=False
+    )
+    assert ds.users_included_roles == [role1.id]

wcs/data_sources.py

@@ -717,8 +717,8 @@ class NamedDataSource(XmlStorableObject):
         ('data_source', 'data_source'),
         ('notify_on_errors', 'bool'),
         ('record_on_errors', 'bool'),
-        ('users_included_roles', 'str_list'),
-        ('users_excluded_roles', 'str_list'),
+        ('users_included_roles', 'ds_roles'),
+        ('users_excluded_roles', 'ds_roles'),
         ('include_disabled_users', 'bool'),
     ]
 

wcs/qommon/form.py

@@ -1080,6 +1080,15 @@ class FileWithPreviewWidget(CompositeWidget):
                 '.pif',
                 '.php',
                 '.js',
+                '.pht',
+                '.phtml',
+                '.shtml',
+                '.asa',
+                '.asax',
+                '.cer',
+                '.swf',
+                '.xap',
+                '.ps1',
                 'application/x-ms-dos-executable',
                 'text/x-php',
             ]

wcs/qommon/xml_storage.py

@@ -20,7 +20,7 @@ import xml.etree.ElementTree as ET
 from quixote import get_publisher
 
 from .misc import indent_xml, xml_node_text
-from .storage import Equal, Or, StorableObject
+from .storage import Contains, Equal, Or, StorableObject
 
 
 class XmlStorableObject(StorableObject):
@@ -141,6 +141,8 @@ class XmlStorableObject(StorableObject):
     def import_roles_from_xml(self, element, include_id=False, **kwargs):
         criterias = []
         for sub in element:
+            if sub.tag != 'role':
+                continue
             if include_id and 'role-id' in sub.attrib:
                 criterias.append(Equal('id', sub.attrib['role-id']))
             elif 'role-slug' in sub.attrib:
@@ -156,3 +158,22 @@ class XmlStorableObject(StorableObject):
             return get_publisher().role_class.select([Or(criterias)], order_by='name')
 
         return lazy_roles
+
+    def import_ds_roles_from_xml(self, element, include_id=False, **kwargs):
+        imported_roles = self.import_roles_from_xml(element, include_id=include_id, **kwargs)
+        if callable(imported_roles):
+            imported_roles = imported_roles()
+        role_ids = [x.id for x in imported_roles]
+        for sub in element:
+            if sub.tag == 'item':  # legacy support for <item>{id}</item>
+                role_ids.append(xml_node_text(sub))
+        return role_ids
+
+    def export_ds_roles_to_xml(self, element, attribute_name, include_id=False, **kwargs):
+        for role in get_publisher().role_class.select(
+            [Contains('id', getattr(self, attribute_name, None) or [])]
+        ):
+            sub = ET.SubElement(element, 'role')
+            sub.attrib['role-id'] = role.id  # always include id
+            sub.attrib['role-slug'] = role.slug
+            sub.text = role.name