misc: add evolution messages to fulltext search (#22651)

tests/test_sql.py

@@ -18,6 +18,7 @@ from wcs.formdata import Evolution
 from wcs.roles import Role
 from wcs.workflows import Workflow, CommentableWorkflowStatusItem, WorkflowCriticalityLevel
 from wcs.wf.jump import JumpWorkflowStatusItem
+from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 from wcs import sql
 import wcs.qommon.storage as st
 
@@ -912,14 +913,31 @@ def test_sql_criteria_fts():
             t.data = {'3': 'foo'}
         else:
             t.data = {'3': 'bar'}
+        t.just_created()
         t.store()
-    t.store()
 
     assert data_class.count() == 50
     assert len(data_class.select()) == 50
 
     assert [x.id for x in data_class.select([st.FtsMatch('BAR')], order_by='id')] == range(21, 51)
 
+    # check fts against data in history
+    assert len(data_class.select([st.FtsMatch('XXX')])) == 0
+    formdata1 = data_class.select([st.FtsMatch('BAR')])[0]
+    formdata1.evolution[0].comment = 'XXX'
+    formdata1.store()
+    assert len(data_class.select([st.FtsMatch('XXX')])) == 1
+    assert data_class.select([st.FtsMatch('XXX')])[0].id == formdata1.id
+
+    assert len(data_class.select([st.FtsMatch('yyy')])) == 0
+    item = RegisterCommenterWorkflowStatusItem()
+    item.comment = '<span>ÿÿÿ</span>'
+    item.perform(formdata1)
+    assert formdata1.evolution[-1].display_parts()[-1] == '<span>ÿÿÿ</span>'
+    formdata1.store()
+    assert len(data_class.select([st.FtsMatch('yyy')])) == 1
+    assert len(data_class.select([st.FtsMatch('span')])) == 0
+
     assert data_class.count([st.FtsMatch('Pierre')]) == 0
     sql.SqlUser.wipe()
     user = sql.SqlUser()

wcs/qommon/misc.py

@@ -37,7 +37,9 @@ except ImportError:
 
 from django.conf import settings
 from django.utils import datetime_safe
+from django.utils.html import strip_tags
 from django.template import engines, TemplateSyntaxError, VariableDoesNotExist
+from django.utils.six.moves.html_parser import HTMLParser
 
 from quixote import get_publisher, get_response, get_request
 from quixote.html import htmltext
@@ -611,3 +613,8 @@ def normalize_geolocation(lat_lon):
     lat = wrap(lat, decimal.Decimal('-90.0'), decimal.Decimal('90.0'))
     lon = wrap(lon, decimal.Decimal('-180.0'), decimal.Decimal('180.0'))
     return {'lat': float(lat), 'lon': float(lon)}
+
+def html2text(text):
+    if isinstance(text, (htmltext, str)):
+        text = unicode(str(text), get_publisher().site_charset)
+    return site_encode(HTMLParser().unescape(strip_tags(text)))

wcs/sql.py

@@ -1426,6 +1426,11 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData):
             for evo in self._evolution:
                 if evo.comment:
                     fts_strings.append(evo.comment)
+                for part in evo.parts or []:
+                    if hasattr(part, 'view'):
+                        html_part = part.view()
+                        if html_part:
+                            fts_strings.append(qommon.misc.html2text(html_part))
         user = self.get_user()
         if user:
             fts_strings.append(user.get_display_name())
@@ -2157,7 +2162,7 @@ def get_yearly_totals(period_start=None, period_end=None, criterias=None):
     return result
 
 
-SQL_LEVEL = 28
+SQL_LEVEL = 29
 
 def migrate_global_views(conn, cur):
     cur.execute('''SELECT COUNT(*) FROM information_schema.tables
@@ -2265,6 +2270,7 @@ def migrate():
         # 21: (third part), add user ascii_names to full-text index
         # 23: (second part) use misc.simplify() over full text queries
         # 28: add display id and formdef name to full-text index
+        # 29: add evolution parts to full-text index
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 24:
         from wcs.formdef import FormDef