get down to postgresql for full text search

tabellio/searchform/batch_macros.pt

@@ -14,7 +14,7 @@ template with a tal:define="batch_base_url YOUR_BASE_URL" tales expression.
      tal:define="request request|context/request|container/request|nothing;
                  batch batch|nothing;
                  batchformkeys batchformkeys|nothing;
-                 batchlinkparams python:batchformkeys and dict([(key, request.form[key]) for key in batchformkeys if key in request]) or request.form;
+                 batchlinkparams view/get_batchlinkparams;
                  mq python:modules['ZTUtils'].make_query;
                  url batch_base_url | request/ACTUAL_URL;
                  currentpage batch/pagenumber;"
@@ -158,7 +158,7 @@ template with a tal:define="batch_base_url YOUR_BASE_URL" tales expression.
      tal:define="
                  batch batch|nothing;
                  batchformkeys batchformkeys|nothing;
-                 batchlinkparams python:batchformkeys and dict([(key, request.form[key]) for key in batchformkeys if key in request]) or request.form;
+                 batchlinkparams view/get_batchlinkparams;
                  ">
 <a tal:attributes="href python: 'feed?%s' % batch.pageurl(batchlinkparams,1);">Abonnez-vous au flux de cette recherche</a>
 </div>

tabellio/searchform/docsearch.pt

@@ -13,7 +13,6 @@
                            mq python:modules['ZTUtils'].make_query;
                            url batch_base_url | request/ACTUAL_URL;
                            batchformkeys batchformkeys|nothing;
-                           batchlinkparams python:batchformkeys and dict([(key, unicode(request.form[key]).encode('utf-8')) for key in batchformkeys if key in request]) or dict([(key, unicode(request.form[key]).encode('utf-8')) for key in request.form]);
                            b_size python:10; b_size request/b_size | b_size;
                            b_start python:0;b_start request/b_start | b_start;">
 

tabellio/searchform/form.py

@@ -1,6 +1,10 @@
 # -*- coding: utf-8 -*-
 
+import re
+import time
+
 from five import grok
+from plone.memoize import instance, ram
 from zope import interface, schema, component
 from z3c.form import form, field, button
 from plone.z3cform.layout import wrap_form
@@ -420,8 +424,45 @@ class GlobalSearchForm(form.Form):
 class SearchView(BrowserView):
     batch_macros = ViewPageTemplateFile('batch_macros.pt')
 
+    def db_connection(self):
+        portal = getToolByName(self.context, 'portal_url').getPortalObject()
+        return portal.db._wrapper.connection
+    db_connection = property(db_connection)
+
+    def _get_ids_cache_key(method, self, text):
+        return (text, )
+
+    @ram.cache(_get_ids_cache_key)
+    def get_ids_from_postgres(self, text):
+        try:
+            cursor = self.db_connection.cursor()
+        except AttributeError:
+            return []
+        # looks like there's no way to quote things properly for to_tsquery,
+        from plone.i18n.normalizer.fr import normalizer
+        text = re.sub(r'[^\w\s]', ' ', normalizer.normalize(text))
+        cursor.execute("""SELECT t_document.id
+                            FROM t_document JOIN t_file
+                              ON (t_document.text1id = t_file.fileid OR
+                                  t_document.textdefid = t_file.fileid)
+                           WHERE object_fts @@ to_tsquery('default_french', '''%s''')""" % text)
+        ids = [x[0] for x in cursor.fetchall()]
+        cursor.close()
+        return ids
+
+    def get_batchlinkparams(self):
+        d = dict()
+        for key in self.request.form:
+            d[key] = self.request.form[key]
+            if type(d[key]) is str:
+                d[key] = unicode(d[key], 'utf-8').encode('utf-8')
+            elif type(d[key]) is unicode:
+                d[key] = d[key].encode('utf-8')
+        return d
+
     def search_results(self, search_type):
-        print 'search type:', search_type
+        #print 'search type:', search_type
+
         if self.request.form.get('document.widgets.search_type_is_document'):
             GlobalSearchForm.prefix = 'document'
         elif self.request.form.get('dossier.widgets.search_type_is_dossier'):
@@ -433,14 +474,9 @@ class SearchView(BrowserView):
         f.update()
 
         data, errors = f.extractData()
-        print 'got data:', data
-
         kw = {}
 
-        print 'data:', data
-
         if not data.get('search_type_is_%s' % search_type):
-            print '  stopping here'
             return None
 
         if data.get('ttitle'):
@@ -489,9 +525,13 @@ class SearchView(BrowserView):
             kw['sort_on'] = 'no'
             kw['sort_order'] = 'ascending'
 
-        if not kw:
+        if not kw and not data.get('text'):
             return []
 
+        if data.get('text'):
+            # plaintext search, get document ids from postgresql
+            kw['id'] = self.get_ids_from_postgres(data.get('text'))
+
         if data.get('search_type_is_document'):
             kw['portal_type'] = 'tabellio.documents.document'
         elif data.get('search_type_is_dossier'):