tabellio.pcfdb/tabellio/pcfdb/sync.py

# -*- coding: utf-8 -*-

import datetime
import re
import time

from zope.i18n.locales import locales

from Products.CMFCore.WorkflowCore import WorkflowException
from Products.Five.browser import BrowserView
from z3c.sqlalchemy import getSAWrapper, createSAWrapper
from Products.CMFCore.utils import getToolByName

from zope.event import notify
from zope.lifecycleevent import ObjectAddedEvent, ObjectModifiedEvent

from zope import component
from zope.app.intid.interfaces import IIntIds
from z3c.relationfield import RelationValue
from zope.i18n.locales import locales

import transaction

from plone.registry.interfaces import IRegistry

from tabellio.config.interfaces import ITabellioSettings


class SyncFromPcfDbView(BrowserView):
    def setup(self):
        self.settings = component.getUtility(IRegistry).forInterface(ITabellioSettings, False)
        self.portal = getToolByName(self.context, 'portal_url').getPortalObject()
        self.plone_utils = getToolByName(self.context, 'plone_utils')
        self.intids = component.getUtility(IIntIds)
        self.portal_workflow = getToolByName(self.context, 'portal_workflow')
        self.portal_types = getToolByName(self.context, 'portal_types')

    def db_connection(self):
        return self.portal.db._wrapper.connection
    db_connection = property(db_connection)

    def get_folder_at_path(self, path):
        current = self.portal
        for part in path.split('/'):
            if not part:
                continue
            current = getattr(current, part)
        return current

    _deputies_folder = None
    def deputies_folder(self):
        if self._deputies_folder:
            return self._deputies_folder
        path = self.settings.deputiesPath
        self._deputies_folder = self.get_folder_at_path(path)
        return self._deputies_folder
    deputies_folder = property(deputies_folder)

    _ministries_folder = None
    def ministries_folder(self):
        if self._ministries_folder:
            return self._ministries_folder
        path = self.settings.ministriesPath
        self._ministries_folder = self.get_folder_at_path(path)
        return self._ministries_folder
    ministries_folder = property(ministries_folder)

    _persons_folder = None
    def persons_folder(self):
        if self._persons_folder:
            return self._persons_folder
        path = self.settings.personsPath
        self._persons_folder = self.get_folder_at_path(path)
        return self._persons_folder
    persons_folder = property(persons_folder)

    _polgroups_folder = None
    def polgroups_folder(self):
        if self._polgroups_folder:
            return self._polgroups_folder
        path = self.settings.polgroupsPath
        self._polgroups_folder = self.get_folder_at_path(path)
        return self._polgroups_folder
    polgroups_folder = property(polgroups_folder)

    _parlevents_folder = None
    def parlevents_folder(self):
        if self._parlevents_folder:
            return self._parlevents_folder
        path = self.settings.parleventsPath
        self._parlevents_folder = self.get_folder_at_path(path)
        return self._parlevents_folder
    parlevents_folder = property(parlevents_folder)

    _polgroups_intids = None
    def get_polgroup_intid(self, title):
        if not self._polgroups_intids:
            self._polgroups_intids = {}

        if title in self._polgroups_intids:
            return self._polgroups_intids.get(title)

        polgroup_id = self.plone_utils.normalizeString(title)
        if not hasattr(self.polgroups_folder, polgroup_id):
            self.polgroups_folder.invokeFactory('themis.datatypes.polgroup', polgroup_id, title=title)
            try:
                self.portal_workflow.doActionFor(getattr(self.polgroups_folder, polgroup_id), 'publish')
            except WorkflowException:
                pass
        polgroup_intid = self.intids.getId(getattr(self.polgroups_folder, polgroup_id))
        self._polgroups_intids[title] = polgroup_intid
        return polgroup_intid

    _documents_folder = None
    def documents_folder(self):
        if self._documents_folder:
            return self._documents_folder
        path = self.settings.documentsPath
        self._documents_folder = self.get_folder_at_path(path)
        return self._documents_folder
    documents_folder = property(documents_folder)

    _dossiers_folder = None
    def dossiers_folder(self):
        if self._dossiers_folder:
            return self._dossiers_folder
        path = self.settings.dossiersPath
        self._dossiers_folder = self.get_folder_at_path(path)
        return self._dossiers_folder
    dossiers_folder = property(dossiers_folder)

    _commissions_folder = None
    def commissions_folder(self):
        if self._commissions_folder:
            return self._commissions_folder
        path = self.settings.commissionsPath
        self._commissions_folder = self.get_folder_at_path(path)
        return self._commissions_folder
    commissions_folder = property(commissions_folder)

    _questions_folder = None
    def questions_folder(self):
        if self._questions_folder:
            return self._questions_folder
        path = self.settings.questionsPath
        self._questions_folder = self.get_folder_at_path(path)
        return self._questions_folder
    questions_folder = property(questions_folder)

    _author_intids = None
    def get_author_intid(self, author_id):
        if not self._author_intids:
            self._author_intids = {}
        if author_id in self._author_intids:
            return self._author_intids.get(author_id)
        cursor = self.db_connection.cursor()
        cursor.execute('''SELECT nom, prenom, st
                            FROM t_pers
                           WHERE t_pers.id = %(id)s''', {'id': author_id})
        row = cursor.fetchone()
        cursor.close()
        lastname, firstname, st = row
        if lastname and firstname:
            fullname = '%s %s' % (firstname, lastname)
        else:
            fullname = lastname
        new_id = self.plone_utils.normalizeString(fullname)
        if hasattr(self.deputies_folder, new_id):
            obj = getattr(self.deputies_folder, new_id)
        else:
            try:
                if st == 'S_PARL':
                    obj = getattr(self.deputies_folder, new_id)
                elif st == 'S_MINISTRE':
                    obj = getattr(self.ministries_folder, new_id)
                elif st in ('S_CREATED', 'S_DEAD'):
                    obj = getattr(self.persons_folder, new_id)
                else:
                    print 'unhandled type', st, author_id, new_id
                    self._author_intids[author_id] = None
                    return None
            except AttributeError:
                print 'missing author:', fullname
                self._author_intids[author_id] = None
                return None
        author_intid = self.intids.getId(obj)
        self._author_intids[author_id] = author_intid
        return author_intid

    def get_author_intid_from_name(self, firstname, lastname, st='S_PARL'):
        if lastname and firstname:
            fullname = '%s %s' % (firstname, lastname)
        else:
            fullname = lastname
        new_id = self.plone_utils.normalizeString(fullname)
        if st == 'S_PARL':
            obj = getattr(self.deputies_folder, new_id)
        elif st == 'S_MINISTRE':
            obj = getattr(self.ministries_folder, new_id)
        author_intid = self.intids.getId(obj)
        return author_intid

    def get_document_and_intid(self, doc_id):
        folder = self.documents_folder
        try:
            obj = getattr(folder, doc_id)
        except AttributeError:
            print 'missing doc', doc_id
            return None, None
        try:
            return obj, self.intids.getId(obj)
        except KeyError:
            self.intids.register(obj)
            return obj, self.intids.getId(obj)

    _commissions = None
    def get_commission(self, com_id):
        if not self._commissions:
            self._commissions = {}
        if com_id in self._commissions:
            return self._commissions.get(com_id)
        cursor = self.db_connection.cursor()
        cursor.execute('''SELECT t_comcat.nom, code
                            FROM t_com JOIN t_comcat
                              ON (t_com.comcat = t_comcat.id)
                           WHERE t_com.id = %(id)s''',
                           {'id': com_id})
        row = cursor.fetchone()
        if row is None:
            return None
        category, code = row

        cat_id = self.plone_utils.normalizeString(category)
        folder = getattr(self.commissions_folder, cat_id)
        code_id = self.plone_utils.normalizeString(code)
        obj = getattr(folder, code_id)
        self._commissions[com_id] = obj
        return obj

    _commission_intids = None
    def get_commission_intid(self, com_id):
        if not self._commission_intids:
            self._commission_intids = {}
        if com_id in self._commission_intids:
            return self._commission_intids.get(com_id)
        obj = self.get_commission(com_id)
        try:
            intid = self.intids.getId(obj)
        except KeyError:
            self.intids.register(obj)
            intid = self.intids.getId(obj)
        self._commission_intids[com_id] = intid
        return intid

    _commission_titles = None
    def get_commission_title(self, com_id):
        # this function should only be used in case we do not mind getting a
        # title without a corresponding object.
        if not self._commission_titles:
            self._commission_titles = {}
        if com_id in self._commission_titles:
            return self._commission_titles.get(com_id)
        cursor = self.db_connection.cursor()
        cursor.execute('''SELECT id, nom
                            FROM t_com
                           WHERE t_com.id = %(id)s''',
                           {'id': com_id})
        row = cursor.fetchone()
        if row is None:
            return None
        id, title = row
        self._commission_titles[com_id] = title
        return title

    def publish(self, object):
        current_state = self.portal_workflow.getStatusOf(
                        self.portal_workflow.getChainFor(object)[0], object).get('review_state')
        if current_state != 'published':
            try:
                self.portal_workflow.doActionFor(object, 'publish')
            except WorkflowException:
                pass
        try:
            self.intids.getId(object)
        except KeyError:
            self.intids.register(object)

    def unpublish(self, object):
        current_state = self.portal_workflow.getStatusOf(
                        self.portal_workflow.getChainFor(object)[0], object).get('review_state')
        if current_state == 'published':
            try:
                self.portal_workflow.doActionFor(object, 'retract')
            except WorkflowException:
                pass

    def get_polgroups_id_to_title(self, cursor):
        polgroups_id_to_title = {}
        cursor.execute('''SELECT id, abbr FROM t_comppol''')
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            polgroups_id_to_title[row[0]] = row[1]
        return polgroups_id_to_title

    def __call__(self):
        self.setup()

        def format_duration(v):
            r = ''
            if v > 3600:
                r = r + '%dh' % int(v/3600)
                v = v % 3600
            if v > 60 or r:
                r = r + '%02dm' % int(v/60)
                v = v % 60
            r = r + '%02ds' % int(v)
            return r

        timestamp = self.request.form.get('timestamp')
        objects = self.request.form.get('objects', '').split(',')
        report = []
        for object in ('polgroups', 'sessions', 'deputies', 'ministries', 'persons', 'commissions',
                       'documents', 'dossiers', 'questions', 'reunions', 'deptables', 'thesaurus'):
            if not 'all' in objects:
                if not object in objects:
                    continue
            t0 = time.time()
            result = getattr(self, 'sync_'+object)(timestamp)
            duration = time.time() - t0
            report.append('%-14s: %6s [%s]' % (object, result, format_duration(duration)))
            transaction.commit()

        print '\n'.join(report)
        return '\n'.join(report) + '\n'

    def get_address(self, address_id):
        if not address_id:
            return None
        cursor = self.db_connection.cursor()
        cursor.execute('''SELECT titre, batbur, rueno, zip, localite, tel1, tel2, fax, email
                            FROM t_adresse
                           WHERE t_adresse.id = %(id)s''', {'id': address_id})
        row = cursor.fetchone()
        if not row:
            return None
        (titre, batbur, street, zipcode, city, phone1, phone2, fax, email) = row
        from themis.datatypes.address import Address
        addr = Address()
        addr.title = titre or batbur
        addr.street = street
        addr.zipcode = zipcode
        addr.city = city
        addr.phone1 = phone1
        addr.phone2 = phone2
        addr.fax = fax
        addr.email = email
        cursor.close()
        return addr

    def sync_sessions(self, timestamp=None):
        cursor = self.db_connection.cursor()

        cursor.execute('''SELECT t_legisl.nom, t_sess.id
                            FROM t_sess
                            JOIN t_legisl
                              ON (t_sess.legislid = t_legisl.id)
                        ORDER BY t_sess.datedeb DESC''')

        sessions = []
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            sessions.append('%s:%s' % row)

        cursor.close()

        self.settings.sessions = '\n'.join(sessions)

        return 'OK'


    def sync_thesaurus(self, timestamp=None):
        cursor = self.db_connection.cursor()

        cursor.execute('''SELECT t_thesaurus_term.id, t_thesaurus_term.term
                            FROM t_thesaurus_term
                           WHERE EXISTS (
                                 SELECT * FROM t_thesaurus_rel
                                  WHERE sectermid = t_thesaurus_term.id
                                    AND rel not in ('HN', 'SN'))''')
        terms = []
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            terms.append('%s|%s' % row)

        cursor.close()

        self.settings.topics = '\n'.join(terms)

        return 'OK'


    def sync_polgroups(self, timestamp=None):
        cursor = self.db_connection.cursor()

        cursor.execute('''SELECT st, abbr FROM t_comppol''')
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            st, title = row
            polgroup_id = self.plone_utils.normalizeString(title)
            if not hasattr(self.polgroups_folder, polgroup_id):
                self.polgroups_folder.invokeFactory('themis.datatypes.polgroup',
                        polgroup_id, title=title)
            object = getattr(self.polgroups_folder, polgroup_id)
            object.active = (st == 'S_ACTIVE')
            notify(ObjectModifiedEvent(object))
            self.publish(object)

        cursor.close()
        return 'OK'


    def sync_deputies(self, timestamp=None):
        cursor = self.db_connection.cursor()
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('AND t_pers.ts > %s', (timestamp,))

        # previous deputies
        cursor.execute('''SELECT t_pers.id, t_pers.nom, prenom, sexe, datenaiss,
                                 t_comppol.abbr
                            FROM t_pers, t_pershistoline, t_comppol
                           WHERE (t_pers.st = 'S_CREATED' or
                                  t_pers.st = 'S_DEAD' or
                                  t_pers.st = 'S_MINISTRE') and
                                 t_pers.prenom is not NULL and
                                 t_pers.id = t_pershistoline.pers and
                                 t_pershistoline.description = t_comppol.id and
                                 t_pershistoline.type = 'P_CMPL' and
                                 t_pershistoline.fin IS NOT NULL
                                 %s
                        ORDER BY t_pershistoline.fin DESC''' % where_ts)

        polgroup_ids = {}
        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            (pers_id, lastname, firstname, sex, birthdate, polgroup) = row
            fullname = '%s %s' % (firstname, lastname)
            new_id = self.plone_utils.normalizeString(fullname)
            if not hasattr(self.deputies_folder, new_id):
                self.deputies_folder.invokeFactory('themis.datatypes.deputy', new_id,
                        firstname=firstname, lastname=lastname)
            object = getattr(self.deputies_folder, new_id)
            object.firstname = firstname
            object.lastname = lastname
            object.sex = sex
            object.birthdate = birthdate
            object.polgroup = RelationValue(self.get_polgroup_intid(polgroup))
            object.active = False
            notify(ObjectModifiedEvent(object))
            self.publish(object)

        # currently active deputies
        cursor.execute('''SELECT t_pers.id, t_pers.nom, prenom, sexe, datenaiss,
                                 t_pers.addrpriv, t_pers.addrprof1, t_pers.addrprof2,
                                 t_comppol.abbr, t_arrond.nom
                            FROM t_pers, t_pershistoline, t_comppol, t_parl, t_arrond
                           WHERE t_pers.st = 'S_PARL' and
                                 t_pers.prenom is not NULL and
                                 t_pers.id = t_pershistoline.pers and
                                 t_pershistoline.description = t_comppol.id and
                                 t_pershistoline.type = 'P_CMPL' and
                                 t_pershistoline.fin IS NULL and
                                 t_pers.id = t_parl.id and
                                 t_parl.arrond = t_arrond.id
                                 %s''' % where_ts)

        polgroup_ids = {}
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            (pers_id, lastname, firstname, sex, birthdate,
             privaddr, workaddr1, workaddr2, polgroup, district) = row
            fullname = '%s %s' % (firstname, lastname)
            new_id = self.plone_utils.normalizeString(fullname)
            if not hasattr(self.deputies_folder, new_id):
                self.deputies_folder.invokeFactory('themis.datatypes.deputy', new_id,
                        firstname=firstname, lastname=lastname)
            object = getattr(self.deputies_folder, new_id)
            object.firstname = firstname
            object.lastname = lastname
            object.sex = sex
            object.birthdate = birthdate
            object.district = district
            object.polgroup = RelationValue(self.get_polgroup_intid(polgroup))
            object.private_address = self.get_address(privaddr)
            object.work_address = self.get_address(workaddr1)
            object.work_address_2 = self.get_address(workaddr2)
            object.active = True
            notify(ObjectModifiedEvent(object))
            self.publish(object)

        cursor.close()
        return count

    def sync_ministries(self, timestamp=None):
        cursor = self.db_connection.cursor()
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('AND t_pers.ts > %s', (timestamp,))
        cursor.execute('''SELECT t_pers.id, t_pers.nom, prenom, sexe, datenaiss
                            FROM t_pers
                           WHERE t_pers.st = 'S_MINISTRE' and
                                 t_pers.prenom is not NULL
                                 %s''' % where_ts)

        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            pers_id, lastname, firstname, sex, birthdate = row
            fullname = '%s %s' % (firstname, lastname)
            new_id = self.plone_utils.normalizeString(fullname)
            if not hasattr(self.ministries_folder, new_id):
                self.ministries_folder.invokeFactory('themis.datatypes.ministry', new_id,
                        firstname=firstname, lastname=lastname)
            object = getattr(self.ministries_folder, new_id)
            object.firstname = firstname
            object.lastname = lastname
            object.sex = sex
            object.birthdate = birthdate
            notify(ObjectModifiedEvent(object))
            self.publish(object)

        cursor.close()
        return count

    def sync_persons(self, timestamp=None):
        cursor = self.db_connection.cursor()
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('AND t_pers.ts > %s', (timestamp,))
        cursor.execute('''SELECT t_pers.id, t_pers.nom, prenom, sexe, datenaiss
                            FROM t_pers
                           WHERE t_pers.st = 'S_CREATED' OR
                                 t_pers.st = 'S_DEAD'
                                 %s''' % where_ts)

        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            pers_id, lastname, firstname, sex, birthdate = row
            if firstname and lastname:
                fullname = '%s %s' % (firstname, lastname)
            else:
                fullname = lastname
            new_id = self.plone_utils.normalizeString(fullname)
            if not hasattr(self.persons_folder, new_id):
                self.persons_folder.invokeFactory('themis.datatypes.contact', new_id,
                        title=fullname)
            object = getattr(self.persons_folder, new_id)
            object.title = fullname
            notify(ObjectModifiedEvent(object))
            self.publish(object)

        cursor.close()
        return count

    def sync_commissions(self, timestamp=None):
        cursor = self.db_connection.cursor()
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('AND t_com.ts > %s', (timestamp,))
        cursor.execute('''SELECT t_com.id, t_com.st, t_com.nom, code, compets, t_comcat.nom
                            FROM t_com JOIN t_comcat
                              ON (t_com.comcat = t_comcat.id)
                           WHERE t_com.st = 'S_ACTIVE'
                                 %s''' % where_ts)
        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            com_id, state, title, code, compets, category = row

            print 'processing com:', com_id, code

            cat_id = self.plone_utils.normalizeString(category)
            if not hasattr(self.commissions_folder, cat_id):
                self.commissions_folder.invokeFactory('Folder',
                                cat_id, title=category)
            commission_folder = getattr(self.commissions_folder, cat_id)

            com_code = self.plone_utils.normalizeString(code)
            if title[0] == title[0].lower():
                title = title[0].upper() + title[1:]
            if not hasattr(commission_folder, com_code):
                commission_folder.invokeFactory('themis.datatypes.commission',
                                com_code, title=title)
            object = getattr(commission_folder, com_code)
            object.title = title
            object.active == (state == 'S_ACTIVE')
            object.code = code

            pers_cursor = self.db_connection.cursor()
            pers_cursor.execute('''SELECT t_pershistoline.type, nom, prenom
                                     FROM t_pers JOIN t_pershistoline
                                       ON t_pershistoline.pers = t_pers.id
                                    WHERE t_pershistoline.description = %(id)s
                                      AND t_pershistoline.fin IS NULL''',
                                      {'id': com_id})
            object.president = None
            object.vicepresidents = []
            object.members = []
            object.substitutes = []
            while True:
                row = pers_cursor.fetchone()
                if row is None:
                    break
                type, lastname, firstname = row
                if not (lastname and firstname):
                    # this is some placeholder for a future person, skip it
                    continue
                if type == 'P_COMM_PR':
                    object.president = RelationValue(
                                    self.get_author_intid_from_name(firstname, lastname))
                elif type == 'P_COMM_VP':
                    object.vicepresidents.append(
                                    RelationValue(self.get_author_intid_from_name(firstname, lastname)))
                elif type == 'P_COMM_TI':
                    object.members.append(
                                    RelationValue(self.get_author_intid_from_name(firstname, lastname)))
                elif type == 'P_COMM_SU':
                    object.substitutes.append(
                                    RelationValue(self.get_author_intid_from_name(firstname, lastname)))
            pers_cursor.close()

            # compets
            compet_cursor = self.db_connection.cursor()
            compet_cursor.execute('''SELECT t_compet.nom
                                FROM t_com JOIN t_com_compets
                                  ON (t_com.id = t_com_compets.id)
                                JOIN t_compet
                                  ON (t_com_compets.value = t_compet.id)
                               WHERE t_compet.st = 'S_ACTIVE' AND t_com.id = %(id)s
                            ORDER BY t_compet.nom''', {'id': com_id})
            object.competences = []
            while True:
                row = compet_cursor.fetchone()
                if row is None:
                    break
                object.competences.append(row[0])
            compet_cursor.close()

            # ministries
            minist_cursor = self.db_connection.cursor()
            minist_cursor.execute('''SELECT t_pers.prenom, t_pers.nom
                                       FROM t_com JOIN t_com_compets
                                         ON (t_com.id = t_com_compets.id)
                                       JOIN t_compet
                                         ON (t_com_compets.value = t_compet.id)
                                       JOIN t_pershistoline
                                         ON (t_compet.id = t_pershistoline.description)
                                       JOIN t_pers ON (t_pershistoline.pers = t_pers.id)
                                      WHERE t_compet.st = 'S_ACTIVE'
                                        AND t_pershistoline.fin IS NULL
                                        AND t_com.id = %(id)s
                                   GROUP BY t_pers.prenom, t_pers.nom''',
                                     {'id': com_id})
            object.ministries = []
            while True:
                row = minist_cursor.fetchone()
                if row is None:
                    break
                firstname, lastname = row
                object.ministries.append(RelationValue(
                                        self.get_author_intid_from_name(
                                                firstname, lastname, 'S_MINISTRE')))
            minist_cursor.close()

            notify(ObjectModifiedEvent(object))
            if object.active:
                self.publish(object)
            else:
                self.unpublish(object)

        cursor.close()
        return count


    def sync_documents(self, timestamp=None):
        cursor = self.db_connection.cursor()

        polgroups_id_to_title = self.get_polgroups_id_to_title(cursor)

        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('AND t_document.ts > %s', (timestamp,))
        cursor.execute('''SELECT t_document.id, type, sess, date, no, nodoc,
                                 intit, auteurs, matiereids, comppols, imageid,
                                 textdefts, textprovts
                            FROM t_document
                            JOIN t_typedoc
                              ON (t_document.type = t_typedoc.id)
                           WHERE t_document.st = 'S_INDEXED' AND
                                 t_typedoc.finet = 't' AND
                                 (t_document.textprovts is not NULL OR
                                  t_document.textdefts is not NULL)
                                 %s''' % where_ts)

        count = 0
        type_info = self.portal_types.getTypeInfo('tabellio.documents.document')
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            (doc_id, doctype, sess, date, no, nodoc, intit, authors, topics,
             polgroups, image_id, textprovts, textdefts) = row

            if intit is None:
                intit = {'RAPP': u'Rapport',
                         'AMENDCOM': u'Amendement en commission',
                         'AMENDSCE': u'Amendement en séance',
                         'CRA': u'Compte rendu analytique',
                         'ACA': u'''Avis d'une commission annexe''',
                         'RP-RACT': u'''Rapport d'activité''',
                         'BQR': u'Bulletin des questions et des réponses'
                        }.get(doctype)

                if intit is None:
                    print 'missing intit, skipping', doc_id, ' (type was %s)' % doctype
                    continue

            print 'doc id:', doc_id

            new_one = False
            if not hasattr(self.documents_folder, doc_id):
                self.documents_folder.invokeFactory('tabellio.documents.document', doc_id)
            object = getattr(self.documents_folder, doc_id)
            object.title = intit
            object.session = sess
            object.date = date
            object.no = no
            object.nodoc = nodoc
            object.doctype = doctype
            object.file_image_id = image_id
            object.topics = topics
            if textdefts:
               object.final_version = True
               object.publication_date = textdefts
            elif textprovts:
               object.final_version = False
               object.publication_date = textprovts
            if authors:
                authors = [self.get_author_intid(x) for x in authors]
                object.authors = [RelationValue(x) for x in authors if x]
                if len(object.authors) != len(authors):
                    print '  different number of authors'
            else:
                object.authors = []
            if polgroups:
                polgroups = [self.get_polgroup_intid(polgroups_id_to_title[x]) for x in polgroups]
                object.polgroups = [RelationValue(x) for x in polgroups]
            else:
                object.polgroups = []

            notify(ObjectModifiedEvent(object))
            self.publish(object)

            if count % 100 == 0:
                transaction.commit()

        cursor.close()
        return count

    def sync_dossiers(self, timestamp=None):
        cursor = self.db_connection.cursor()

        polgroups_id_to_title = self.get_polgroups_id_to_title(cursor)

        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('''AND EXISTS (SELECT ts FROM t_histoline
                                                       WHERE idhisto = t_histo.id
                                                         AND ts > %s)''', (timestamp,))

        ### Handle t_dossiers
        cursor.execute('''SELECT t_histo.id, t_histo.st, t_histo.type,
                                 t_histo.sess, t_histo.no, t_histo.intit,
                                 t_histo.auteurs,
                                 t_histo.curcom, t_histo.comppols,
                                 t_histo.matiereids
                            FROM t_histo
                           WHERE t_histo.intit IS NOT NULL
                             AND t_histo.type NOT IN ('QA', 'QE', 'QO', 'INTERP')
                                 %s''' % where_ts)

        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            dos_id, state, dostype, sess, no, intit, authors, curcom, polgroups, topics = row

            print 'dos id:', dos_id

            new_one = False
            if not hasattr(self.dossiers_folder, dos_id):
                self.dossiers_folder.invokeFactory('tabellio.documents.dossier', dos_id)
            object = getattr(self.dossiers_folder, dos_id)
            object.title = intit
            object.state = state
            object.session = sess
            object.no = no
            object.dostype = dostype
            object.topics = topics
            if authors:
                authors = [self.get_author_intid(x) for x in authors]
                object.authors = [RelationValue(x) for x in authors if x]
                if len(object.authors) != len(authors):
                    print '  different number of authors'
            else:
                object.authors = []

            object.commissions = [RelationValue(self.get_commission_intid(x)) for x in curcom or []]

            if polgroups:
                polgroups = [self.get_polgroup_intid(polgroups_id_to_title[x]) for x in polgroups]
                object.polgroups = [RelationValue(x) for x in polgroups]
            else:
                object.polgroups = []

            self.sync_dossier_history(object)
            if object.histolines:
                object.date = object.histolines[0].date

            notify(ObjectModifiedEvent(object))
            self.publish(object)

            if count % 100 == 0:
                transaction.commit()

        cursor.close()
        return count


    def sync_questions(self, timestamp=None):
        cursor = self.db_connection.cursor()

        polgroups_id_to_title = self.get_polgroups_id_to_title(cursor)
        ### Handle t_qx
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('''AND EXISTS (SELECT ts FROM t_histoline
                                                       WHERE idhisto = t_histo.id
                                                         AND ts > %s)''', (timestamp,))

        cursor.execute('''SELECT t_histo.id, t_histo.st, t_histo.type, t_histo.sess,
                                 t_histo.no, t_histo.intit, t_histo.auteurs,
                                 t_histo.comppols, t_histo.matiereids
                            FROM t_histo
                           WHERE t_histo.intit IS NOT NULL
                             AND t_histo.type IN ('QA', 'QE', 'QO', 'INTERP')
                                 %s''' % where_ts)

        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            dos_id, state, dostype, sess, no, intit, authors, polgroups, topics = row
            print 't_qx/dos id:', dos_id
            if not hasattr(self.questions_folder, dos_id):
                self.questions_folder.invokeFactory('tabellio.documents.question', dos_id)
            object = getattr(self.questions_folder, dos_id)
            if intit and '\n' in intit:
                intit = intit.replace('\n', ' ').replace('\r', '')
            object.title = intit
            object.session = sess
            object.state = state
            object.no = no
            object.questype = dostype
            object.topics = topics

            if authors:
                authors = [self.get_author_intid(x) for x in authors]
                object.authors = [RelationValue(x) for x in authors if x]
                if len(object.authors) != len(authors):
                    print '  different number of authors'
            else:
                object.authors = []

            if polgroups:
                polgroups = [self.get_polgroup_intid(polgroups_id_to_title[x]) for x in polgroups]
                object.polgroups = [RelationValue(x) for x in polgroups]
            else:
                object.polgroups = []

            self.sync_dossier_history(object)
            if object.histolines:
                object.date = object.histolines[0].date

            notify(ObjectModifiedEvent(object))
            self.publish(object)

            if count % 100 == 0:
                transaction.commit()

        cursor.close()
        return count


    def sync_dossier_history(self, object):
        hist_cursor = self.db_connection.cursor()
        # timestamp is not considered here as the field value is
        # reconstructured anew.
        hist_cursor.execute('''SELECT t_histoline.date, t_histoline.descr,
                                 t_histoline.auteurs, t_document.id, docpages,
                                 t_histoline.comment, t_histoline.com,
                                 t_histoline.views
                            FROM t_histoline LEFT JOIN t_document
                              ON (t_histoline.docno = t_document.no AND
                                  t_histoline.docsess = t_document.sess AND
                                  t_histoline.doctype = t_document.type AND
                                  ((t_histoline.docnodoc = t_document.nodoc) OR
                                   (t_histoline.docnodoc is NULL AND
                                    t_document.nodoc is NULL)) AND
                                  ((t_histoline.docanx = t_document.anx) OR
                                   (t_histoline.docanx is NULL AND t_document.anx is NULL)))
                           WHERE t_histoline.idhisto = %(objid)s
                        ORDER BY t_histoline.date, t_histoline.ordre, t_histoline.id''',
                              {'objid': object.id})
        object.reset_histolines()
        has_decret = False
        while True:
            histo_row = hist_cursor.fetchone()
            if histo_row is None:
                break
            date, desc, authors, docid, docpages, comment, comid, views = histo_row
            if not views:
                continue
            if 'TBC' in views:
                pass # this line is ok, we will take it
            elif 'ARL' in views:
                # special handling here
                acceptable_descrs = (u"Arrêt de la Cour d'Arbitrage",
                                u"Arrêté d'application",
                                u"Erratum",
                                u"Sanction et promulgation",
                                u"Publication au Moniteur belge",
                                u"Décret",
                                u"Décret modifié",
                                u"Décret modifiant")
                acceptable_comments = (u"Arrêt de la Cour d'Arbitrage",
                                u"Arrêté d'application",
                                u"Erratum",
                                u"Publication au Moniteur belge")
                for s in acceptable_descrs:
                    if desc and desc.startswith(s):
                        break
                else:
                    for s in acceptable_comments:
                        if comment and comment.startswith(s):
                            break
                    else:
                        # description and comment didn't match our special
                        # cases, do not include line
                        continue
            else:
                continue
            if authors:
                authors = [self.get_author_intid(x) for x in authors]
                authors = [RelationValue(x) for x in authors if x]
            else:
                authors = []
            if docid:
                document, docintid = self.get_document_and_intid(docid)
            else:
                document, docintid = (None, None)

            commission_title = None
            if comid:
                commission_title = self.get_commission_title(comid)

            # decide on the text to attach to that line
            text = desc
            if desc == u'Vote' and comment:
                text = u'%s - %s' % (desc, comment)
            elif has_decret:
                text = comment
            elif desc == u'Décret':
                has_decret = True
            object.add_histoline(date, text, authors, commission_title,
                                 docintid, document, docpages)

    def sync_reunions(self, timestamp=None):
        cursor = self.db_connection.cursor()
        where_ts = ''
        if timestamp:
            where_ts = cursor.mogrify('t_reunion.ts > %s', (timestamp,))
        cursor.execute('''SELECT id, type, comid, sess, lieu, datedeb, heuredeb, datefin, heurefin,
                                 nobtr, nobtrcom, nocri, nocric, nocricom,
                                 debreel, finreel
                            FROM t_reunion
                           WHERE
                                 %s''' % where_ts)

        # unfortunately this forces the locale
        formatter = locales.getLocale('fr').dates.getFormatter('dateTime', 'medium')
        formatter.setPattern(u'd MMMM yyyy à HH:mm')

        count = 0
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            count += 1
            reu_id, type, comid, sess, lieu, datedeb, heuredeb, datefin, \
                    heurefin, notr, nobtrcom, nocri, nocric, nocricom, \
                    debreel, finreel = row

            commission = None
            if comid:
                commission = self.get_commission(comid)

            if datedeb:
                if heuredeb:
                    start = datetime.datetime.strptime(
                                    '%s %s' % (datedeb, heuredeb),
                                    '%Y-%m-%d %H:%M:%S')
                else:
                    start = datetime.datetime.strptime(
                                    '%s' % datedeb, '%Y-%m-%d')

            title = None
            if type == 'SE':
                title = u'Séance plénière'
                basetype = 'tabellio.agenda.parlevent'
            elif type == 'CP':
                title = u'Commission plénière'
                basetype = 'tabellio.agenda.comevent'
            elif type == 'CO':
                if not comid:
                    title = u'Commission'
                else:
                    title = u'%s' % commission.title
                basetype = 'tabellio.agenda.comevent'

            if not hasattr(self.parlevents_folder, reu_id):
                self.parlevents_folder.invokeFactory(basetype, reu_id, title=title)
            object = getattr(self.parlevents_folder, reu_id)
            if not (debreel and object.title):
                # do not set the title if it was already set and the real
                # beginning time has been set, to keep titles with nice at
                # the hour times.
                object.title = title
            if lieu == u'Hémicycle':
                lieu = u'Hôtel de Ligne - Hémicycle'
            elif lieu:
                lieu = lieu.replace(u'HL ', u'Hôtel de Ligne - ')
                lieu = lieu.replace(u'HG ', u'Hôtel du Greffe - ')
            object.place = lieu
            object.session = sess
            object.start = start
            if datefin:
                if heurefin:
                    object.end = datetime.datetime.strptime(
                                    '%s %s' % (datefin, heurefin),
                                    '%Y-%m-%d %H:%M:%S')
                else:
                    object.end = datetime.datetime.strptime(
                                    '%s' % datefin, '%Y-%m-%d')
                    if object.end < object.start:
                        object.end = object.start

            if comid:
                comid_int = self.get_commission_intid(comid)
                if comid_int:
                    object.commission = RelationValue(comid_int)
            else:
                object.commission = None

            if (nocri or nocric or nocricom):
                if nocri:
                    doctype = 'CRI'
                    docno = nocri
                elif nocric:
                    doctype = 'CRICOM'
                    docno = '%s-%s%s' % (nocric, commission.code, nocricom)
                doc_cursor = self.db_connection.cursor()
                doc_cursor.execute(doc_cursor.mogrify('''SELECT id, sess
                                        FROM t_document
                                       WHERE sess = %s
                                         AND type = %s
                                         AND no = %s''', (sess, doctype, docno)))
                row = doc_cursor.fetchone()
                doc_cursor.close()
                if row is None:
                    print 'failed to get document', sess, doctype, docno
                else:
                    doc_id, docsess = row
                    doc, doc_intid = self.get_document_and_intid(doc_id)
                    object.cri = RelationValue(doc_intid)

            notify(ObjectModifiedEvent(object))

            self.publish(object)

            if count % 100 == 0:
                transaction.commit()

        cursor.close()
        return count

    def sync_deptables(self, ignored_timestamp=None):
        # the different tables listing specific deputies
        cursor = self.db_connection.cursor()

        # bureau
        cursor.execute('''SELECT nom, prenom, description
                            FROM t_pershistoline
                            JOIN t_pers
                              ON (t_pershistoline.pers = t_pers.id)
                           WHERE type = 'P_FONC' and fin is null''')
        president = None
        vicepresidents = []
        secretaires = []

        def get_link(nom, prenom):
            if nom and prenom:
                fullname = '%s %s' % (prenom, nom)
            else:
                fullname = nom
            pers_id = self.plone_utils.normalizeString(fullname)
            return '<a href="/%s/%s">%s</a>' % (
                            self.settings.deputiesPath,
                            pers_id,
                            fullname)

        while True:
            row = cursor.fetchone()
            if row is None:
                break
            nom, prenom, description = row
            if description.startswith('P'):
                president = (nom, prenom)
            elif 'sident' in description:
                vicepresidents.append((description, nom, prenom))
            elif 'taire' in description:
                secretaires.append((description, nom, prenom))

        vicepresidents.sort()
        secretaires.sort()
        table =  u'<table>'
        table += u'<tr><td>Président</td><td>%s</td></tr>' % get_link(president[0], president[1])
        for vicepresident in vicepresidents:
            table += u'<tr><td>%s</td><td>%s</td></tr>' % (
                            vicepresident[0], get_link(vicepresident[1], vicepresident[2]))
        secretaire = secretaires[0]
        table += u'<tr><td>Secrétaires du Parlement</td><td><ul>'
        if len(secretaires) > 1:
            for secretaire in secretaires[:-1]:
                table += '<li>%s</li>' % get_link(secretaire[1], secretaire[2])
        table += u'<li>%s</li></ul></td></tr>' % get_link(secretaires[-1][1], secretaires[-1][2])
        conf_pres_table = table

        table += u'''<tr><td>Greffier (Secrétaire général)</td>
                     <td>%s<br /><br />
                         %s<br />
                         %s<br />
                         <br />
                         Courriel : %s<br />
                         Téléphone : %s<br />
                         Fax : %s
                     </td></tr>''' % (
                        self.settings.greffier_name,
                        self.settings.greffier_address_line1,
                        self.settings.greffier_address_line2,
                        self.settings.greffier_email,
                        self.settings.greffier_phone,
                        self.settings.greffier_fax)

        table += u'</table>'

        regex = re.compile(r'<table.*<\/table>', re.DOTALL)
        article = getattr(self.deputies_folder, 'bureau')
        orig_text = article.getText()
        if type(orig_text) is not unicode:
            orig_text = unicode(orig_text, 'utf-8')
        article.setText(regex.sub(table, orig_text), mimetype='text/html')

        # conférence des présidents
        cursor.execute('''SELECT nom, prenom, description
                            FROM t_pershistoline
                            JOIN t_pers
                              ON (t_pershistoline.pers = t_pers.id)
                           WHERE type = 'P_PRES' and fin is null
                        ORDER BY nom''')

        presgroups = []
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            nom, prenom, description = row
            presgroups.append((description, nom, prenom))

        presgroup = presgroups[0]
        table = conf_pres_table
        table += u'<tr><td>Présidents de groupe</td><td><ul>'
        if len(presgroups) > 1:
            for presgroup in presgroups[:-1]:
                table += u'<li>%s</li>' % get_link(presgroup[1], presgroup[2])
        table += u'<li>%s</li></ul></td></tr>' % get_link(presgroups[-1][1], presgroups[-1][2])


        table += u'<tr><td>Greffier (Secrétaire général)</td><td>%s</td></tr>' % self.settings.greffier_name

        table += u'</table>'

        article = getattr(self.deputies_folder, 'conference-des-presidents')
        orig_text = article.getText()
        if type(orig_text) is not unicode:
            orig_text = unicode(orig_text, 'utf-8')
        article.setText(regex.sub(table, orig_text), mimetype='text/html')


        # sénateurs de communauté
        cursor.execute('''SELECT nom, prenom
                            FROM t_pershistoline
                            JOIN t_pers
                              ON (t_pershistoline.pers = t_pers.id)
                           WHERE type = 'P_SNTR' and fin is null
                        ORDER BY nom''')

        table = u'<table><tr><th>Noms</th><th>Appartenance politique</th></tr>'
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            nom, prenom = row
            if nom and prenom:
                fullname = '%s %s' % (prenom, nom)
            else:
                fullname = nom
            pers_id = self.plone_utils.normalizeString(fullname)
            pers = getattr(self.deputies_folder, pers_id)
            polgroup = pers.polgroup.to_object.title

            table += u'<tr><td>%s</td><td>%s</td></tr>' % (
                            get_link(nom, prenom), polgroup)

        table += u'</table>'

        article = getattr(self.deputies_folder, 'senateurs-de-communaute')
        orig_text = article.getText()
        if type(orig_text) is not unicode:
            orig_text = unicode(orig_text, 'utf-8')
        article.setText(regex.sub(table, orig_text), mimetype='text/html')

        # anciens présidents
        cursor.execute('''SELECT nom, prenom, debut, fin
                            FROM t_pershistoline
                            JOIN t_pers
                              ON (t_pershistoline.pers = t_pers.id)
                           WHERE type = 'P_FONC' and description like 'P%'
                        ORDER BY debut DESC''')

        formatter = locales.getLocale('fr').dates.getFormatter('dateTime', 'medium')
        formatter.setPattern(u'd MMMM yyyy')

        table = u'<table>'
        while True:
            row = cursor.fetchone()
            if row is None:
                break
            nom, prenom, debut, fin = row
            if nom and prenom:
                fullname = '%s %s' % (prenom, nom)
            else:
                fullname = nom
            pers_id = self.plone_utils.normalizeString(fullname)

            if fin:
                period = u'%s → %s' % (formatter.format(debut), formatter.format(fin))
            else:
                period = u'%s → aujourd\'hui' % formatter.format(debut)

            if hasattr(self.deputies_folder, pers_id):
                table += u'<tr><td>%s</td><td>%s</td></tr>' % (
                            get_link(nom, prenom), period)
            else:
                table += u'<tr><td>%s</td><td>%s</td></tr>' % (fullname, period)

        table += u'</table>'

        article = getattr(self.deputies_folder, 'anciens-presidents')
        orig_text = article.getText()
        if type(orig_text) is not unicode:
            orig_text = unicode(orig_text, 'utf-8')
        article.setText(regex.sub(table, orig_text), mimetype='text/html')

        cursor.close()
        return 'OK'