themis.libellioimport/themis/libellioimport/migration.py

# -*- coding: utf-8 -*-

import cPickle
import os
import datetime
import cStringIO
import xml.etree.ElementTree as ET

from zope.app.intid.interfaces import IIntIds
from z3c.relationfield import RelationValue
from Products.Five.browser import BrowserView
from Products.CMFCore.utils import getToolByName
from plone.dexterity.factory import DexterityFactory
from plone.dexterity.utils import addContentToContainer
try:
    from plone.namedfile.file import NamedBlobFile as NamedFile
except ImportError:
    from plone.namedfile.file import NamedFile
try:
    from plone.namedfile.file import NamedBlobImage as NamedImage
except ImportError:
    from plone.namedfile.file import NamedImage
from Products.CMFCore.WorkflowCore import WorkflowException

from plone.registry.interfaces import IRegistry
from zope import component

from tabellio.config.interfaces import ITabellioSettings

class Migrate(BrowserView):
    def __call__(self):
        self.logfile = file('/tmp/migration.log', 'a+')
        if self.request['REQUEST_METHOD'] == 'GET':
            filename = self.request.form.get('filename')
            if not filename:
                return 'Missing filename\n'
            fd = file(os.path.join('/tmp/', filename))
        else:
            try:
                fd = cStringIO.StringIO(self.request.form['data'])
            except KeyError:
                return 'Missing data\n'

        doc = cPickle.load(fd)
        portal = getToolByName(self.context, 'portal_url').getPortalObject()
        workflow_tool = getToolByName(self.context, 'portal_workflow')
        plone_tool = getToolByName(self.context, 'plone_utils')

        maildir = getattr(self.context, 'all')

        print 'Processing', doc.get('id')
        print >> self.logfile, 'I: Processing', doc.get('id')

        if doc.get('meta_type') == 'PFBArbitralCourt' and doc.get('id').startswith('cour-d-arbitrage'):
            print >> self.logfile, 'I: skipped, arbitral court document'
            return 'Arbitral Court documents are not to be imported\n'

        # first step is to create a "courrier entrant" object
        typename = 'courrier_entrant'
        factory = DexterityFactory(portal_type=typename)

        if not hasattr(maildir, doc.get('id')):
            ob = factory(id=doc.get('id'), title=doc.get('title'))
            maildir._setObject(ob.id, ob)

        ob = getattr(maildir, doc.get('id'))

        ob.date_reception = datetime.datetime.strptime(doc.get('deliverydate'), '%Y-%m-%d').date()
        ob.numero_courrier = doc.get('mailnumber')

        if ob.date_reception < (datetime.datetime.today()-datetime.timedelta(days=90)).date():
            # all documents created more than three months ago are considered
            # done.
            doc['state'] = 'filed'

        ob.fichier = NamedFile(fd.read(), filename=unicode(doc.get('filename')))

        ob.categorie_de_courrier = [{
            'PFBActivityAndExpertReport': u'''Rapport d'activités/d'experts''',
            'PFBAddressChange': u'Changement de coordonnées',
            'PFBArbitralCourt': u'Relations publiques',
            'PFBAuditingCommitteeCommunication': u'Communication gouvernementale',
            'PFBChallenge': u'Interpellation',
            'PFBCollegialNotification': u'Notifications du Collège',
            'PFBCurrentEventsQuestion': u'''Question d'actualité''',
            'PFBGeneralCommission': u'Commissions divers',
            'PFBGeneralMail': u'Courrier général divers',
            'PFBMeetingExcuses': u'Excusés',
            'PFBOralRequest': u'Question orale',
            'PFBProject': u'Projet',
            'PFBProposal': u'Proposition',
            'PFBSubventionRequest': u'Demande de subvention',
            'PFBWriteAnswer': u'Réponse écrite',
            'PFBWriteRequest': u'Question écrite',
        }.get(doc.get('meta_type'))]

        if ob.categorie_de_courrier is None:
            print >> self.logfile, 'W: failed to get a category'

        ob.expediteur = []
        shipper_id = None
        if doc.get('shippers'):
            for shipper in doc.get('shippers'):
                shipper = unicode(shipper, 'utf-8')
                shipper_id = plone_tool.normalizeString(shipper)
                if not portal.contacts.has_key(shipper_id):
                    portal.contacts.invokeFactory('themis.datatypes.contact', shipper_id, title=shipper)
                shipper_id = 'contact:' + shipper_id
                ob.expediteur.append(shipper_id)
        else:
            if doc.get('meta_type') in ('PFBWriteAnswer', 'PFBProject'):
                # shipper is a ministry
                if doc.get('authors'):
                    for author in doc.get('authors'):
                        shipper_id = self.get_ministry(author)
                        ob.expediteur.append(shipper_id)
                else:
                    print >> self.logfile, 'W: document without an author'
            elif doc.get('meta_type') in ('PFBChallenge',
                            'PFBCurrentEventsQuestion', 'PFBOralRequest',
                            'PFBProposal', 'PFBWriteRequest'):
                # shipper is a deputy
                if doc.get('authors'):
                    for author in doc.get('authors'):
                        shipper_id = self.get_deputy(author)
                        ob.expediteur.append(shipper_id)
                else:
                    print >> self.logfile, 'W: document without an author'

        if doc.get('type'):
            ob.sous_categorie_de_courrier = {
                'proj_decret': u'Projet de décret',
                'proj_reglement': u'Projet de règlement',
                'prop_decret': u'Proposition de décret',
                'prop_reglement': u'Proposition de règlement',
                'prop_resolution': u'Proposition de résolution',
                'prop_modif_regl': u'Proposition de modification du Règlement du PFB',
                'prop_modif_statut': u'Proposition de modification du statut du personnel'
            }.get(doc.get('type'))

        if doc.get('state') in ('filed', 'preDocumented'):
            try:
                workflow_tool.doActionFor(ob, 'publish')
            except WorkflowException:
                pass
            except Exception, e:
                print >> self.logfile, 'E: Exception %r' % e
                raise

        second_object_typename = {
            'PFBChallenge': 'interpellationD',
            'PFBWriteAnswer': 'reponse_a_question_ecriteD',
            'PFBWriteRequest': 'Question_ecriteD',
            'PFBCurrentEventsQuestion': 'questionactualiteD',
            'PFBOralRequest': 'QuestionoraleD',
            'PFBProject': 'ProjetD',
            'PFBProposal': 'PropositionD',
        }.get(doc.get('meta_type'))

        if not doc.get('authors') and doc.get('state') == 'initial':
            # this document was not completed, do not create a secondary object
            second_object_typename = None

        if second_object_typename and not ob.numero_courrier:
            # if there's no mailnumber, do not create a secondary object
            print >> self.logfile, 'W: document without a mail number'
            second_object_typename = None

        if second_object_typename:
            docdir = getattr(self.context.aq_parent, 'documents-diffusables')
            docdir = getattr(docdir, 'all')

            if not hasattr(docdir, doc.get('id')):
                factory = DexterityFactory(portal_type=second_object_typename)
                ob2 = factory(id=doc.get('id'), title=doc.get('title'))
                docdir._setObject(ob2.id, ob2)

            ob2 = getattr(docdir, doc.get('id'))

            ob2.date_reception = datetime.datetime.strptime(doc.get('deliverydate'), '%Y-%m-%d').date()
            ob2.mail_ref_id = ob.numero_courrier
            ob2.session = doc.get('session')

            if second_object_typename == 'reponse_a_question_ecriteD':
                if doc.get('authors'):
                    if len(doc.get('authors')) > 1:
                        print >> self.logfile, 'W: answer to written question with more than one author'
                    ob2.ministre_auteur_reponse = self.get_ministry(doc.get('authors')[0])
            elif second_object_typename in ('interpellationD', 'Question_ecriteD',
                            'questionactualiteD', 'QuestionoraleD'):
                if doc.get('authors'):
                    ob2.auteur = [self.get_deputy(x) for x in doc.get('authors')]
                if doc.get('recipients'):
                    ob2.ministres_concernes = [self.get_ministry(x) for x in doc.get('recipients')]
            elif second_object_typename == 'ProjetD':
                ob2.type_de_projet = ob.sous_categorie_de_courrier
                if doc.get('authors'):
                    if len(doc.get('authors')) > 1:
                        print >> self.logfile, 'W: project with more than one author'
                    ob2.auteur = [self.get_ministry(x) for x in doc.get('authors')][0]
            elif second_object_typename == 'PropositionD':
                ob2.type_de_proposition = ob.sous_categorie_de_courrier
                if doc.get('authors'):
                    ob2.auteurs = [self.get_deputy(x) for x in doc.get('authors')]

            if False and doc.get('state') in ('filed', 'preDocumented'):
                try:
                    workflow_tool.doActionFor(ob2, 'publish')
                except WorkflowException:
                    pass
                except Exception, e:
                    print >> self.logfile, 'E: Exception(b) %r' % e
                    raise



        return u'→ OK\n'

    def get_ministry(self, author):
        if not author:
            return None
        plone_tool = getToolByName(self.context, 'plone_utils')
        portal = getToolByName(self.context, 'portal_url').getPortalObject()
        author = unicode(author, 'utf-8')
        author = {
                u'de Donn\xe9a Fran\xe7ois-Xavier': u'de Donn\xe9a, Fran\xe7ois-Xavier',
                u'Doulkeridis Christos': u'Doulkeridis, Christos',
                }.get(author, author)
        author_id = plone_tool.normalizeString(author)
        if author_id == 'college':
            return 'ministry:college'
        if not portal.ministres.has_key(author_id):
            workflow_tool = getToolByName(self.context, 'portal_workflow')
            try:
                lastname, firstname = author.split(',')
            except ValueError:
                print >> self.logfile, 'E: ministry??? (%r)' % author
                raise
            portal.ministres.invokeFactory('themis.datatypes.ministry', author_id,
                            firstname=firstname.strip(),
                            lastname=lastname.strip())
            ministry = getattr(portal.ministres, author_id)
            try:
                workflow_tool.doActionFor(ministry, 'publish')
            except WorkflowException:
                pass

        return 'ministry:'+author_id

    def get_deputy(self, author):
        plone_tool = getToolByName(self.context, 'plone_utils')
        portal = getToolByName(self.context, 'portal_url').getPortalObject()
        author = unicode(author, 'utf-8')
        author = {
                u'Brotchi Jacques': u'Brotchi, Jacques',
                u'De Bock Emmanuel': u'De Bock, Emmanuel',
                u'Defoss\xe9 Jean-Claude': u'Defoss\xe9, Jean-Claude',
                u"d'Ursel Anne Charlotte": u"d'Ursel, Anne Charlotte",
                u'El Khannouss Ahmed': u'El Khannouss, Ahmed',
                u'Ikazban Jamal': u'Ikazban, Jamal',
                u'Lurquin Vincent': u'Lurquin, Vincent',
                u'Mandaila Gis\xe8le': u'Mandaila, Gis\xe8le',
                u'Maron Alain': u'Maron, Alain',
                u'Migisha Pierre': u'Migisha, Pierre',
                u'Morel Jacques': u'Morel, Jacques',
                u'Mouhssin Ahmed': u'Mouhssin, Ahmed',
                u'Moureaux Catherine': u'Moureaux, Catherine',
                u'Ouriaghli Mohamed': u'Ouriaghli, Mohamed',
                u'Ozdemir Mahinur': u'Ozdemir, Mahinur',
                u'Pinxteren Arnaud': u'Pinxteren, Arnaud',
                u'Sessler Patrick': u'Sessler, Patrick',
                u'Sidibe Fatoumata': u'Sidibe, Fatoumata',
                u'Trachte Barbara': u'Trachte, Barbara',
                u'Van Goidsenhoven Ga\xebtan': u'Van Goidsenhoven, Ga\xebtan',
                u'Vanhalewyn Vincent': u'Vanhalewyn, Vincent',
                u'Willame Boonen Magdeleine': u'Willame Boonen,  Magdeleine',
                u'Govers, Pascale, ': u'Govers, Pascale',
                }.get(author, author)
        author_id = plone_tool.normalizeString(author)
        if not portal.deputes.has_key(author_id):
            workflow_tool = getToolByName(self.context, 'portal_workflow')
            try:
                lastname, firstname = author.split(',')
            except ValueError:
                print >> self.logfile, 'E: deputy??? (%r)' % author
                raise
            portal.deputes.invokeFactory('themis.datatypes.deputy', author_id,
                            firstname=firstname.strip(),
                            lastname=lastname.strip())
            deputy = getattr(portal.deputes, author_id)
            try:
                workflow_tool.doActionFor(deputy, 'publish')
            except WorkflowException:
                pass

        return 'deputy:'+author_id

class ImportDeputies(BrowserView):
    def __call__(self):
        self.settings = component.getUtility(IRegistry).forInterface(ITabellioSettings, False)
        self.portal = getToolByName(self.context, 'portal_url').getPortalObject()
        self.plone_utils = getToolByName(self.context, 'plone_utils')
        self.portal_workflow = getToolByName(self.context, 'portal_workflow')
        self.intids = component.getUtility(IIntIds)
        self.portal_types = getToolByName(self.context, 'portal_types')

        path = self.request.form.get('path')
        if not path:
            path = '/tmp/Datas_pertinentes_sitePFBactu.xml'
        base_picdir = os.path.join(os.path.dirname(path), 'pictures')

        plone_tool = getToolByName(self.context, 'plone_utils')

        doc = ET.parse(path)

        # get mapping for political groups
        polgroups = {}
        for d in doc.findall('groupesPolitiques'):
            polgroup_id = d.find('grpPolId').text
            polgroup_name = d.find('grpPolName').text
            polgroups[polgroup_id] = polgroup_name

        for d in self.deputies_folder.objectValues():
            d.active = False

        # import deputies
        for d in doc.findall('parlementaires'):
            firstname = d.find('parlFirstName').text
            lastname = d.find('parlLastName').text
            parlname = u'%s %s' % (lastname, firstname)
            parl_id = plone_tool.normalizeString(parlname)
            if not hasattr(self.deputies_folder, parl_id):
                print 'Created deputy:', firstname, lastname
                self.deputies_folder.invokeFactory('themis.datatypes.deputy',
                                parl_id, firstname=firstname, lastname=lastname)
            object = getattr(self.deputies_folder, parl_id)
            day, month, year = d.find('parlBirthday').text.split('-')
            object.active = True
            object.birthdate = datetime.datetime(int(year), int(month), int(day))
            object.birthplace = d.find('parlLocBirthDay').text
            object.polgroup = RelationValue(self.get_polgroup_intid(
                polgroups.get(d.find('grpPolId').text)))
            pic_filename = d.find('parlPicture').text
            pic_filepath = os.path.join(base_picdir, pic_filename)
            if os.path.exists(pic_filepath):
                fd = file(pic_filepath)
                object.picture = NamedImage(fd.read(), filename=unicode(pic_filename))
                fd.close()


    _polgroups_folder = None
    def polgroups_folder(self):
        if self._polgroups_folder:
            return self._polgroups_folder
        path = self.settings.polgroupsPath
        self._polgroups_folder = self.get_folder_at_path(path)
        return self._polgroups_folder
    polgroups_folder = property(polgroups_folder)


    _polgroups_intids = None
    def get_polgroup_intid(self, title):
        if not self._polgroups_intids:
            self._polgroups_intids = {}

        if title in self._polgroups_intids:
            return self._polgroups_intids.get(title)

        polgroup_id = self.plone_utils.normalizeString(title)
        if not hasattr(self.polgroups_folder, polgroup_id):
            self.polgroups_folder.invokeFactory('themis.datatypes.polgroup', polgroup_id, title=title)
            try:
                self.portal_workflow.doActionFor(getattr(self.polgroups_folder, polgroup_id), 'publish')
            except WorkflowException:
                pass
        polgroup_intid = self.intids.getId(getattr(self.polgroups_folder, polgroup_id))
        self._polgroups_intids[title] = polgroup_intid
        return polgroup_intid

    _deputies_folder = None
    def deputies_folder(self):
        if self._deputies_folder:
            return self._deputies_folder
        path = self.settings.deputiesPath
        self._deputies_folder = self.get_folder_at_path(path)
        return self._deputies_folder
    deputies_folder = property(deputies_folder)


    def get_folder_at_path(self, path):
        current = self.portal
        for part in path.split('/'):
            if not part:
                continue
            current = getattr(current, part)
        return current


class FixDeputies(BrowserView):
    def __call__(self):
        self.settings = component.getUtility(IRegistry).forInterface(ITabellioSettings, False)
        self.from_id = self.request.form.get('from')
        self.to_id = self.request.form.get('to')
        catalog = getToolByName(self.context, 'portal_catalog')
        for brain in catalog({'auteurs': self.from_id}):
            obj = brain.getObject()
            for attr in ('orateurs', 'orateurs_en_commission', 'auteurs',
                            'rapporteurs', 'orateurs_seance_reponse_orale',
                            'auteur', 'expediteur'):
                if hasattr(obj, attr) and type(getattr(obj, attr)) is list:
                    setattr(obj, attr, self.replace_in_list(getattr(obj, attr)))
            for attr in ('auteur',):
                if hasattr(obj, attr) and type(getattr(obj, attr)) is str:
                    if getattr(obj, attr) == self.from_id:
                        setattr(obj, attr, self.to_id)

    def replace_in_list(self, oldlist):
        newlist = []
        for item in oldlist:
            if item == self.from_id:
                newlist.append(self.to_id)
            else:
                newlist.append(item)
        return newlist

    _documents_folder = None
    def documents_folder(self):
        if self._documents_folder:
            return self._documents_folder
        path = self.settings.documentsPath
        self._documents_folder = self.get_folder_at_path(path)
        return self._documents_folder
    documents_folder = property(documents_folder)


class FixRelatedDocs(BrowserView):
    def __call__(self):
        catalog = getToolByName(self.context, 'portal_catalog')
        for brain in catalog.search({'review_state': 'pending'}):
            object = brain.getObject()
            for attr in ("bulletin_des_travaux",
                    "bulletin_questions_ecrites",
                    "courrier_reponse_accuse_de_reception",
                    "cr_debats_declaration_politique_generale",
                    "cr_debats_declaration_programme",
                    "cr_reponse_orale",
                    "cr_seance",
                    "cr_seance_ou_biq",
                    "cr_seance_pleniere",
                    "cr_seance_prise_en_consideration",
                    "cr_seance_vote_motion",
                    "cr_vote_declaration_programme",
                    "documents_lies",
                    "lien_qo_ou_qe",
                    "rapport",
                    "rapport_de_commission",
                    "lien_vers_interpellation_ou_qe",
                    "reponse"):
                try:
                    v = getattr(object, attr)
                except AttributeError:
                    continue
                if v is None:
                    continue
                if type(v) is list:
                    continue
                setattr(object, attr, [v])


class UpdateToBlobs(BrowserView):
    def __call__(self):
        catalog = getToolByName(self.context, 'portal_catalog')
        for brain in catalog.search({'portal_type': ['bqrD', 'bulletintravauxD']}):
            object = brain.getObject()
            for attr in ("fichier", "document_imprime"):
                try:
                    v = getattr(object, attr)
                except AttributeError:
                    continue
                if v is None:
                    continue
                print 'transforming file in', object.id
                newv = NamedFile(v.data, filemane=v.filename)
                setattr(object, attr, newv)