# -*- coding: utf-8 -*- import cPickle import os import datetime import cStringIO import xml.etree.ElementTree as ET from zope.app.intid.interfaces import IIntIds from z3c.relationfield import RelationValue from Products.Five.browser import BrowserView from Products.CMFCore.utils import getToolByName from plone.dexterity.factory import DexterityFactory from plone.dexterity.utils import addContentToContainer try: from plone.namedfile.file import NamedBlobFile as NamedFile except ImportError: from plone.namedfile.file import NamedFile try: from plone.namedfile.file import NamedBlobImage as NamedImage except ImportError: from plone.namedfile.file import NamedImage from Products.CMFCore.WorkflowCore import WorkflowException from plone.registry.interfaces import IRegistry from zope import component from tabellio.config.interfaces import ITabellioSettings class Migrate(BrowserView): def __call__(self): self.logfile = file('/tmp/migration.log', 'a+') if self.request['REQUEST_METHOD'] == 'GET': filename = self.request.form.get('filename') if not filename: return 'Missing filename\n' fd = file(os.path.join('/tmp/', filename)) else: try: fd = cStringIO.StringIO(self.request.form['data']) except KeyError: return 'Missing data\n' doc = cPickle.load(fd) portal = getToolByName(self.context, 'portal_url').getPortalObject() workflow_tool = getToolByName(self.context, 'portal_workflow') plone_tool = getToolByName(self.context, 'plone_utils') maildir = getattr(self.context, 'all') print 'Processing', doc.get('id') print >> self.logfile, 'I: Processing', doc.get('id') if doc.get('meta_type') == 'PFBArbitralCourt' and doc.get('id').startswith('cour-d-arbitrage'): print >> self.logfile, 'I: skipped, arbitral court document' return 'Arbitral Court documents are not to be imported\n' # first step is to create a "courrier entrant" object typename = 'courrier_entrant' factory = DexterityFactory(portal_type=typename) if not hasattr(maildir, doc.get('id')): ob = factory(id=doc.get('id'), title=doc.get('title')) maildir._setObject(ob.id, ob) ob = getattr(maildir, doc.get('id')) ob.date_reception = datetime.datetime.strptime(doc.get('deliverydate'), '%Y-%m-%d').date() ob.numero_courrier = doc.get('mailnumber') if ob.date_reception < (datetime.datetime.today()-datetime.timedelta(days=90)).date(): # all documents created more than three months ago are considered # done. doc['state'] = 'filed' ob.fichier = NamedFile(fd.read(), filename=unicode(doc.get('filename'))) ob.categorie_de_courrier = [{ 'PFBActivityAndExpertReport': u'''Rapport d'activités/d'experts''', 'PFBAddressChange': u'Changement de coordonnées', 'PFBArbitralCourt': u'Relations publiques', 'PFBAuditingCommitteeCommunication': u'Communication gouvernementale', 'PFBChallenge': u'Interpellation', 'PFBCollegialNotification': u'Notifications du Collège', 'PFBCurrentEventsQuestion': u'''Question d'actualité''', 'PFBGeneralCommission': u'Commissions divers', 'PFBGeneralMail': u'Courrier général divers', 'PFBMeetingExcuses': u'Excusés', 'PFBOralRequest': u'Question orale', 'PFBProject': u'Projet', 'PFBProposal': u'Proposition', 'PFBSubventionRequest': u'Demande de subvention', 'PFBWriteAnswer': u'Réponse écrite', 'PFBWriteRequest': u'Question écrite', }.get(doc.get('meta_type'))] if ob.categorie_de_courrier is None: print >> self.logfile, 'W: failed to get a category' ob.expediteur = [] shipper_id = None if doc.get('shippers'): for shipper in doc.get('shippers'): shipper = unicode(shipper, 'utf-8') shipper_id = plone_tool.normalizeString(shipper) if not portal.contacts.has_key(shipper_id): portal.contacts.invokeFactory('themis.datatypes.contact', shipper_id, title=shipper) shipper_id = 'contact:' + shipper_id ob.expediteur.append(shipper_id) else: if doc.get('meta_type') in ('PFBWriteAnswer', 'PFBProject'): # shipper is a ministry if doc.get('authors'): for author in doc.get('authors'): shipper_id = self.get_ministry(author) ob.expediteur.append(shipper_id) else: print >> self.logfile, 'W: document without an author' elif doc.get('meta_type') in ('PFBChallenge', 'PFBCurrentEventsQuestion', 'PFBOralRequest', 'PFBProposal', 'PFBWriteRequest'): # shipper is a deputy if doc.get('authors'): for author in doc.get('authors'): shipper_id = self.get_deputy(author) ob.expediteur.append(shipper_id) else: print >> self.logfile, 'W: document without an author' if doc.get('type'): ob.sous_categorie_de_courrier = { 'proj_decret': u'Projet de décret', 'proj_reglement': u'Projet de règlement', 'prop_decret': u'Proposition de décret', 'prop_reglement': u'Proposition de règlement', 'prop_resolution': u'Proposition de résolution', 'prop_modif_regl': u'Proposition de modification du Règlement du PFB', 'prop_modif_statut': u'Proposition de modification du statut du personnel' }.get(doc.get('type')) if doc.get('state') in ('filed', 'preDocumented'): try: workflow_tool.doActionFor(ob, 'publish') except WorkflowException: pass except Exception, e: print >> self.logfile, 'E: Exception %r' % e raise second_object_typename = { 'PFBChallenge': 'interpellationD', 'PFBWriteAnswer': 'reponse_a_question_ecriteD', 'PFBWriteRequest': 'Question_ecriteD', 'PFBCurrentEventsQuestion': 'questionactualiteD', 'PFBOralRequest': 'QuestionoraleD', 'PFBProject': 'ProjetD', 'PFBProposal': 'PropositionD', }.get(doc.get('meta_type')) if not doc.get('authors') and doc.get('state') == 'initial': # this document was not completed, do not create a secondary object second_object_typename = None if second_object_typename and not ob.numero_courrier: # if there's no mailnumber, do not create a secondary object print >> self.logfile, 'W: document without a mail number' second_object_typename = None if second_object_typename: docdir = getattr(self.context.aq_parent, 'documents-diffusables') docdir = getattr(docdir, 'all') if not hasattr(docdir, doc.get('id')): factory = DexterityFactory(portal_type=second_object_typename) ob2 = factory(id=doc.get('id'), title=doc.get('title')) docdir._setObject(ob2.id, ob2) ob2 = getattr(docdir, doc.get('id')) ob2.date_reception = datetime.datetime.strptime(doc.get('deliverydate'), '%Y-%m-%d').date() ob2.mail_ref_id = ob.numero_courrier ob2.session = doc.get('session') if second_object_typename == 'reponse_a_question_ecriteD': if doc.get('authors'): if len(doc.get('authors')) > 1: print >> self.logfile, 'W: answer to written question with more than one author' ob2.ministre_auteur_reponse = self.get_ministry(doc.get('authors')[0]) elif second_object_typename in ('interpellationD', 'Question_ecriteD', 'questionactualiteD', 'QuestionoraleD'): if doc.get('authors'): ob2.auteur = [self.get_deputy(x) for x in doc.get('authors')] if doc.get('recipients'): ob2.ministres_concernes = [self.get_ministry(x) for x in doc.get('recipients')] elif second_object_typename == 'ProjetD': ob2.type_de_projet = ob.sous_categorie_de_courrier if doc.get('authors'): if len(doc.get('authors')) > 1: print >> self.logfile, 'W: project with more than one author' ob2.auteur = [self.get_ministry(x) for x in doc.get('authors')][0] elif second_object_typename == 'PropositionD': ob2.type_de_proposition = ob.sous_categorie_de_courrier if doc.get('authors'): ob2.auteurs = [self.get_deputy(x) for x in doc.get('authors')] if False and doc.get('state') in ('filed', 'preDocumented'): try: workflow_tool.doActionFor(ob2, 'publish') except WorkflowException: pass except Exception, e: print >> self.logfile, 'E: Exception(b) %r' % e raise return u'→ OK\n' def get_ministry(self, author): if not author: return None plone_tool = getToolByName(self.context, 'plone_utils') portal = getToolByName(self.context, 'portal_url').getPortalObject() author = unicode(author, 'utf-8') author = { u'de Donn\xe9a Fran\xe7ois-Xavier': u'de Donn\xe9a, Fran\xe7ois-Xavier', u'Doulkeridis Christos': u'Doulkeridis, Christos', }.get(author, author) author_id = plone_tool.normalizeString(author) if author_id == 'college': return 'ministry:college' if not portal.ministres.has_key(author_id): workflow_tool = getToolByName(self.context, 'portal_workflow') try: lastname, firstname = author.split(',') except ValueError: print >> self.logfile, 'E: ministry??? (%r)' % author raise portal.ministres.invokeFactory('themis.datatypes.ministry', author_id, firstname=firstname.strip(), lastname=lastname.strip()) ministry = getattr(portal.ministres, author_id) try: workflow_tool.doActionFor(ministry, 'publish') except WorkflowException: pass return 'ministry:'+author_id def get_deputy(self, author): plone_tool = getToolByName(self.context, 'plone_utils') portal = getToolByName(self.context, 'portal_url').getPortalObject() author = unicode(author, 'utf-8') author = { u'Brotchi Jacques': u'Brotchi, Jacques', u'De Bock Emmanuel': u'De Bock, Emmanuel', u'Defoss\xe9 Jean-Claude': u'Defoss\xe9, Jean-Claude', u"d'Ursel Anne Charlotte": u"d'Ursel, Anne Charlotte", u'El Khannouss Ahmed': u'El Khannouss, Ahmed', u'Ikazban Jamal': u'Ikazban, Jamal', u'Lurquin Vincent': u'Lurquin, Vincent', u'Mandaila Gis\xe8le': u'Mandaila, Gis\xe8le', u'Maron Alain': u'Maron, Alain', u'Migisha Pierre': u'Migisha, Pierre', u'Morel Jacques': u'Morel, Jacques', u'Mouhssin Ahmed': u'Mouhssin, Ahmed', u'Moureaux Catherine': u'Moureaux, Catherine', u'Ouriaghli Mohamed': u'Ouriaghli, Mohamed', u'Ozdemir Mahinur': u'Ozdemir, Mahinur', u'Pinxteren Arnaud': u'Pinxteren, Arnaud', u'Sessler Patrick': u'Sessler, Patrick', u'Sidibe Fatoumata': u'Sidibe, Fatoumata', u'Trachte Barbara': u'Trachte, Barbara', u'Van Goidsenhoven Ga\xebtan': u'Van Goidsenhoven, Ga\xebtan', u'Vanhalewyn Vincent': u'Vanhalewyn, Vincent', u'Willame Boonen Magdeleine': u'Willame Boonen, Magdeleine', u'Govers, Pascale, ': u'Govers, Pascale', }.get(author, author) author_id = plone_tool.normalizeString(author) if not portal.deputes.has_key(author_id): workflow_tool = getToolByName(self.context, 'portal_workflow') try: lastname, firstname = author.split(',') except ValueError: print >> self.logfile, 'E: deputy??? (%r)' % author raise portal.deputes.invokeFactory('themis.datatypes.deputy', author_id, firstname=firstname.strip(), lastname=lastname.strip()) deputy = getattr(portal.deputes, author_id) try: workflow_tool.doActionFor(deputy, 'publish') except WorkflowException: pass return 'deputy:'+author_id class ImportDeputies(BrowserView): def __call__(self): self.settings = component.getUtility(IRegistry).forInterface(ITabellioSettings, False) self.portal = getToolByName(self.context, 'portal_url').getPortalObject() self.plone_utils = getToolByName(self.context, 'plone_utils') self.portal_workflow = getToolByName(self.context, 'portal_workflow') self.intids = component.getUtility(IIntIds) self.portal_types = getToolByName(self.context, 'portal_types') path = self.request.form.get('path') if not path: path = '/tmp/Datas_pertinentes_sitePFBactu.xml' base_picdir = os.path.join(os.path.dirname(path), 'pictures') plone_tool = getToolByName(self.context, 'plone_utils') doc = ET.parse(path) # get mapping for political groups polgroups = {} for d in doc.findall('groupesPolitiques'): polgroup_id = d.find('grpPolId').text polgroup_name = d.find('grpPolName').text polgroups[polgroup_id] = polgroup_name for d in self.deputies_folder.objectValues(): d.active = False # import deputies for d in doc.findall('parlementaires'): firstname = d.find('parlFirstName').text lastname = d.find('parlLastName').text parlname = u'%s %s' % (lastname, firstname) parl_id = plone_tool.normalizeString(parlname) if not hasattr(self.deputies_folder, parl_id): print 'Created deputy:', firstname, lastname self.deputies_folder.invokeFactory('themis.datatypes.deputy', parl_id, firstname=firstname, lastname=lastname) object = getattr(self.deputies_folder, parl_id) day, month, year = d.find('parlBirthday').text.split('-') object.active = True object.birthdate = datetime.datetime(int(year), int(month), int(day)) object.birthplace = d.find('parlLocBirthDay').text object.polgroup = RelationValue(self.get_polgroup_intid( polgroups.get(d.find('grpPolId').text))) pic_filename = d.find('parlPicture').text pic_filepath = os.path.join(base_picdir, pic_filename) if os.path.exists(pic_filepath): fd = file(pic_filepath) object.picture = NamedImage(fd.read(), filename=unicode(pic_filename)) fd.close() _polgroups_folder = None def polgroups_folder(self): if self._polgroups_folder: return self._polgroups_folder path = self.settings.polgroupsPath self._polgroups_folder = self.get_folder_at_path(path) return self._polgroups_folder polgroups_folder = property(polgroups_folder) _polgroups_intids = None def get_polgroup_intid(self, title): if not self._polgroups_intids: self._polgroups_intids = {} if title in self._polgroups_intids: return self._polgroups_intids.get(title) polgroup_id = self.plone_utils.normalizeString(title) if not hasattr(self.polgroups_folder, polgroup_id): self.polgroups_folder.invokeFactory('themis.datatypes.polgroup', polgroup_id, title=title) try: self.portal_workflow.doActionFor(getattr(self.polgroups_folder, polgroup_id), 'publish') except WorkflowException: pass polgroup_intid = self.intids.getId(getattr(self.polgroups_folder, polgroup_id)) self._polgroups_intids[title] = polgroup_intid return polgroup_intid _deputies_folder = None def deputies_folder(self): if self._deputies_folder: return self._deputies_folder path = self.settings.deputiesPath self._deputies_folder = self.get_folder_at_path(path) return self._deputies_folder deputies_folder = property(deputies_folder) def get_folder_at_path(self, path): current = self.portal for part in path.split('/'): if not part: continue current = getattr(current, part) return current class FixDeputies(BrowserView): def __call__(self): self.settings = component.getUtility(IRegistry).forInterface(ITabellioSettings, False) self.from_id = self.request.form.get('from') self.to_id = self.request.form.get('to') catalog = getToolByName(self.context, 'portal_catalog') for brain in catalog({'auteurs': self.from_id}): obj = brain.getObject() for attr in ('orateurs', 'orateurs_en_commission', 'auteurs', 'rapporteurs', 'orateurs_seance_reponse_orale', 'auteur', 'expediteur'): if hasattr(obj, attr) and type(getattr(obj, attr)) is list: setattr(obj, attr, self.replace_in_list(getattr(obj, attr))) for attr in ('auteur',): if hasattr(obj, attr) and type(getattr(obj, attr)) is str: if getattr(obj, attr) == self.from_id: setattr(obj, attr, self.to_id) def replace_in_list(self, oldlist): newlist = [] for item in oldlist: if item == self.from_id: newlist.append(self.to_id) else: newlist.append(item) return newlist _documents_folder = None def documents_folder(self): if self._documents_folder: return self._documents_folder path = self.settings.documentsPath self._documents_folder = self.get_folder_at_path(path) return self._documents_folder documents_folder = property(documents_folder) class FixRelatedDocs(BrowserView): def __call__(self): catalog = getToolByName(self.context, 'portal_catalog') for brain in catalog.search({'review_state': 'pending'}): object = brain.getObject() for attr in ("bulletin_des_travaux", "bulletin_questions_ecrites", "courrier_reponse_accuse_de_reception", "cr_debats_declaration_politique_generale", "cr_debats_declaration_programme", "cr_reponse_orale", "cr_seance", "cr_seance_ou_biq", "cr_seance_pleniere", "cr_seance_prise_en_consideration", "cr_seance_vote_motion", "cr_vote_declaration_programme", "documents_lies", "lien_qo_ou_qe", "rapport", "rapport_de_commission", "lien_vers_interpellation_ou_qe", "reponse"): try: v = getattr(object, attr) except AttributeError: continue if v is None: continue if type(v) is list: continue setattr(object, attr, [v]) class UpdateToBlobs(BrowserView): def __call__(self): catalog = getToolByName(self.context, 'portal_catalog') for brain in catalog.search({'portal_type': ['bqrD', 'bulletintravauxD']}): object = brain.getObject() for attr in ("fichier", "document_imprime"): try: v = getattr(object, attr) except AttributeError: continue if v is None: continue print 'transforming file in', object.id newv = NamedFile(v.data, filemane=v.filename) setattr(object, attr, newv)