271 lines
12 KiB
Python
271 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import datetime
|
|
import json
|
|
import urllib2
|
|
|
|
from DateTime.DateTime import DateTime
|
|
|
|
import transaction
|
|
from zope.event import notify
|
|
from zope.lifecycleevent import ObjectAddedEvent, ObjectModifiedEvent
|
|
from z3c.relationfield import RelationValue
|
|
from plone.app.textfield.value import RichTextValue
|
|
|
|
from plone.namedfile.file import NamedBlobImage, NamedBlobFile
|
|
|
|
from themis.datatypes.address import Address
|
|
|
|
from utils import UtilityView
|
|
|
|
class SyncFromThemis(UtilityView):
|
|
src_url = 'http://localhost:8081/themis'
|
|
|
|
def __call__(self):
|
|
self.setup()
|
|
self.src_url = self.request.form.get('src', self.src_url)
|
|
|
|
timestamp = self.request.form.get('timestamp')
|
|
objects = self.request.form.get('objects', '').split(',')
|
|
for object in ('deputies', 'ministries', 'commissions', 'documents'):
|
|
if not 'all' in objects:
|
|
if not object in objects:
|
|
continue
|
|
getattr(self, 'sync_'+object)(timestamp)
|
|
|
|
return 'OK'
|
|
|
|
def sync_deputies(self, timestamp=None):
|
|
deputies = json.load(urllib2.urlopen('%s/@@listDeputies' % self.src_url)) # XXX: add timestamp
|
|
|
|
for deputy_url in deputies:
|
|
data = json.load(urllib2.urlopen('%s/json' % deputy_url))
|
|
|
|
new_id = data.get('id')
|
|
if not hasattr(self.deputies_folder, new_id):
|
|
self.deputies_folder.invokeFactory('themis.datatypes.deputy', new_id,
|
|
firstname=data.get('firstname'),
|
|
lastname=data.get('lastname'))
|
|
object = getattr(self.deputies_folder, new_id)
|
|
object.firstname = data.get('firstname')
|
|
object.lastname = data.get('lastname')
|
|
object.active = data.get('active')
|
|
object.sex = data.get('sex')
|
|
object.district = data.get('district')
|
|
object.birthplace = data.get('birthplace')
|
|
object.bio = data.get('bio')
|
|
object.website = data.get('website')
|
|
object.degrees = data.get('degrees')
|
|
object.mandates = data.get('mandates')
|
|
object.profession = data.get('profession')
|
|
object.seat_number = data.get('seat_number')
|
|
if data.get('birthdate'):
|
|
object.birthdate = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get('birthdate'), '%Y-%m-%d').toordinal())
|
|
else:
|
|
object.birthdate = None
|
|
if data.get('polgroup'):
|
|
object.polgroup = RelationValue(self.get_polgroup_intid(data.get('polgroup')))
|
|
else:
|
|
object.polgroup = None
|
|
if data.get('picture'):
|
|
# this will be an url
|
|
object.picture = NamedBlobImage(urllib2.urlopen(data.get('picture')).read(),
|
|
filename=data.get('id'))
|
|
if data.get('past_functions'):
|
|
object.past_functions = RichTextValue(
|
|
raw=data.get('past_functions'),
|
|
mimeType='text/plain',
|
|
outputMimeType='text/x-html-safe')
|
|
else:
|
|
object.past_functions = None
|
|
if data.get('current_functions'):
|
|
object.current_functions = RichTextValue(
|
|
raw=data.get('current_functions'),
|
|
mimeType='text/plain',
|
|
outputMimeType='text/x-html-safe')
|
|
else:
|
|
object.current_functions = None
|
|
for address in ('private_address', 'work_address', 'work_address_2'):
|
|
if not data.get(address):
|
|
continue
|
|
setattr(object, address, Address.from_dict(data.get(address)))
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
|
|
def sync_ministries(self, timestamp=None):
|
|
pass
|
|
|
|
def sync_commissions(self, timestamp=None):
|
|
commissions = json.load(urllib2.urlopen('%s/@@listCommissions' % self.src_url)) # XXX: add timestamp
|
|
|
|
for commission_url in commissions:
|
|
data = json.load(urllib2.urlopen('%s/json' % commission_url))
|
|
|
|
new_id = data.get('id')
|
|
if not hasattr(self.commissions_folder, new_id):
|
|
self.commissions_folder.invokeFactory('themis.datatypes.commission', new_id,
|
|
title=data.get('title'))
|
|
object = getattr(self.commissions_folder, new_id)
|
|
object.active = data.get('active')
|
|
if data.get('president'):
|
|
object.president = RelationValue(self.get_author_intid('deputy:%s' % data.get('president')))
|
|
if data.get('vicepresidents'):
|
|
object.vicepresidents = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('vicepresidents')]
|
|
if data.get('members'):
|
|
object.members = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('members')]
|
|
if data.get('substitutes'):
|
|
object.substitutes = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('substitutes')]
|
|
if data.get('competences'):
|
|
object.competences = data.get('competences')
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
def get_author_intid(self, author):
|
|
try:
|
|
obj = getattr(self.deputies_folder, author.split(':', 1)[1])
|
|
except AttributeError:
|
|
print 'failed to get author', author
|
|
return None
|
|
obj_intid = self.intids.getId(obj)
|
|
return obj_intid
|
|
|
|
def sync_documents(self, timestamp=None):
|
|
qs = ''
|
|
if timestamp:
|
|
qs = '?timestamp=%s' % timestamp
|
|
print 'loading documents list from', self.src_url
|
|
documents = json.load(urllib2.urlopen('%s/@@listDocuments%s' % (self.src_url, qs)))
|
|
print 'done'
|
|
|
|
for count, document_url in enumerate(documents):
|
|
print 'loading', document_url
|
|
data = json.load(urllib2.urlopen('%s/json' % document_url))
|
|
|
|
new_id = data.get('id')
|
|
is_question = False
|
|
if data.get('portal_type') in ('Question_ecriteD',
|
|
'questionactualiteD', 'QuestionoraleD',
|
|
'interpellationD'):
|
|
if not hasattr(self.questions_folder, new_id):
|
|
self.questions_folder.invokeFactory('tabellio.documents.question', new_id,
|
|
title = data.get('title'))
|
|
is_question = True
|
|
object = getattr(self.questions_folder, new_id)
|
|
else:
|
|
if not hasattr(self.documents_folder, new_id):
|
|
self.documents_folder.invokeFactory('tabellio.documents.document', new_id,
|
|
title = data.get('title'))
|
|
object = getattr(self.documents_folder, new_id)
|
|
if data.get('title'):
|
|
object.title = data.get('title')
|
|
if data.get('session'):
|
|
object.session = data.get('session')
|
|
if data.get('fichier'):
|
|
object.file = NamedBlobFile(urllib2.urlopen(data.get('fichier')).read(),
|
|
filename=data.get('id')+'.pdf')
|
|
if data.get('document_imprime'):
|
|
object.file = NamedBlobFile(urllib2.urlopen(data.get('document_imprime')).read(),
|
|
filename=data.get('id')+'.pdf')
|
|
if data.get('object_type'):
|
|
if is_question:
|
|
object.questype = data.get('object_type').replace(' (D)', '')
|
|
else:
|
|
object.doctype = data.get('object_type').replace(' (D)', '')
|
|
if data.get('matieres'):
|
|
object.topics = data.get('matieres')
|
|
|
|
for attr in ('date_seance', 'date_seance_pleniere',
|
|
'date_de_la_commission', 'date_document_imprime',
|
|
'date_de_reception', 'date_reception',
|
|
'date_du_document', 'date_de_publication',
|
|
'date_du_rapport'):
|
|
if data.get(attr):
|
|
object.date = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get(attr), '%Y-%m-%d').toordinal())
|
|
|
|
for attr in ('numero_biq', 'numero_document', 'numero_bqr'):
|
|
if data.get(attr):
|
|
object.no = data.get(attr)
|
|
|
|
if data.get('numero_suite'):
|
|
object.nodoc = data.get('numero_suite')
|
|
|
|
related_doc_ids = []
|
|
for attr in ('cr_seance', 'documents_lies', 'bulletin_des_travaux',
|
|
'bulletin_questions_ecrites',
|
|
'courrier_reponse_accuse_de_reception',
|
|
'cr_debats_declaration_politique_generale',
|
|
'cr_debats_declaration_programme',
|
|
'cr_reponse_orale',
|
|
'cr_seance_ou_biq',
|
|
'cr_seance_pleniere',
|
|
'cr_seance_prise_en_consideration',
|
|
'cr_seance_vote_motion',
|
|
'cr_vote_declaration_programme',
|
|
'lien_qo_ou_qe',
|
|
'lien_vers_interpellation_ou_qe',
|
|
'question__ecrite_lien',
|
|
'rapport_de_commission',
|
|
'documents_contenus_dans_biq',
|
|
'documents_contenus_dans_bqr',
|
|
'documents_lies_budgetpfb',
|
|
'documents_r_f_ren_ant_ce_compte_rendu',
|
|
'document_s__li__s____ce_projet__bulletin_des_travaux__rapport_de_commission_',
|
|
'documents_lies_proposition_bt_rapporcom',
|
|
'reponses_ministre_lien',
|
|
):
|
|
if not data.get(attr):
|
|
continue
|
|
related_doc_ids.extend(data.get(attr))
|
|
if related_doc_ids:
|
|
related_docs = []
|
|
for related_doc_id in related_doc_ids:
|
|
if related_doc_id is None:
|
|
continue
|
|
if hasattr(self.questions_folder, related_doc_id):
|
|
related_doc = getattr(self.questions_folder, related_doc_id)
|
|
elif hasattr(self.documents_folder, related_doc_id):
|
|
related_doc = getattr(self.documents_folder, related_doc_id)
|
|
else:
|
|
continue
|
|
related_docs.append(self.intids.getId(related_doc))
|
|
object.related_docs = [RelationValue(x) for x in related_docs if x]
|
|
if data.get('auteur'):
|
|
value = data.get('auteur')
|
|
if type(value) is list:
|
|
authors = [self.get_author_intid(x) for x in value]
|
|
else:
|
|
authors = [self.get_author_intid(value)]
|
|
object.authors = [RelationValue(x) for x in authors if x]
|
|
if data.get('auteurs'):
|
|
authors = [self.get_author_intid(x) for x in data.get('auteurs')]
|
|
object.authors = [RelationValue(x) for x in authors if x]
|
|
|
|
if data.get('portal_type') in ('PropositionD', 'ProjetD'):
|
|
if data.get('vote_seance') == u'Adoption':
|
|
object.adopted = True
|
|
object.adoption_date = DateTime(data.get('date_seance'))
|
|
object.moniteur_date = DateTime(data.get('date_publication_moniteur_belge'))
|
|
else:
|
|
object.adopted = False
|
|
object.adoption_date = None
|
|
object.moniteur_date = None
|
|
|
|
if is_question:
|
|
pass # XXX: import link to CRI
|
|
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
if count % 100 == 0:
|
|
transaction.commit()
|
|
|
|
print 'documents done'
|