389 lines
18 KiB
Python
389 lines
18 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import datetime
|
|
import json
|
|
import urllib2
|
|
import base64
|
|
|
|
from DateTime.DateTime import DateTime
|
|
|
|
import transaction
|
|
from zope.event import notify
|
|
from zope.lifecycleevent import ObjectAddedEvent, ObjectModifiedEvent
|
|
from z3c.relationfield import RelationValue
|
|
from plone.app.textfield.value import RichTextValue
|
|
|
|
from plone.namedfile.file import NamedBlobImage, NamedBlobFile
|
|
|
|
from themis.datatypes.address import Address
|
|
|
|
from utils import UtilityView
|
|
|
|
class SyncFromThemis(UtilityView):
|
|
src_url = 'http://localhost:8081/themis'
|
|
|
|
def urlopen(self, url):
|
|
if not self.request.form.get('username'):
|
|
return urllib2.urlopen(url)
|
|
credentials = (self.request.form.get('username'),
|
|
self.request.form.get('password'))
|
|
# Plone won't get a first answer with a 401/Authenticate, this means
|
|
# we have to force the authorization header manually, we can't rely
|
|
# on urllib2 http auth handling.
|
|
req = urllib2.Request(url, None,
|
|
{'Authorization': 'Basic ' + base64.b64encode("%s:%s" % credentials).strip()})
|
|
return urllib2.urlopen(req)
|
|
|
|
def __call__(self):
|
|
self.setup()
|
|
self.src_url = self.request.form.get('src', self.src_url)
|
|
|
|
timestamp = self.request.form.get('timestamp')
|
|
objects = self.request.form.get('objects', '').split(',')
|
|
for object in ('deputies', 'ministries', 'commissions', 'documents',
|
|
'blocked_documents'):
|
|
if not 'all' in objects:
|
|
if not object in objects:
|
|
continue
|
|
getattr(self, 'sync_'+object)(timestamp)
|
|
|
|
return 'OK'
|
|
|
|
def sync_deputies(self, timestamp=None):
|
|
deputies = json.load(self.urlopen('%s/@@listDeputies' % self.src_url)) # XXX: add timestamp
|
|
|
|
for deputy_url in deputies:
|
|
data = json.load(self.urlopen('%s/json' % deputy_url))
|
|
|
|
new_id = data.get('id')
|
|
if not hasattr(self.deputies_folder, new_id):
|
|
self.deputies_folder.invokeFactory('themis.datatypes.deputy', new_id,
|
|
firstname=data.get('firstname'),
|
|
lastname=data.get('lastname'))
|
|
object = getattr(self.deputies_folder, new_id)
|
|
object.firstname = data.get('firstname')
|
|
object.lastname = data.get('lastname')
|
|
object.active = data.get('active')
|
|
object.sex = data.get('sex')
|
|
object.district = data.get('district')
|
|
object.birthplace = data.get('birthplace')
|
|
object.bio = data.get('bio')
|
|
object.website = data.get('website')
|
|
object.degrees = data.get('degrees')
|
|
object.mandates = data.get('mandates')
|
|
object.profession = data.get('profession')
|
|
object.seat_number = data.get('seat_number')
|
|
if data.get('birthdate'):
|
|
object.birthdate = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get('birthdate'), '%Y-%m-%d').toordinal())
|
|
else:
|
|
object.birthdate = None
|
|
if data.get('polgroup'):
|
|
object.polgroup = RelationValue(self.get_polgroup_intid(data.get('polgroup')))
|
|
else:
|
|
object.polgroup = None
|
|
if data.get('picture'):
|
|
# this will be an url
|
|
object.picture = NamedBlobImage(self.urlopen(data.get('picture')).read(),
|
|
filename=data.get('id'))
|
|
if data.get('past_functions'):
|
|
object.past_functions = RichTextValue(
|
|
raw=data.get('past_functions'),
|
|
mimeType='text/html',
|
|
outputMimeType='text/x-html-safe')
|
|
else:
|
|
object.past_functions = None
|
|
if data.get('current_functions'):
|
|
object.current_functions = RichTextValue(
|
|
raw=data.get('current_functions'),
|
|
mimeType='text/html',
|
|
outputMimeType='text/x-html-safe')
|
|
else:
|
|
object.current_functions = None
|
|
if data.get('biography'):
|
|
object.biography = RichTextValue(
|
|
raw=data.get('biography'),
|
|
mimeType='text/html',
|
|
outputMimeType='text/x-html-safe')
|
|
else:
|
|
object.biography = None
|
|
for address in ('private_address', 'work_address', 'work_address_2'):
|
|
if not data.get(address):
|
|
setattr(object, address, None)
|
|
continue
|
|
setattr(object, address, Address.from_dict(data.get(address)))
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
|
|
def sync_ministries(self, timestamp=None):
|
|
pass
|
|
|
|
def sync_commissions(self, timestamp=None):
|
|
commissions = json.load(self.urlopen('%s/@@listCommissions' % self.src_url)) # XXX: add timestamp
|
|
|
|
for commission_url in commissions:
|
|
data = json.load(self.urlopen('%s/json' % commission_url))
|
|
|
|
new_id = data.get('id')
|
|
if not hasattr(self.commissions_folder, new_id):
|
|
self.commissions_folder.invokeFactory('themis.datatypes.commission', new_id,
|
|
title=data.get('title'))
|
|
object = getattr(self.commissions_folder, new_id)
|
|
object.active = data.get('active')
|
|
if data.get('president'):
|
|
object.president = RelationValue(self.get_author_intid('deputy:%s' % data.get('president')))
|
|
if data.get('vicepresidents'):
|
|
object.vicepresidents = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('vicepresidents')]
|
|
if data.get('members'):
|
|
object.members = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('members')]
|
|
if data.get('substitutes'):
|
|
object.substitutes = [RelationValue(self.get_author_intid('deputy:%s' % x))
|
|
for x in data.get('substitutes')]
|
|
if data.get('competences'):
|
|
object.competences = data.get('competences')
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
def get_author_intid(self, author):
|
|
try:
|
|
obj = getattr(self.deputies_folder, author.split(':', 1)[1])
|
|
except AttributeError:
|
|
print 'failed to get author', author
|
|
return None
|
|
obj_intid = self.intids.getId(obj)
|
|
return obj_intid
|
|
|
|
def sync_documents(self, timestamp=None):
|
|
qs = ''
|
|
if timestamp:
|
|
qs = '?timestamp=%s' % timestamp
|
|
print 'loading documents list from', self.src_url
|
|
documents = json.load(self.urlopen('%s/@@listDocuments%s' % (self.src_url, qs)))
|
|
print 'done'
|
|
|
|
for count, document_url in enumerate(documents):
|
|
print 'loading', document_url
|
|
data = json.load(self.urlopen('%s/json' % document_url))
|
|
|
|
new_id = data.get('id')
|
|
is_question = False
|
|
if data.get('portal_type') in ('Question_ecriteD',
|
|
'questionactualiteD', 'QuestionoraleD',
|
|
'interpellationD'):
|
|
if not hasattr(self.questions_folder, new_id):
|
|
self.questions_folder.invokeFactory('tabellio.documents.question', new_id,
|
|
title = data.get('title'))
|
|
is_question = True
|
|
object = getattr(self.questions_folder, new_id)
|
|
elif data.get('portal_type') in ('convocation_commissionD',
|
|
'convocation_seanceD'):
|
|
if not hasattr(self.convocations_folder, new_id):
|
|
self.convocations_folder.invokeFactory('tabellio.documents.document', new_id,
|
|
title = data.get('title'))
|
|
object = getattr(self.convocations_folder, new_id)
|
|
else:
|
|
if not hasattr(self.documents_folder, new_id):
|
|
self.documents_folder.invokeFactory('tabellio.documents.document', new_id,
|
|
title = data.get('title'))
|
|
object = getattr(self.documents_folder, new_id)
|
|
if data.get('title'):
|
|
object.title = data.get('title')
|
|
if data.get('session'):
|
|
object.session = data.get('session')
|
|
if data.get('fichier'):
|
|
object.file = NamedBlobFile(self.urlopen(data.get('fichier')).read(),
|
|
filename=data.get('id')+'.pdf')
|
|
if data.get('document_imprime'):
|
|
object.file = NamedBlobFile(self.urlopen(data.get('document_imprime')).read(),
|
|
filename=data.get('id')+'.pdf')
|
|
if data.get('object_type'):
|
|
if is_question:
|
|
object.questype = data.get('object_type').replace(' (D)', '')
|
|
else:
|
|
object.doctype = data.get('object_type').replace(' (D)', '')
|
|
if data.get('matieres'):
|
|
object.topics = data.get('matieres')
|
|
|
|
for attr in ('date_seance', 'date_seance_pleniere',
|
|
'date_de_la_commission', 'date_document_imprime',
|
|
'date_de_reception', 'date_reception',
|
|
'date_du_document', 'date_de_publication',
|
|
'date_du_rapport'):
|
|
if data.get(attr):
|
|
object.date = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get(attr), '%Y-%m-%d').toordinal())
|
|
|
|
if data.get('date_sanction_promulgation'):
|
|
object.sanction_date = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get('date_sanction_promulgation'), '%Y-%m-%d').toordinal())
|
|
else:
|
|
object.sanction_date = None
|
|
|
|
if data.get('reponse_a_question_ecrite'):
|
|
object.written_answer = data.get('reponse_a_question_ecrite')[0]
|
|
else:
|
|
object.written_answer = None
|
|
|
|
if data.get('date_seance_ou_commission'):
|
|
object.seance_com_date = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get('date_seance_ou_commission'),
|
|
'%Y-%m-%d').toordinal())
|
|
else:
|
|
object.seance_com_date = None
|
|
|
|
for attr in ('numero_biq', 'numero_document', 'numero_bqr'):
|
|
if data.get(attr):
|
|
object.no = data.get(attr)
|
|
|
|
if data.get('numero_suite'):
|
|
object.nodoc = data.get('numero_suite')
|
|
|
|
related_doc_ids = []
|
|
for attr in ( 'autres_documents_lies_a_proposition',
|
|
'autres_documents_lies_au_projet',
|
|
'autres_documents_ou_dossiers_en_relation',
|
|
'bulletin_questions_ecrites',
|
|
'courriers_lies',
|
|
'cr_debats_declaration_politique_generale',
|
|
'cr_debats_declaration_programme',
|
|
'cr_reponse_orale',
|
|
'cr_seance',
|
|
'cr_seance_ou_biq',
|
|
'cr_seance_pleniere',
|
|
'cr_seance_prise_en_consideration',
|
|
'cr_seance_vote_motion',
|
|
'cr_vote_declaration_programme',
|
|
'docs_pointing',
|
|
'documents_contenus_dans_biq',
|
|
'documents_contenus_dans_bqr',
|
|
'documents_lies',
|
|
'documents_lies_budgetpfb',
|
|
'documents_lies_proposition_bt_rapporcom',
|
|
'document_s__li__s____ce_projet__bulletin_des_travaux__rapport_de_commission_',
|
|
'documents_r_f_ren_ant_ce_compte_rendu',
|
|
'documents_r_f_ren_ants',
|
|
'documents_r_f_renc_s',
|
|
'lien_qo_ou_qe',
|
|
'lien_vers_courrier_reponse',
|
|
'lien_vers_interpellation_ou_qe',
|
|
'question__ecrite_lien',
|
|
'reponses_ministre_lien'):
|
|
if not data.get(attr):
|
|
continue
|
|
related_doc_ids.extend(data.get(attr))
|
|
if related_doc_ids:
|
|
related_docs = []
|
|
for related_doc_id in related_doc_ids:
|
|
if related_doc_id is None:
|
|
continue
|
|
if hasattr(self.questions_folder, related_doc_id):
|
|
related_doc = getattr(self.questions_folder, related_doc_id)
|
|
elif hasattr(self.documents_folder, related_doc_id):
|
|
related_doc = getattr(self.documents_folder, related_doc_id)
|
|
elif hasattr(self.convocations_folder, related_doc_id):
|
|
related_doc = getattr(self.convocations_folder, related_doc_id)
|
|
else:
|
|
continue
|
|
related_docs.append(self.intids.getId(related_doc))
|
|
object.related_docs = [RelationValue(x) for x in related_docs if x]
|
|
|
|
authors_v = None
|
|
if data.get('auteur'):
|
|
authors_v = data.get('auteur')
|
|
if type(authors_v) is not list:
|
|
authors_v = [authors_v]
|
|
elif data.get('auteurs'):
|
|
authors_v = data.get('auteurs')
|
|
if authors_v == ['ministry:college']:
|
|
object.author_is_government = True
|
|
elif authors_v:
|
|
authors_intids = [self.get_author_intid(x) for x in authors_v]
|
|
object.authors = [RelationValue(x) for x in authors_intids if x]
|
|
object.author_is_government = False
|
|
else:
|
|
object.authors = None
|
|
object.author_is_government = False
|
|
|
|
if data.get('etat_en_commission') and data.get('etat_en_commission') != u'Indéterminé':
|
|
object.commission_status = data.get('etat_en_commission')
|
|
else:
|
|
object.commission_status = None
|
|
|
|
if data.get('vote_seance') and data.get('vote_seance') != u'Indéterminé':
|
|
object.seance_vote = data.get('vote_seance')
|
|
else:
|
|
object.seance_vote = None
|
|
|
|
if data.get('portal_type') in ('PropositionD', 'ProjetD'):
|
|
if data.get('date_seance'):
|
|
object.seance_vote_date = datetime.date.fromordinal(
|
|
datetime.datetime.strptime(
|
|
data.get('date_seance'),
|
|
'%Y-%m-%d').toordinal())
|
|
else:
|
|
object.seance_vote_date = None
|
|
if data.get('vote_seance') == u'Adoption':
|
|
object.adopted = True
|
|
if data.get('date_approbation_tutelle'):
|
|
object.adoption_date = DateTime(data.get('date_approbation_tutelle'))
|
|
else:
|
|
object.adoption_date = None
|
|
if data.get('date_publication_moniteur_belge'):
|
|
object.moniteur_date = DateTime(data.get('date_publication_moniteur_belge'))
|
|
else:
|
|
object.moniteur_date = None
|
|
else:
|
|
object.adopted = False
|
|
object.adoption_date = None
|
|
object.moniteur_date = None
|
|
|
|
if is_question:
|
|
pass # XXX: import link to CRI
|
|
|
|
notify(ObjectModifiedEvent(object))
|
|
self.publish(object)
|
|
|
|
if count % 100 == 0:
|
|
transaction.commit()
|
|
|
|
print 'documents done'
|
|
|
|
def sync_blocked_documents(self, timestamp=None):
|
|
qs = ''
|
|
if timestamp:
|
|
qs = '?timestamp=%s' % timestamp
|
|
print 'loading blocked documents list from', self.src_url
|
|
documents = json.load(self.urlopen('%s/@@listBlockedDocuments%s' % (self.src_url, qs)))
|
|
print 'done'
|
|
|
|
for count, document_url in enumerate(documents):
|
|
print 'loading', document_url
|
|
data = json.load(self.urlopen('%s/json' % document_url))
|
|
|
|
new_id = data.get('id')
|
|
is_question = False
|
|
if data.get('portal_type') in ('Question_ecriteD',
|
|
'questionactualiteD', 'QuestionoraleD',
|
|
'interpellationD'):
|
|
if hasattr(self.questions_folder, new_id):
|
|
self.questions_folder.manage_delObjects([new_id])
|
|
elif data.get('portal_type') in ('convocation_commissionD',
|
|
'convocation_seanceD'):
|
|
if not hasattr(self.convocations_folder, new_id):
|
|
self.convocations_folder.manage_delObjects([new_id])
|
|
else:
|
|
if hasattr(self.documents_folder, new_id):
|
|
self.documents_folder.manage_delObjects([new_id])
|
|
|
|
if count % 100 == 0:
|
|
transaction.commit()
|
|
|
|
print 'blocked documents done'
|