docbow/docbow_project/pfwb/management/commands/sync-tabellio.py

from __future__ import print_function

from django.conf import settings
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from django.db import transaction
from django.template.defaultfilters import slugify
from sqlalchemy import create_engine, engine as sqla_engine, or_

from docbow_project.docbow.models import MailingList, DocbowProfile, FileType
from docbow_project.pfwb.models import TabellioDocType
from docbow_project.pfwb.tabellio import DBSession, TAdresse, TCom, TComppol, TPer, TPershistoline, TTypedoc

""" The following variables should be defined somewhere in your
local configurration.

MINISTRES_MAILING_ID
PARLEMENTAIRES_MAILING_ID
TABELLIO_DBHOST
TABELLIO_DBPORT
TABELLIO_DBNAME
TABELLIO_DBUSER
TABELLIO_DBPASSWORD
"""


def get_username(last_name, first_name):
    username = '%s.%s' % (slugify(first_name), slugify(last_name))
    return username


def get_or_create_user(last_name, first_name, email, verbose, tabellio_id):
    profile = None
    try:
        profile = DocbowProfile.objects.get(external_id=tabellio_id)
        user = profile.user
    except DocbowProfile.DoesNotExist:
        try:
            if email == 'noreply@pfwb.be':
                raise User.DoesNotExist()
            user = User.objects.get(email=email)
        except User.DoesNotExist:
            try:
                user = User.objects.get(first_name=first_name, last_name=last_name)
            except User.DoesNotExist:
                user = None

    if user is None:
        if verbose:
            print('creating account for %s %s <%s>' % (first_name, last_name, email))
        username = get_username(last_name, first_name)
        user, created = User.objects.get_or_create(username=username)
        user.last_name = last_name
        user.first_name = first_name
        user.email = email

    if user.email == 'noreply@pfwb.be':
        # we only set the email when the current email was the placeholder
        # address.
        user.email = email

    user.is_active = True
    user.save()

    if not profile:
        # create/update matching docbow profile
        profile, created = DocbowProfile.objects.get_or_create(user=user)

        if profile.external_id != tabellio_id:
            profile.external_id = tabellio_id
            profile.save()

    return user


class Command(BaseCommand):
    @transaction.atomic
    def handle(self, *args, **options):
        verbose = options.get('verbosity') > 1

        parl_list = MailingList.objects.get(id=settings.PARLEMENTAIRES_MAILING_ID)
        ministres_list = MailingList.objects.get(id=settings.MINISTRES_MAILING_ID)

        comppols = {}
        commissions_infos = {}  # dict of (name, [list of pers id])
        commissions = {}  # dict of list of User

        engine = create_engine(
            sqla_engine.url.URL(
                drivername='postgresql',
                username=settings.TABELLIO_DBUSER,
                password=settings.TABELLIO_DBPASSWORD,
                host=settings.TABELLIO_DBHOST,
                port=settings.TABELLIO_DBPORT or None,
                database=settings.TABELLIO_DBNAME,
            )
        )
        session = DBSession(bind=engine.connect())

        # get list of persons that are neither deputy or ministre so they can
        # be disabled if they exists in docbow
        for pers, pers_histo in session.query(TPer, TPershistoline).filter(
            TPer.id == TPershistoline.pers,
            TPer.prenom.isnot(None),
            TPershistoline.type.in_(['P_CMPL', 'M_MINT']),
            TPershistoline.fin.isnot(None),
        ):

            last_name, first_name = pers.nom, pers.prenom

            username = get_username(last_name, first_name)

            to_disable_users = User.objects.filter(first_name=first_name, last_name=last_name, is_active=True)
            if not to_disable_users.count():
                # a user didn't exist for that person, fine.
                continue
            for user in to_disable_users:
                if verbose:
                    print('disabling', username)
                # disable the user
                user.is_active = False
                user.save()

                # remove the user from lists
                if user in parl_list.members.all():
                    parl_list.members.remove(user)
                    parl_list.save()
                if user in ministres_list.members.all():
                    ministres_list.members.remove(user)
                    ministres_list.save()

        # get current deputies
        deputies = {}

        # in a first pass, get all of them
        for pers, pers_histo, comppol in session.query(TPer, TPershistoline, TComppol).filter(
            TPer.prenom.isnot(None),
            TPershistoline.description == TComppol.id,
            TPer.id == TPershistoline.pers,
            TPershistoline.type == 'P_CMPL',
            TPershistoline.fin.is_(None),
        ):

            pers_id, last_name, first_name, comppol = pers.id, pers.nom, pers.prenom, comppol.abbr
            deputies[pers_id] = (last_name, first_name, 'noreply@pfwb.be', comppol)

        # in a second pass, overwrite those who have emails
        for pers, pers_histo, addr, comppol in session.query(TPer, TPershistoline, TAdresse, TComppol).filter(
            TPer.prenom.isnot(None),
            TPershistoline.description == TComppol.id,
            TPer.id == TPershistoline.pers,
            TPershistoline.type == 'P_CMPL',
            TPershistoline.fin.is_(None),
            TAdresse.email.isnot(None),
            or_(TAdresse.id == TPer.addrpriv, TAdresse.id == TPer.addrprof1, TAdresse.id == TPer.addrprof2),
            TAdresse.courriel.is_(True),
        ):

            pers_id, last_name, first_name, email, comppol = (
                pers.id,
                pers.nom,
                pers.prenom,
                addr.email,
                comppol.abbr,
            )
            deputies[pers_id] = (last_name, first_name, email, comppol)

        # get commissions
        for com in session.query(TCom).filter(TCom.st == 'S_ACTIVE'):
            com_id, nom = com.id, com.nom
            commissions_infos[com_id] = (nom, [])
            commissions[com_id] = []

        for com_id in commissions.keys():
            for pers in (
                session.query(TPer)
                .join(TPershistoline, TPershistoline.pers == TPer.id)
                .filter(TPershistoline.description == com_id, TPershistoline.fin.is_(None))
            ):
                commissions_infos[com_id][1].append(pers.id)

        deputy_users = []
        for pers_id, deputy in deputies.items():
            last_name, first_name, email, comppol = deputy

            user = get_or_create_user(last_name, first_name, email, verbose=verbose, tabellio_id=pers_id)
            deputy_users.append(user)

            # update mailing lists
            if not user in parl_list.members.all():
                parl_list.members.add(user)
                parl_list.save()

            # keep track of its political group, to be used later
            if not comppol in comppols:
                comppols[comppol] = []
            comppols[comppol].append(user)

            # keep track of commissions, to be used later
            for com_id, (com_name, com_members) in commissions_infos.items():
                if pers_id in com_members:
                    commissions[com_id].append(user)

        # create mailing lists for political groups
        for comppol, members in comppols.items():
            maillist, created = MailingList.objects.get_or_create(
                name=u'Appartenance politique - %s' % comppol
            )
            # remove members of the list that should no longer be in there
            for member in maillist.members.all():
                if member not in members:
                    maillist.members.remove(member)
            # add new members
            for member in members:
                if member not in maillist.members.all():
                    maillist.members.add(member)

        # get inactive comppol
        query = session.query(TComppol).filter(TComppol.st == 'S_INACTIVE')
        inactive_comppols = [u'Appartenance politique - %s' % comppol.abbr for comppol in query]
        comppol_names = [u'Appartenance politique - %s' % comppol for comppol in comppols]
        for maillist in MailingList.objects.filter(name__startswith='Appartenance politique -').exclude(
            name__in=comppol_names
        ):
            maillist.members.clear()
            if verbose:
                print('clear', maillist)
            if maillist.name in inactive_comppols:
                maillist.is_active = False
                maillist.save()
                if verbose:
                    print('deactivate', maillist)

        existing_commissions = list(MailingList.objects.filter(name__startswith='Commission - '))
        # create mailing lists for commissions
        for com_id, members in commissions.items():
            com_name = commissions_infos[com_id][0]
            maillist, created = MailingList.objects.get_or_create(name=u'Commission - %s' % com_name)
            if maillist in existing_commissions:
                existing_commissions.remove(maillist)
            # remove members of the list that should no longer be in there
            for member in maillist.members.all():
                if member not in members:
                    maillist.members.remove(member)
            # add new members
            for member in members:
                if member not in maillist.members.all():
                    maillist.members.add(member)

        # makes sure remaining commissions are emptied
        for maillist in existing_commissions:
            for member in maillist.members.all():
                maillist.members.remove(member)
            maillist.is_active = False
            maillist.save()

        # get current ministres
        ministres = {}

        # like for deputies, we have a first pass to get all of them
        for pers, pers_histo in session.query(TPer, TPershistoline).filter(
            TPer.prenom.isnot(None),
            TPer.id == TPershistoline.pers,
            TPershistoline.type == 'M_MINT',
            TPershistoline.fin.is_(None),
        ):

            pers_id, last_name, first_name = pers.id, pers.nom, pers.prenom
            ministres[pers_id] = (last_name, first_name, 'noreply@pfwb.be')

        # and a second pass to get the email addresses
        for pers, pers_histo, addr, in session.query(TPer, TPershistoline, TAdresse).filter(
            TPer.st == 'S_MINISTRE',
            TPer.prenom.isnot(None),
            TPer.id == TPershistoline.pers,
            TPershistoline.type == 'M_MINT',
            TPershistoline.fin.is_(None),
            TAdresse.email.isnot(None),
            or_(TAdresse.id == TPer.addrpriv, TAdresse.id == TPer.addrprof1, TAdresse.id == TPer.addrprof2),
            TAdresse.courriel.is_(True),
        ):

            pers_id, last_name, first_name, email = pers.id, pers.nom, pers.prenom, addr.email
            ministres[pers_id] = (last_name, first_name, email)

        ministres_users = []
        for pers_id, ministre in ministres.items():
            last_name, first_name, email = ministre

            user = get_or_create_user(last_name, first_name, email, verbose=verbose, tabellio_id=pers_id)
            ministres_users.append(user)

            # update mailing lists
            if not user in ministres_list.members.all():
                ministres_list.members.add(user)
                ministres_list.save()

        # remove users which are not deputies anymore
        for user in parl_list.members.all():
            if user not in deputy_users:
                parl_list.members.remove(user)
                if verbose:
                    print('remove', user, 'from deputies')

        # remove users which are not ministre anymore
        for user in ministres_list.members.all():
            if user not in ministres_users:
                ministres_list.members.remove(user)
                if verbose:
                    print('remove', user, 'from ministres')

        # get list of document types
        for doc in session.query(TTypedoc):
            id, descr = doc.id, doc.descr
            if descr[0] == descr[0].lower():
                descr = descr[0].upper() + descr[1:]

            try:
                doctype = TabellioDocType.objects.get(tabellio_doc_type=id)
            except TabellioDocType.DoesNotExist:
                filetype = FileType()
                filetype.name = descr
                filetype.save()

                doctype = TabellioDocType()
                doctype.tabellio_doc_type = id
                doctype.filetype = filetype
                doctype.save()
            else:
                if doctype.filetype.name != descr:
                    doctype.filetype.name = descr
                    doctype.filetype.save()

        session.close()
        engine.dispose()