1089 lines
39 KiB
Python
1089 lines
39 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# zoo - versatile objects management
|
|
# Copyright (C) 2016 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import print_function
|
|
import six
|
|
import functools
|
|
import uuid
|
|
|
|
import sys
|
|
import re
|
|
import datetime
|
|
import isodate
|
|
import operator
|
|
import copy
|
|
import collections
|
|
import json
|
|
import time
|
|
|
|
import psycopg2
|
|
|
|
from django.conf import settings
|
|
from django.contrib.postgres.search import TrigramDistance
|
|
from django.db import connection
|
|
from django.db.models import Q, F, Value
|
|
from django.db.models.functions import Least, Greatest, Coalesce, Concat
|
|
from django.db import transaction
|
|
from django.utils.timezone import now, make_aware
|
|
from django.contrib.auth.hashers import make_password
|
|
|
|
from zoo.zoo_meta.models import EntitySchema, RelationSchema
|
|
from zoo.zoo_data.models import Entity, Relation, Transaction, Log
|
|
from zoo.zoo_data.search import JSONTextRef, Normalize
|
|
|
|
today = datetime.date.today
|
|
|
|
|
|
INDIVIDU_ENT = 'individu'
|
|
ADRESSE_ENT = 'adresse'
|
|
|
|
HABITE_REL = 'habite'
|
|
UNION_REL = 'union'
|
|
RESPONSABILITE_LEGALE_REL = 'responsabilite-legale'
|
|
|
|
|
|
def make_date(date_var):
|
|
'''Extract a date from a datetime, a date, a struct_time or a string'''
|
|
if isinstance(date_var, datetime.datetime):
|
|
return date_var.date()
|
|
if isinstance(date_var, datetime.date):
|
|
return date_var
|
|
return isodate.parse_date(date_var)
|
|
|
|
|
|
def date_delta(t1, t2):
|
|
'''Return the timedelta between two date like values'''
|
|
t1, t2 = make_date(t1), make_date(t2)
|
|
return t1 - t2
|
|
|
|
|
|
def age_in_years_and_months(born, today=None):
|
|
'''Compute age since today as the number of years and months elapsed'''
|
|
born = make_date(born)
|
|
if today is None:
|
|
today = datetime.date.today()
|
|
today = make_date(today)
|
|
before = (today.month, today.day) < (born.month, born.day)
|
|
years = today.year - born.year
|
|
months = today.month - born.month
|
|
if before:
|
|
years -= 1
|
|
months += 12
|
|
if today.day < born.day:
|
|
months -= 1
|
|
return years, months
|
|
|
|
|
|
def age_in_years(born, today=None):
|
|
'''Compute age since today as the number of years elapsed'''
|
|
return age_in_years_and_months(born, today=today)[0]
|
|
|
|
|
|
def conjoint(individu):
|
|
for relation in individu.left_relations.all():
|
|
if relation.schema.slug != UNION_REL:
|
|
continue
|
|
return relation.right, relation
|
|
|
|
for relation in individu.right_relations.all():
|
|
if relation.schema.slug != UNION_REL:
|
|
continue
|
|
return relation.left, relation
|
|
|
|
return None, None
|
|
|
|
|
|
def enfants(individu):
|
|
for relation in individu.left_relations.all():
|
|
if relation.schema.slug != RESPONSABILITE_LEGALE_REL:
|
|
continue
|
|
yield relation.right, relation
|
|
|
|
|
|
def parents(individu):
|
|
for relation in individu.right_relations.all():
|
|
if relation.schema.slug != RESPONSABILITE_LEGALE_REL:
|
|
continue
|
|
yield relation.left, relation
|
|
|
|
|
|
def adresses(individu):
|
|
adresses = []
|
|
for relation in individu.left_relations.all():
|
|
if relation.schema.slug != HABITE_REL:
|
|
continue
|
|
adresses.append((relation.right, relation))
|
|
adresses.sort(key=lambda t: (not t[1].content.get('principale', False), t[0].id))
|
|
return adresses
|
|
|
|
|
|
def adresse(individu):
|
|
'''Récupérer l'adresse d'un individu majeur (il ne doit en avoir qu'une seule)'''
|
|
l = list(adresses(individu))
|
|
assert len(l) == 1, 'individu a plus d\'une adresse'
|
|
return l[0][0]
|
|
|
|
|
|
class PersonSearch(object):
|
|
EMAIL_RE = re.compile(
|
|
'^[a-zA-Z.+_-]*@[a-zA-Z.+_-]*$')
|
|
DATE_RE1 = re.compile(
|
|
'^(?:(?P<year>\d\d|\d\d\d\d)(?:-(?P<month>\d{1,2})(?:-(?P<day>\d{1,2}))?)?)$')
|
|
DATE_RE2 = re.compile(
|
|
'^(?:(?:(?:(?P<day>\d{1,2})/)?(?P<month>\d{1,2})/)?(?P<year>\d\d|\d\d\d\d))$')
|
|
|
|
@classmethod
|
|
def match_birthdate(cls, birthdate):
|
|
return cls.DATE_RE1.match(birthdate) or cls.DATE_RE2.match(birthdate)
|
|
|
|
@classmethod
|
|
def lu(cls, x):
|
|
return Normalize(x)
|
|
|
|
@classmethod
|
|
def luv(cls, x):
|
|
return cls.lu(Value(x))
|
|
|
|
@classmethod
|
|
def applications(cls):
|
|
applications = getattr(settings, 'ZOO_NANTERRE_APPLICATIONS', {})
|
|
|
|
def helper():
|
|
for application, value in applications.items():
|
|
yield application, value['name']
|
|
return list(helper())
|
|
|
|
def __init__(self, limit=0.5, base_limit=0.3):
|
|
self.birthdates_filters = []
|
|
self.name_filters = []
|
|
self.name_similarities = []
|
|
self.email_similarities = []
|
|
self.key_filters = []
|
|
self.email_filters = []
|
|
self.statut_legal_filter = None
|
|
self.schema = EntitySchema.objects.get(slug=INDIVIDU_ENT)
|
|
self.limit = limit
|
|
self.base_limit = base_limit
|
|
|
|
def search_statut_legal(self, statut_legal):
|
|
self.statut_legal_filter = statut_legal
|
|
|
|
def search_query(self, query):
|
|
'''Take a one line query and try to build a search filter from it'''
|
|
emails = []
|
|
identifiers = []
|
|
birthdates = []
|
|
names = []
|
|
|
|
parts = query.strip().split()
|
|
for part in parts:
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
if self.EMAIL_RE.match(part):
|
|
emails.append(part)
|
|
elif part.startswith('#'):
|
|
if part[1:]:
|
|
identifiers.append(part[1:])
|
|
elif self.match_birthdate(part):
|
|
birthdates.append(self.match_birthdate(part).groupdict())
|
|
else:
|
|
names.append(part)
|
|
for email in emails:
|
|
self = self.search_email(email)
|
|
for identifier in identifiers:
|
|
self = self.search_identifier(identifier)
|
|
for birthdate in birthdates:
|
|
self = self.search_birthdate(birthdate)
|
|
self = self.search_names(names)
|
|
return self
|
|
|
|
def search_email(self, email):
|
|
self = copy.deepcopy(self)
|
|
|
|
f = self.q_normalize('email', email)
|
|
self.email_filters.append(f)
|
|
self.email_similarities.append(Value(1.0) - self.distance('email', email))
|
|
return self
|
|
|
|
def search_identifier(self, identifier, key=None):
|
|
self = copy.deepcopy(self)
|
|
|
|
if key:
|
|
if identifier == '*':
|
|
q = Q(**{'content__cles_de_federation__%s__isnull' % key: False})
|
|
else:
|
|
q = Q(**{'content__cles_de_federation__%s' % key: identifier})
|
|
else:
|
|
filters = []
|
|
try:
|
|
individu_id = int(identifier)
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
filters.append(Q(id=individu_id))
|
|
for key, name in self.applications():
|
|
filters.append(Q(**{'content__cles_de_federation__%s' % key: identifier}))
|
|
q = functools.reduce(Q.__or__, filters)
|
|
self.key_filters.append(q)
|
|
return self
|
|
|
|
def search_birthdate(self, birthdate, window_days=0):
|
|
self = copy.deepcopy(self)
|
|
|
|
if hasattr(birthdate, 'keys'):
|
|
# case of dict
|
|
pass
|
|
elif hasattr(birthdate, 'year'):
|
|
if hasattr(birthdate, 'date'):
|
|
birthdate = birthdate.date()
|
|
# fast path for date / datetime
|
|
if window_days:
|
|
before = birthdate - datetime.timedelta(days=window_days)
|
|
after = birthdate + datetime.timedelta(days=window_days)
|
|
self.birthdates_filters.append(
|
|
Q(content__date_de_naissance__timestamp__gte=before)
|
|
& Q(content__date_de_naissance__timestamp__lte=after)
|
|
)
|
|
else:
|
|
self.birthdates_filters.append(
|
|
Q(content__date_de_naissance__timestamp=birthdate)
|
|
)
|
|
return self
|
|
else:
|
|
# case of strings
|
|
birthdate = self.match_birthdate(birthdate).groupdict()
|
|
|
|
this_year = datetime.date.today().year % 100
|
|
year = int(birthdate['year'])
|
|
if year < 100:
|
|
if year > this_year:
|
|
year += 1900
|
|
else:
|
|
year += 2000
|
|
q = Q(content__date_de_naissance__date__year=year)
|
|
if birthdate['month']:
|
|
q &= Q(content__date_de_naissance__date__month=birthdate['month'])
|
|
if birthdate['day']:
|
|
q &= Q(content__date_de_naissance__date__day=birthdate['day'])
|
|
self.birthdates_filters.append(q)
|
|
return self
|
|
|
|
@classmethod
|
|
def distance(cls, field, value):
|
|
return TrigramDistance(cls.lu(JSONTextRef(F('content'), field)), cls.luv(value))
|
|
|
|
@classmethod
|
|
def q_normalize(self, field, value):
|
|
return Q(**{'content__%s__normalize__trigram_similar' % field: self.luv(value)})
|
|
|
|
def search_name(self, first_name=None, last_name=None, factor=1.0, first_name_weight=1.0,
|
|
last_name_weight=1.0):
|
|
self = copy.deepcopy(self)
|
|
|
|
q = Q()
|
|
if not first_name or not last_name:
|
|
factor *= 0.8
|
|
if last_name:
|
|
q &= (self.q_normalize('nom_d_usage', last_name)
|
|
| self.q_normalize('nom_de_naissance', last_name))
|
|
if first_name:
|
|
q &= self.q_normalize('prenoms', first_name)
|
|
self.name_filters.append(q)
|
|
|
|
fname_d = self.distance('prenoms', first_name)
|
|
name_of_use_d = self.distance('nom_d_usage', last_name)
|
|
name_of_birth_d = self.distance('nom_de_naissance', last_name)
|
|
|
|
if first_name and last_name:
|
|
similarity = Value(first_name_weight) * fname_d
|
|
similarity += Value(last_name_weight) * Least(name_of_use_d, name_of_birth_d)
|
|
similarity /= Value(first_name_weight + last_name_weight)
|
|
elif first_name:
|
|
similarity = fname_d
|
|
else:
|
|
similarity = Least(name_of_use_d, name_of_birth_d)
|
|
similarity = (Value(1.0) - similarity) * Value(factor)
|
|
self.name_similarities.append(similarity)
|
|
return self
|
|
|
|
def search_names(self, names):
|
|
if not names:
|
|
return self
|
|
|
|
self = copy.deepcopy(self)
|
|
|
|
for i in range(0, len(names) + 1):
|
|
first_name, last_name = ' '.join(names[:i]), ' '.join(names[i:])
|
|
self = self.search_name(first_name, last_name)
|
|
if len(names) > 1:
|
|
self = self.search_name(last_name, first_name, factor=0.8)
|
|
return self
|
|
|
|
def copy(self):
|
|
return copy.deepcopy(self)
|
|
|
|
@classmethod
|
|
def or_filters(self, filters):
|
|
return functools.reduce(operator.__or__, filters, Q())
|
|
|
|
@classmethod
|
|
def add_age(cls, individu):
|
|
birthdate = make_date(individu.content['date_de_naissance'])
|
|
|
|
if birthdate >= datetime.date.today():
|
|
age = u'à naître'
|
|
else:
|
|
individu.age = years, months = age_in_years_and_months(
|
|
individu.content['date_de_naissance'])
|
|
if (months, years) == (0, 0):
|
|
age = u'moins d\'un mois'
|
|
elif years < 1:
|
|
age = u'%s mois' % months
|
|
elif years < 2:
|
|
age = u'%s mois' % (months + 12)
|
|
else:
|
|
age = u'%s ans' % years
|
|
individu.age_label = age
|
|
|
|
@classmethod
|
|
def add_adresses(cls, individu):
|
|
individu.adresses = []
|
|
for adresse, relation in adresses(individu):
|
|
a = adresse.content.copy()
|
|
a.update(relation.content)
|
|
a['id'] = adresse.id
|
|
individu.adresses.append(a)
|
|
|
|
@classmethod
|
|
def add_enfants(cls, individu):
|
|
enfants_l = []
|
|
for enfant, relation in enfants(individu):
|
|
cls.add_age(enfant)
|
|
cls.add_federations(enfant)
|
|
cls.add_adresses(enfant)
|
|
enfant.responsabilite_legale = relation.content['statut']
|
|
enfants_l.append(enfant)
|
|
if enfants_l:
|
|
individu.enfants = enfants_l
|
|
|
|
@classmethod
|
|
def add_parents(cls, individu):
|
|
parents = []
|
|
for relation in individu.right_relations.all():
|
|
if relation.schema.slug != 'responsabilite-legale':
|
|
continue
|
|
parent = relation.left
|
|
cls.add_age(parent)
|
|
cls.add_federations(parent)
|
|
parent.responsabilite_legale = relation.content['statut']
|
|
parents.append(parent)
|
|
if parents:
|
|
individu.parents = parents
|
|
|
|
@classmethod
|
|
def add_union(cls, individu):
|
|
con, conjoint_rel = conjoint(individu)
|
|
if con:
|
|
cls.add_age(con)
|
|
cls.add_federations(con)
|
|
individu.union = con
|
|
individu.union_statut = conjoint_rel.content['statut']
|
|
|
|
@classmethod
|
|
def add_federations(cls, individu):
|
|
individu.federations = []
|
|
cles_de_federation = individu.content.get('cles_de_federation', {})
|
|
for federation_key, federation_name in cls.applications():
|
|
if cles_de_federation.get(federation_key):
|
|
individu.federations.append(federation_name)
|
|
|
|
def queryset(self, prefetch=True):
|
|
connection.cursor().execute('SELECT SET_LIMIT(%s)' % self.base_limit)
|
|
|
|
qs = Entity.objects.filter(schema=self.schema)
|
|
|
|
qs = qs.filter(
|
|
self.or_filters(
|
|
self.birthdates_filters))
|
|
qs = qs.filter(
|
|
self.or_filters(self.key_filters))
|
|
qs = qs.filter(
|
|
self.or_filters(self.email_filters))
|
|
qs = qs.filter(
|
|
self.or_filters(self.name_filters))
|
|
if self.statut_legal_filter:
|
|
qs = qs.filter(content__statut_legal=self.statut_legal_filter)
|
|
|
|
qs = qs.annotate(
|
|
fullname=Concat(
|
|
Coalesce(
|
|
JSONTextRef(F('content'), 'nom_d_usage'),
|
|
JSONTextRef(F('content'), 'nom_de_naissance'),
|
|
Value(' ')
|
|
),
|
|
Value(' '),
|
|
JSONTextRef(F('content'), 'prenoms'))
|
|
)
|
|
|
|
# order by similarities or fullname
|
|
similarities = []
|
|
|
|
if self.name_similarities:
|
|
e = (Greatest(*self.name_similarities) if len(self.name_similarities) > 1
|
|
else self.name_similarities[0])
|
|
similarities.append(e)
|
|
|
|
if self.email_similarities:
|
|
e = (Greatest(*self.email_similarities) if len(self.email_similarities) > 1
|
|
else self.email_similarities[0])
|
|
similarities.append(e)
|
|
|
|
if similarities:
|
|
qs = qs.annotate(similarity=functools.reduce(operator.__add__, similarities) /
|
|
Value(len(similarities)))
|
|
qs = qs.filter(similarity__gte=self.limit)
|
|
qs = qs.order_by('-similarity', 'fullname')
|
|
else:
|
|
qs = qs.order_by('fullname')
|
|
|
|
if prefetch:
|
|
qs = qs.prefetch_related(
|
|
'left_relations__schema', 'left_relations__right',
|
|
'right_relations__schema', 'right_relations__left',
|
|
)
|
|
return qs
|
|
|
|
def __getitem__(self, item):
|
|
if hasattr(item, 'start'):
|
|
return self.decorate_iter(self.queryset()[item.start:item.stop])
|
|
return self.decorate_individu(self.queryset()[item])
|
|
|
|
@classmethod
|
|
def decorate_individu(self, individu):
|
|
self.add_age(individu)
|
|
self.add_adresses(individu)
|
|
self.add_federations(individu)
|
|
self.add_enfants(individu)
|
|
self.add_parents(individu)
|
|
self.add_union(individu)
|
|
|
|
@classmethod
|
|
def decorate_iter(self, qs):
|
|
for individu in qs:
|
|
self.decorate_individu(individu)
|
|
yield individu
|
|
|
|
def __iter__(self):
|
|
return self.decorate_iter(self.queryset())
|
|
|
|
|
|
def integrity_check():
|
|
count_union_rels = collections.Counter()
|
|
count_parent_rels = collections.Counter()
|
|
count_habite_rels = collections.Counter()
|
|
|
|
rels = Relation.objects.select_related()
|
|
|
|
for rel in rels.filter(schema__slug=HABITE_REL):
|
|
count_habite_rels[rel.left] += 1
|
|
|
|
for rel in rels.filter(schema__slug=UNION_REL):
|
|
key = tuple(sorted([rel.left, rel.right], key=lambda x: x.id))
|
|
count_union_rels[key] += 1
|
|
count_union_rels[rel.left] += 1
|
|
count_union_rels[rel.right] += 1
|
|
|
|
for rel in rels.filter(schema__slug=RESPONSABILITE_LEGALE_REL):
|
|
count_parent_rels[rel.right] += 1
|
|
|
|
for key, value in count_union_rels.items():
|
|
if value > 1:
|
|
if isinstance(key, tuple):
|
|
yield ("le couple %s / %s est en union plusieurs fois: %s"
|
|
% (key[0], key[1], value))
|
|
else:
|
|
yield ("l'individu %s est en union avec plus d'une personne: %s"
|
|
% (key, value))
|
|
|
|
for key, value in count_parent_rels.items():
|
|
if value > 2:
|
|
yield ("l'enfant %s a plus de deux parents: %s"
|
|
% (key, value))
|
|
|
|
for key, value in count_habite_rels.items():
|
|
if key.content['statut_legal'] == 'majeur' and value < 1:
|
|
yield ("l'adulte %s n'a pas d'adresse" % key)
|
|
|
|
if key.content['statut_legal'] == 'majeur' and value > 1:
|
|
yield ("l'adulte %s a plus d'une adresse: %s" % (key, value))
|
|
|
|
if key.content['statut_legal'] == 'mineur' and value > 2:
|
|
yield ("l'enfant %s a plus de deux adresses: %s" % (key, value))
|
|
|
|
if key.content['statut_legal'] == 'mineur' and value > 2:
|
|
yield ("l'enfant %s n'a pas d'adresse" % key)
|
|
|
|
|
|
def upper_dict(d):
|
|
'''Transform all string values in d to uppercase'''
|
|
for key, value in d.items():
|
|
if isinstance(value, six.text_type):
|
|
d[key] = value.upper()
|
|
|
|
|
|
class UserFixture(object):
|
|
def __init__(self, path):
|
|
self.stream = open(path, 'w')
|
|
self.stream.write('[\n')
|
|
self.first = True
|
|
|
|
def write_user(self, swarmid, username, password, first_name, last_name, email, last_login):
|
|
uid = 'swarmid-%s' % swarmid
|
|
date_joined = now().isoformat()
|
|
if password:
|
|
password = 'bcrypt$' + password
|
|
else:
|
|
password = make_password(uuid.uuid4().hex)
|
|
d = {
|
|
'model': 'custom_user.user',
|
|
'fields': {
|
|
'email': email,
|
|
'first_name': first_name,
|
|
'last_name': last_name,
|
|
'last_login': make_aware(last_login).isoformat(),
|
|
'username': username,
|
|
'uuid': uid,
|
|
'password': password,
|
|
'modified': date_joined,
|
|
'date_joined': date_joined,
|
|
}
|
|
}
|
|
if not self.first:
|
|
self.stream.write(',\n')
|
|
self.stream.write(json.dumps(d))
|
|
self.first = False
|
|
return uid
|
|
|
|
def close(self):
|
|
self.stream.write(']')
|
|
self.stream.flush()
|
|
self.stream.close()
|
|
|
|
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
|
|
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)
|
|
|
|
class DryLoad(Exception):
|
|
pass
|
|
|
|
|
|
class LoadDump(object):
|
|
EMAIL_RE = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
|
|
|
|
def __init__(self, pg_dsn, authentic_fixture_path, verbosity=1, dry=False):
|
|
self.authentic_fixture_path = authentic_fixture_path
|
|
self.connection = psycopg2.connect(dsn=pg_dsn)
|
|
self.cursor = self.connection.cursor()
|
|
self.counters = collections.Counter()
|
|
self.tr = Transaction.objects.create(meta="initial import")
|
|
self.clock_stack = []
|
|
self.verbosity = verbosity
|
|
self.dry = dry
|
|
for cause in ['RG1', 'RG2', 'RG4', 'RG3']:
|
|
self.counters['individus.ignores.%s' % cause] = 0
|
|
|
|
def start(self, msg=None):
|
|
if msg and self.verbosity > 0:
|
|
print(msg, end=' ')
|
|
sys.stdout.flush()
|
|
self.clock_stack.append(time.time())
|
|
|
|
def end(self):
|
|
if self.verbosity > 0:
|
|
print(' DONE (%s s)' % (time.time() - self.clock_stack.pop()))
|
|
|
|
def log(msg=None):
|
|
def decorator(func):
|
|
def wrapper(self):
|
|
self.start(msg)
|
|
func(self)
|
|
self.end()
|
|
return wrapper
|
|
return decorator
|
|
|
|
@log()
|
|
def load(self):
|
|
try:
|
|
with transaction.atomic():
|
|
self.reset_db()
|
|
self.load_individuals()
|
|
self.load_swarm()
|
|
self.load_addresses()
|
|
self.load_relations()
|
|
if self.dry:
|
|
raise DryLoad
|
|
except DryLoad:
|
|
pass
|
|
# tally counters
|
|
if self.verbosity > 1:
|
|
print()
|
|
print('Rapport d\'import :')
|
|
print()
|
|
for key in sorted(self.counters):
|
|
print(' %-80s' % key, ':', '%10d' % self.counters[key])
|
|
print()
|
|
print(' Compteur de séquence SAGA :', self.saga_sequence)
|
|
|
|
@log('Resetting db... ')
|
|
def reset_db(self):
|
|
if not self.dry:
|
|
# pas besoin de nettoyer si on ne va pas garder les modifications
|
|
Relation.objects.all().delete()
|
|
Entity.objects.all().delete()
|
|
|
|
@log('Loading swarm users... ')
|
|
def load_swarm(self):
|
|
user_fixture = UserFixture(self.authentic_fixture_path)
|
|
self.cursor.execute('''SELECT id, username, password, firstname, lastname, email,
|
|
last_login, status FROM users''')
|
|
for (swarmid, username, password, first_name, last_name, email,
|
|
last_login, status) in self.cursor.fetchall():
|
|
self.counters['swarmids'] += 1
|
|
self.counters['swarmids.%s' % status] += 1
|
|
if swarmid not in self.swarmids:
|
|
self.counters['swarmids.ignores.individu_ignore'] += 1
|
|
continue
|
|
if status == 'suspended':
|
|
# RG14: on ne garde pas les mots de passe des comptes suspendus
|
|
password = None
|
|
# RG14: on ne garde pas les comptes inactifs
|
|
if status == 'inactive':
|
|
self.counters['swarmids.ignores.inactive'] += 1
|
|
continue
|
|
self.counters['swarmids.importes'] += 1
|
|
self.counters['swarmids.importes.%s' % status] += 1
|
|
user_fixture.write_user(
|
|
swarmid=swarmid,
|
|
username=username,
|
|
password=password,
|
|
first_name=first_name,
|
|
last_name=last_name,
|
|
email=email,
|
|
last_login=last_login)
|
|
user_fixture.close()
|
|
|
|
def ignore_individu(self, cause, individualid):
|
|
self.counters['individus.ignores'] += 1
|
|
self.counters['individus.ignores.%s' % cause] += 1
|
|
|
|
@log('Loading individual... ')
|
|
def load_individuals(self):
|
|
sql = '''
|
|
WITH agg AS (SELECT ind.id as id, array_agg(ir1.relationtype) as relt1,
|
|
array_agg(ir2.relationtype) as relt2,
|
|
array_agg(users.id) as swarmids
|
|
FROM individual as ind
|
|
LEFT JOIN individualrelation as ir1 ON ir1.subjectid = ind.id
|
|
LEFT JOIN individualrelation as ir2 ON ir2.responsibleid = ind.id
|
|
LEFT JOIN users ON users.id = ind.swarmid GROUP BY ind.id)
|
|
SELECT DISTINCT ON (ind.id) ind.id, ind.state, ind.swarmid, ind.gender, ind.firstname, ind.lastname,
|
|
ind.nameofuse, ind.email, ind.phones::json,
|
|
ind.legalstatus, ind.birthdate, ind.mappings::json,
|
|
agg.relt1, agg.relt2, agg.swarmids
|
|
FROM individual as ind, agg WHERE ind.id = agg.id'''
|
|
|
|
self.cursor.execute(sql)
|
|
individu_batch = []
|
|
|
|
individu_schema = EntitySchema.objects.get(slug='individu')
|
|
self.individu_mapping = individu_mapping = {}
|
|
self.swarmids = set()
|
|
|
|
for (individualid, state, swarmid, gender, firstname, lastname, nameofuse, email, phones,
|
|
legalstatus, birthdate, mappings, subject_relations, actor_relations, swarmids) in self.cursor.fetchall():
|
|
# comptage des individus
|
|
self.counters['individus'] += 1
|
|
|
|
if gender == 'Female':
|
|
genre = 'femme'
|
|
elif gender == 'Male':
|
|
genre = 'homme'
|
|
elif gender == 'Other':
|
|
genre = 'autre'
|
|
else:
|
|
raise NotImplementedError('unknown gender: %s' % gender)
|
|
|
|
if legalstatus == 'Adulte':
|
|
statut_legal = 'majeur'
|
|
elif legalstatus == 'Enfant':
|
|
statut_legal = 'mineur'
|
|
elif legalstatus == 'Emancipe':
|
|
statut_legal = 'emancipe'
|
|
else:
|
|
raise NotImplementedError('unknown legalstatus: %s' % legalstatus)
|
|
|
|
self.counters['individus.%s' % statut_legal] += 1
|
|
self.counters['individus.%s.%s' % (statut_legal, genre)] += 1
|
|
if email:
|
|
self.counters['emails'] += 1
|
|
|
|
for key in mappings:
|
|
self.counters['cles_de_federation.%s' % key] += 1
|
|
|
|
telephones = []
|
|
for phone in phones:
|
|
if phone['phoneType'] == 'OtherPhone':
|
|
kind = 'autre'
|
|
elif phone['phoneType'] == 'Pro':
|
|
kind = 'pro'
|
|
elif phone['phoneType'] == 'Mobile':
|
|
kind = 'mobile'
|
|
elif phone['phoneType'] == 'Home':
|
|
kind = 'autre'
|
|
else:
|
|
raise NotImplementedError('unknown phoneType: %s' % phone['phoneType'])
|
|
telephones.append({'type': kind, 'numero': phone['number']})
|
|
|
|
mappings = mappings or {}
|
|
if swarmid and swarmids:
|
|
mappings['authentic'] = 'swarmid-%s' % swarmid
|
|
# RG8, RG9, RG9bis: reprise nom de naissance, nom d'usage
|
|
# on a tout on garde
|
|
if lastname and nameofuse:
|
|
nom_de_naissance = lastname.upper()
|
|
nom_d_usage = nameofuse.upper()
|
|
# cas des enfants et des hommes
|
|
elif statut_legal == 'mineur' or genre == 'homme':
|
|
nom_de_naissance = nameofuse.upper()
|
|
nom_d_usage = u''
|
|
# cas des femmes
|
|
else:
|
|
nom_de_naissance == ''
|
|
nom_d_usage = nameofuse.upper()
|
|
content = {
|
|
'genre': genre,
|
|
'prenoms': firstname.upper(),
|
|
'nom_de_naissance': nom_de_naissance,
|
|
'nom_d_usage': nom_d_usage,
|
|
'statut_legal': statut_legal,
|
|
'date_de_naissance': birthdate.isoformat(),
|
|
'cles_de_federation': mappings,
|
|
}
|
|
if not mappings and not subject_relations and not actor_relations:
|
|
# individu ignoré RG1: pas de clés, pas de réseau (pas de relations)
|
|
self.ignore_individu('RG1', individualid)
|
|
continue
|
|
if not mappings:
|
|
new_cursor = self.connection.cursor()
|
|
new_cursor.execute('''
|
|
SELECT DISTINCT(ind.id), ind.mappings::jsonb
|
|
FROM individual as ind
|
|
INNER JOIN individualrelation as ir ON
|
|
(ind.id = ir.subjectid AND ir.responsibleid = %s) OR
|
|
(ind.id = ir.responsibleid AND ir.subjectid = %s)''',
|
|
(individualid, individualid))
|
|
no_mappings = all(not reseau_mappings for reseau_id, reseau_mappings in
|
|
new_cursor.fetchall())
|
|
if no_mappings:
|
|
# individu ignoré RG2: pas de clés, et réseau sans clés
|
|
self.ignore_individu('RG2', individualid)
|
|
continue
|
|
if state == 'invalid':
|
|
# individu ignoré RG3: ficher invalide
|
|
self.ignore_individu('RG3', individualid)
|
|
continue
|
|
if state == 'invisible' and not mappings:
|
|
# individu ignoré RG4: fiche invisible et aucune clé
|
|
self.ignore_individu('RG4', individualid)
|
|
continue
|
|
if state == 'archived':
|
|
# individu ignoré RG5: fiche archivée
|
|
self.ignore_individu('RG5', individualid)
|
|
continue
|
|
if state == 'temp':
|
|
# individu ignoré RG5: fiche archivée
|
|
self.ignore_individu('RG6', individualid)
|
|
continue
|
|
|
|
if telephones:
|
|
content['telephones'] = telephones
|
|
if email:
|
|
self.counters['emails_importables'] += 1
|
|
email = email.strip()
|
|
if self.EMAIL_RE.match(email):
|
|
content['email'] = email
|
|
else: # RG11: on ignore les emails mal formattée
|
|
self.counters['emails_importables.ignores.RG11'] += 1
|
|
self.counters['individus_importes.%s.%s' % (statut_legal, genre)] += 1
|
|
for key in mappings:
|
|
self.counters['cles_de_federation.importes'] += 1
|
|
self.counters['cles_de_federation.importes.%s' % key] += 1
|
|
# enregistre les swarmid à importer
|
|
self.swarmids.add(swarmid)
|
|
new_id = individualid
|
|
if self.dry:
|
|
new_id = None
|
|
e = Entity(id=new_id,
|
|
created=self.tr,
|
|
schema=individu_schema,
|
|
content=content)
|
|
individu_mapping[individualid] = e
|
|
individu_batch.append(e)
|
|
|
|
self.cursor.execute('''SELECT id, individ FROM onlinepayment''')
|
|
|
|
max_tiers_saga = 0
|
|
for tiers_saga, individualid in self.cursor.fetchall():
|
|
self.counters['cles_de_federation.saga_tiers'] += 1
|
|
if individualid not in individu_mapping:
|
|
continue
|
|
self.counters['cles_de_federation.importes'] += 1
|
|
self.counters['cles_de_federation.importes.saga_tiers'] += 1
|
|
cles = individu_mapping[individualid].content['cles_de_federation']
|
|
max_tiers_saga = max(max_tiers_saga, int(str(tiers_saga[2:])))
|
|
cles['saga_tiers'] = str(tiers_saga)
|
|
|
|
Entity.objects.bulk_create(individu_batch)
|
|
connection.cursor().execute("SELECT setval(pg_get_serial_sequence('zoo_data_entity', 'id'),"
|
|
" coalesce(max(id),0) + 1, false) FROM zoo_data_entity")
|
|
self.saga_sequence = max_tiers_saga + 1
|
|
set_saga_sequence(self.saga_sequence)
|
|
|
|
@log('Loading addresses... ')
|
|
def load_addresses(self):
|
|
adresse_schema = EntitySchema.objects.get(slug='adresse')
|
|
habite_schema = RelationSchema.objects.get(slug=HABITE_REL)
|
|
|
|
adresse_batch = []
|
|
adresse_mapping = {}
|
|
|
|
self.individu_adresse_mapping = individu_adresse_mapping = {}
|
|
|
|
# Création des adresses
|
|
self.cursor.execute('''
|
|
SELECT
|
|
ia.individualid,
|
|
a.id,
|
|
ia.isprimary,
|
|
a.streetnumber,
|
|
a.streetnumberext,
|
|
a.streetname,
|
|
a.streetmatriculation,
|
|
a.ext1,
|
|
a.ext2,
|
|
a.at,
|
|
a.city,
|
|
a.zipcode,
|
|
a.country,
|
|
a.inseecode
|
|
FROM individualaddress as ia, address as a
|
|
WHERE
|
|
ia.addressid = a.id''')
|
|
for individualid, addressid, is_primary, streetnumber, streetnumberext, streetname, streetmatriculation, \
|
|
ext1, ext2, at, city, zipcode, country, inseecode in self.cursor.fetchall():
|
|
self.counters['relations_adresse'] += 1
|
|
# RG10 bis: on retire le préfixe chez
|
|
if at and at.lower().strip().startswith('chez'):
|
|
at = at.strip()[4:].strip()
|
|
if addressid not in adresse_mapping:
|
|
self.counters['adresses'] += 1
|
|
|
|
if individualid not in self.individu_mapping:
|
|
self.counters['relations_adresse.ignores'] += 1
|
|
self.counters['relations_adresse.ignores.individu_ignore'] += 1
|
|
continue
|
|
if streetnumber:
|
|
streetnumber = streetnumber.strip()
|
|
content = {
|
|
# RG10: initialisation du numéro de rue à zéro si vide
|
|
'streetnumber': streetnumber or '0',
|
|
'streetnumberext': streetnumberext,
|
|
'streetname': streetname,
|
|
'streetmatriculation': streetmatriculation,
|
|
'ext1': ext1,
|
|
'ext2': ext2,
|
|
'at': at,
|
|
'city': city,
|
|
'zipcode': zipcode,
|
|
'country': country,
|
|
'inseecode': inseecode
|
|
}
|
|
e = Entity(created=self.tr, schema=adresse_schema, content=content)
|
|
adresse_batch.append(e)
|
|
adresse_mapping[addressid] = e
|
|
if individualid not in self.individu_mapping:
|
|
self.counters['relations_adresse.ignores.individu_ignore'] += 1
|
|
continue
|
|
self.counters['relations_adresse.importes'] += 1
|
|
individu_adresse_mapping[(individualid, addressid)] = is_primary
|
|
|
|
self.counters['adresses.importes'] = len(adresse_mapping)
|
|
Entity.objects.bulk_create(adresse_batch)
|
|
|
|
relation_batch = []
|
|
for (a, b), is_primary in individu_adresse_mapping.items():
|
|
content = {
|
|
'principale': is_primary,
|
|
}
|
|
e = Relation(created=self.tr,
|
|
left=self.individu_mapping[a],
|
|
right=adresse_mapping[b],
|
|
schema=habite_schema,
|
|
content=content)
|
|
relation_batch.append(e)
|
|
|
|
Relation.objects.bulk_create(relation_batch)
|
|
|
|
@log('Loading relations... ')
|
|
def load_relations(self):
|
|
responsabilite_legale_schema = RelationSchema.objects.get(slug=RESPONSABILITE_LEGALE_REL)
|
|
union_schema = RelationSchema.objects.get(slug=UNION_REL)
|
|
|
|
# Création des relations entre individus
|
|
self.cursor.execute('''
|
|
SELECT
|
|
label,
|
|
relationtype,
|
|
responsibleid,
|
|
subjectid
|
|
FROM individualrelation''')
|
|
|
|
relation_batch = []
|
|
|
|
for label, relationtype, responsibleid, subjectid in self.cursor.fetchall():
|
|
self.counters['relations'] += 1
|
|
self.counters['relations.%s' % relationtype] += 1
|
|
self.counters['relations.%s.%s' % (relationtype, label)] += 1
|
|
|
|
if relationtype == 'SituationFamiliale':
|
|
schema = union_schema
|
|
if label == 'Marie':
|
|
kind = 'pacs/mariage'
|
|
elif label == 'Pacse':
|
|
kind = 'pacs/mariage'
|
|
elif label == 'UnionLibre':
|
|
kind = 'unionlibre'
|
|
else:
|
|
raise NotImplementedError('unknown label for relationtype: %s, %s'
|
|
% (label, relationtype))
|
|
content = {'statut': kind}
|
|
elif relationtype == 'ResponsabiliteLegale':
|
|
schema = responsabilite_legale_schema
|
|
if label == 'Parent':
|
|
kind = 'parent'
|
|
elif label == 'TiersDeConfiance':
|
|
kind = 'tiers_de_confiance'
|
|
elif label == 'RepresentantPersonneMoraleQualifiee':
|
|
kind = 'representant_personne_morale_qualifiee'
|
|
elif label == 'Tuteur':
|
|
# conversion tutelle en tiers de confiance
|
|
self.counters['relations_modifies.RG7bis'] += 1
|
|
kind = 'tiers_de_confiance'
|
|
elif label == 'Curatelle':
|
|
self.counters['relations.ignores.RG7' % relationtype] += 1
|
|
continue
|
|
else:
|
|
raise NotImplementedError('unknown label for relationtype: %s, %s'
|
|
% (label, relationtype))
|
|
content = {'statut': kind}
|
|
if responsibleid not in self.individu_mapping or subjectid not in self.individu_mapping:
|
|
self.counters['relations.ignores.individu_ignore'] += 1
|
|
continue
|
|
if relationtype == 'ResponsabiliteLegale' and label == 'Tuteur':
|
|
self.counters['relations_modifies.RG7bis.importes'] += 1
|
|
self.counters['relations_importes'] += 1
|
|
self.counters['relations_importes.%s' % relationtype] += 1
|
|
self.counters['relations_importes.%s.%s' % (relationtype, kind)] += 1
|
|
e = Relation(created=self.tr, schema=schema, left=self.individu_mapping[responsibleid],
|
|
right=self.individu_mapping[subjectid], content=content)
|
|
relation_batch.append(e)
|
|
|
|
Relation.objects.bulk_create(relation_batch)
|
|
|
|
|
|
def is_majeur(individu):
|
|
return individu.content['statut_legal'] == 'majeur'
|
|
|
|
|
|
def is_mineur(individu):
|
|
return individu.content['statut_legal'] == 'mineur'
|
|
|
|
|
|
def get_application(application):
|
|
app_dfn = settings.ZOO_NANTERRE_APPLICATIONS.get(application)
|
|
assert app_dfn, 'application %s does not exist' % application
|
|
return app_dfn
|
|
|
|
|
|
def get_applications(**kwargs):
|
|
def helper():
|
|
for app_id, app_dfn in settings.ZOO_NANTERRE_APPLICATIONS.items():
|
|
for key, value in kwargs.items():
|
|
if value is True and not app_dfn.get(key):
|
|
continue
|
|
yield app_id
|
|
return list(helper())
|
|
|
|
|
|
def journalize(individu, transaction=None, meta=None, **kwargs):
|
|
content = kwargs
|
|
if meta:
|
|
content['meta'] = meta
|
|
Log.objects.create(
|
|
entity=individu,
|
|
transaction=transaction,
|
|
content=content)
|
|
|
|
# Saga sequence helper
|
|
|
|
|
|
def set_saga_sequence(value):
|
|
cursor = connection.cursor()
|
|
cursor.execute('SELECT 1 FROM information_schema.sequences '
|
|
'WHERE sequence_schema = current_schema() '
|
|
'AND sequence_name = \'zoo_nanterre_saga_seq\'')
|
|
if not cursor.fetchall():
|
|
cursor.execute('CREATE SEQUENCE zoo_nanterre_saga_seq START %d' % value)
|
|
cursor.execute('SELECT setval(\'zoo_nanterre_saga_seq\', %d, false)' % value)
|
|
|
|
|
|
def get_next_saga_sequence():
|
|
cursor = connection.cursor()
|
|
cursor.execute('SELECT nextval(\'zoo_nanterre_saga_seq\')')
|
|
return cursor.fetchone()[0]
|
|
|
|
|
|
def pair_sort(a, b):
|
|
return (a, b) if a < b else (b, a)
|
|
|
|
|
|
def individu_caption(individu):
|
|
c = individu.content
|
|
s = c['nom_de_naissance']
|
|
if c['nom_d_usage']:
|
|
if s:
|
|
s = u' (%s)' % s
|
|
s = c['nom_d_usage'] + s
|
|
if c['prenoms']:
|
|
if s:
|
|
s = ' ' + s
|
|
s = c['prenoms'] + s
|
|
if c['date_de_naissance']:
|
|
s += u' - ' + c['date_de_naissance']
|
|
return s
|