cd06/senior: improve loader and prevent provisionning

This commit is contained in:
Benjamin Dauvergne 2020-12-11 10:23:58 +01:00
parent 4759fd6b0f
commit d55d9fba55
3 changed files with 172 additions and 148 deletions

View File

@ -1,6 +1,5 @@
import sys
import loader
import re
keys, data = loader.load(sys.argv[1])

View File

@ -1,5 +1,5 @@
import sys
import loader
import logging
from django.db.transaction import atomic
@ -8,149 +8,157 @@ from authentic2.a2_rbac.models import OrganizationalUnit as OU
from django.contrib.contenttypes.models import ContentType
from authentic2.models import UserExternalId, Attribute, AttributeValue
user_ct = ContentType.objects.get_for_model(User)
user_attributes_mapping = {
'Prenom': 'first_name',
'Nom': 'last_name',
'Nom_JF': 'nom_de_naissance',
'Civilite': 'title',
'birthdate': 'birthdate',
'Lieu_Naissance': 'lieu_de_naissance',
'tel_mobile': 'mobile',
'tel_fixe': 'phone',
'NO_Voie': 'address',
'Batiment_Residence': 'complement_d_adresse',
'CP': 'zipcode',
'Ville': 'city',
}
from hobo.agent.authentic2.provisionning import provisionning
attributes = {attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values())}
assert set(attributes.keys()) == set(user_attributes_mapping.values())
from . import loader
logger = loader.logger
try:
user_ct = ContentType.objects.get_for_model(User)
keys, data = loader.load(sys.argv[1])
user_attributes_mapping = {
'Prenom': 'first_name',
'Nom': 'last_name',
'Nom_JF': 'nom_de_naissance',
'Civilite': 'title',
'birthdate': 'birthdate',
'Lieu_Naissance': 'lieu_de_naissance',
'tel_mobile': 'mobile',
'tel_fixe': 'phone',
'NO_Voie': 'address',
'Batiment_Residence': 'complement_d_adresse',
'CP': 'zipcode',
'Ville': 'city',
}
logger.handlers[0].level = logging.INFO
attributes = {
attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values())
}
assert set(attributes.keys()) == set(user_attributes_mapping.values())
with atomic():
senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'})
logger = loader.logger
guids = {row['guid']: row for row in data}
keys, data = loader.load(sys.argv[1])
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
logger.handlers[0].level = logging.INFO
with atomic():
senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'})
to_bulk_create = []
to_save = []
guids = {row['guid']: row for row in data}
logger.info('Creating users...')
for i, row in enumerate(data):
guid = row['guid']
user = user_by_guid.get(guid, User(uuid=guid))
save = not bool(user.pk)
defaults = {
'ou': senior_ou,
'first_name': row['Prenom'],
'last_name': row['Nom'],
'email': row['email'] or '',
'email_verified': bool(row['Email']),
}
for key, value in defaults.items():
if getattr(user, key) != value:
setattr(user, key, value)
save = True
if save:
if user.pk:
to_save.append(user)
else:
to_bulk_create.append(user)
User.objects.bulk_create(to_bulk_create)
for user in to_save:
user.save()
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
user_ids = set(user.pk for user in user_by_guid.values())
external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)}
logger.info('Created %s users...', len(to_bulk_create))
logger.info('Updated %s users...', len(to_save))
logger.info('Creating UserExternalId...')
to_bulk_create = []
to_save = []
for user in user_by_guid.values():
ppid = guids[user.uuid]['ppid']
uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet'))
save = not bool(uei.pk)
if uei.source != 'eudonet':
uei.source = 'eudonet'
save = True
if uei.external_id != ppid:
uei.external_id = ppid
save = True
if save:
if uei.pk:
to_save.append(uei)
else:
to_bulk_create.append(uei)
UserExternalId.objects.bulk_create(to_bulk_create)
for uei in to_save:
uei.save()
logger.info('Created %s user external ids...', len(to_bulk_create))
logger.info('Updated %s user external ids...', len(to_save))
for eudonet_key, attribute_name in user_attributes_mapping.items():
attribute = attributes[attribute_name]
serialize = attribute.get_kind()['serialize']
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
to_bulk_create = []
to_save = []
to_delete = []
logger.info('Creating attributes %s...', attribute_name)
atvs = {atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute)}
for row in data:
user = user_by_guid[row['guid']]
value = row[eudonet_key]
atv = atvs.get(user.pk, AttributeValue(
content_type=user_ct,
object_id=user.pk,
attribute=attribute,
verified=False,
multiple=False))
if not value:
if atv.pk:
to_delete.append(atv.pk)
else:
serialized = serialize(value)
if not atv.pk:
atv.content = serialized
to_bulk_create.append(atv)
elif atv.content != serialized:
atv.content = serialized
to_save.append(atv)
logger.info('Creating users...')
for i, row in enumerate(data):
guid = row['guid']
AttributeValue.objects.bulk_create(to_bulk_create)
for atv in to_save:
atv.save()
AttributeValue.objects.filter(pk__in=to_delete).delete()
user = user_by_guid.get(guid, User(uuid=guid))
logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name)
logger.info('Updated %s %s attributes ...', len(to_save), attribute_name)
logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name)
save = not bool(user.pk)
if len(sys.argv) < 3 or sys.argv[2] != 'nofake':
raise ValueError('fake')
defaults = {
'ou': senior_ou,
'first_name': row['Prenom'],
'last_name': row['Nom'],
'email': row['email'] or '',
'email_verified': bool(row['Email']),
}
for key, value in defaults.items():
if getattr(user, key) != value:
setattr(user, key, value)
save = True
if save:
if user.pk:
to_save.append(user)
else:
to_bulk_create.append(user)
User.objects.bulk_create(to_bulk_create)
for user in to_save:
user.save()
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
user_ids = set(user.pk for user in user_by_guid.values())
external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)}
logger.info('Created %s users...', len(to_bulk_create))
logger.info('Updated %s users...', len(to_save))
logger.info('Creating UserExternalId...')
to_bulk_create = []
to_save = []
for user in user_by_guid.values():
ppid = guids[user.uuid]['ppid']
uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet'))
save = not bool(uei.pk)
if uei.source != 'eudonet':
uei.source = 'eudonet'
save = True
if uei.external_id != ppid:
uei.external_id = ppid
save = True
if save:
if uei.pk:
to_save.append(uei)
else:
to_bulk_create.append(uei)
UserExternalId.objects.bulk_create(to_bulk_create)
for uei in to_save:
uei.save()
logger.info('Created %s user external ids...', len(to_bulk_create))
logger.info('Updated %s user external ids...', len(to_save))
for eudonet_key, attribute_name in user_attributes_mapping.items():
attribute = attributes[attribute_name]
serialize = attribute.get_kind()['serialize']
to_bulk_create = []
to_save = []
to_delete = []
logger.info('Creating attributes %s...', attribute_name)
atvs = {
atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute)
}
for row in data:
user = user_by_guid[row['guid']]
value = row[eudonet_key]
atv = atvs.get(user.pk, AttributeValue(
content_type=user_ct,
object_id=user.pk,
attribute=attribute,
verified=False,
multiple=False))
if not value:
if atv.pk:
to_delete.append(atv.pk)
else:
serialized = serialize(value)
if not atv.pk:
atv.content = serialized
to_bulk_create.append(atv)
elif atv.content != serialized:
atv.content = serialized
to_save.append(atv)
AttributeValue.objects.bulk_create(to_bulk_create)
for atv in to_save:
atv.save()
AttributeValue.objects.filter(pk__in=to_delete).delete()
logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name)
logger.info('Updated %s %s attributes ...', len(to_save), attribute_name)
logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name)
if len(sys.argv) < 3 or sys.argv[2] != 'nofake':
raise ValueError('fake')
finally:
provisionning.clear()

View File

@ -18,7 +18,7 @@ logger.addHandler(handler)
# Adresse_Principale
# Annee_Inscription
# Annotations_particulières2
# Archivé
# Contact_Archivé
# Autorise_Photos
# Batiment_Residence
# CP
@ -68,7 +68,7 @@ expected_keys = set([
'Lien_relationnel',
'Annotations_particulières2',
'Profil_Contact',
'Archivé',
'Contact_Archivé',
'Téléphone_fixe',
'NO_Voie',
'Batiment_Residence',
@ -89,11 +89,21 @@ def telephone(row, key):
mobile = row[key]
if mobile:
mobile = mobile.strip()
if mobile == 'NULL':
if re.match(r'PAS.*(TEL|FIXE)', mobile) or re.match(r'n[ée]ant', mobile.strip().lower()):
mobile = ''
elif mobile == 'NULL':
mobile = ''
else:
mobile = re.sub(r'[\s.-]', '', mobile).strip()
if not mobile.isascii() or not mobile.isdigit() or len(mobile) not in (9, 10):
def check_mobile():
if not mobile.isascii():
return False
if not mobile.isdigit():
return False
if not mobile.startswith('00') and len(mobile) not in (9, 10):
return False
return True
if not check_mobile():
logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid {key} {row[key]}')
mobile = ''
if len(mobile) == 9:
@ -123,7 +133,7 @@ def normalize(row, ppid_index):
assert row['ppid'] not in ppids, 'duplicate ppid line %s' % line
ppids.add(row['ppid'])
assert row['Civilite'] in (None, 'Monsieur', 'Madame')
# email
email = row['Email']
if email:
@ -134,18 +144,23 @@ def normalize(row, ppid_index):
# CP
cp = row['CP']
if cp == '0600':
cp = '06000'
if re.match(r'6\s\d\d\d', cp):
cp = re.sub(r'\s', '', cp)
if re.match(r'6\d\d\d', cp):
cp = '0' + cp
if cp != row['CP']:
row['CP'] = cp
if cp is None:
pass
else:
if cp == '0600':
cp = '06000'
if re.match(r'6\s\d\d\d', cp):
cp = re.sub(r'\s', '', cp)
if re.match(r'6\d\d\d', cp):
cp = '0' + cp
if len(cp) != 5:
logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid CP {row["CP"]}')
if cp != row['CP']:
row['CP'] = cp
# Archivé
if row['Archivé'] == 'OUI':
logger.warning(f'line {line} ppid {row["ppid"]} : Archivé==OUI')
# Contact_Archivé
if row['Contact_Archivé'] == 'OUI':
logger.warning(f'line {line} ppid {row["ppid"]} : Contact_Archivé==OUI')
row['import'] = False
# UUID
@ -218,7 +233,7 @@ def load(filename, number_of_rows=None):
ppid_index = {row['ppid'].strip(): i for i, row in enumerate(data)}
for i, row in enumerate(data):
assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {row.keys()} != {keys}'
assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {set(row.keys()) - expected_keys} and {expected_keys - set(row.keys())}'
error = False
for i, row in enumerate(data):
@ -230,4 +245,6 @@ def load(filename, number_of_rows=None):
error = True
assert not error
print('archivé:', len([d for d in data if d['Contact_Archivé'] == 'OUI']))
return reader._fieldnames, [row for row in data if row['import']]