diff --git a/cd06/senior/analyse.py b/cd06/senior/analyse.py index 07c1103..027cd7e 100644 --- a/cd06/senior/analyse.py +++ b/cd06/senior/analyse.py @@ -1,6 +1,5 @@ import sys import loader -import re keys, data = loader.load(sys.argv[1]) diff --git a/cd06/senior/load-in-authentic.py b/cd06/senior/load-in-authentic.py index 9f08a0b..94ed99b 100644 --- a/cd06/senior/load-in-authentic.py +++ b/cd06/senior/load-in-authentic.py @@ -1,5 +1,5 @@ import sys -import loader +import logging from django.db.transaction import atomic @@ -8,149 +8,157 @@ from authentic2.a2_rbac.models import OrganizationalUnit as OU from django.contrib.contenttypes.models import ContentType from authentic2.models import UserExternalId, Attribute, AttributeValue -user_ct = ContentType.objects.get_for_model(User) - -user_attributes_mapping = { - 'Prenom': 'first_name', - 'Nom': 'last_name', - 'Nom_JF': 'nom_de_naissance', - 'Civilite': 'title', - 'birthdate': 'birthdate', - 'Lieu_Naissance': 'lieu_de_naissance', - 'tel_mobile': 'mobile', - 'tel_fixe': 'phone', - 'NO_Voie': 'address', - 'Batiment_Residence': 'complement_d_adresse', - 'CP': 'zipcode', - 'Ville': 'city', -} +from hobo.agent.authentic2.provisionning import provisionning -attributes = {attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values())} -assert set(attributes.keys()) == set(user_attributes_mapping.values()) +from . import loader -logger = loader.logger +try: + user_ct = ContentType.objects.get_for_model(User) -keys, data = loader.load(sys.argv[1]) + user_attributes_mapping = { + 'Prenom': 'first_name', + 'Nom': 'last_name', + 'Nom_JF': 'nom_de_naissance', + 'Civilite': 'title', + 'birthdate': 'birthdate', + 'Lieu_Naissance': 'lieu_de_naissance', + 'tel_mobile': 'mobile', + 'tel_fixe': 'phone', + 'NO_Voie': 'address', + 'Batiment_Residence': 'complement_d_adresse', + 'CP': 'zipcode', + 'Ville': 'city', + } -logger.handlers[0].level = logging.INFO + attributes = { + attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values()) + } + assert set(attributes.keys()) == set(user_attributes_mapping.values()) -with atomic(): - senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'}) + logger = loader.logger - guids = {row['guid']: row for row in data} + keys, data = loader.load(sys.argv[1]) - user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())} + logger.handlers[0].level = logging.INFO + with atomic(): + senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'}) - to_bulk_create = [] - to_save = [] + guids = {row['guid']: row for row in data} - logger.info('Creating users...') - for i, row in enumerate(data): - guid = row['guid'] - - user = user_by_guid.get(guid, User(uuid=guid)) - - save = not bool(user.pk) - - defaults = { - 'ou': senior_ou, - 'first_name': row['Prenom'], - 'last_name': row['Nom'], - 'email': row['email'] or '', - 'email_verified': bool(row['Email']), - } - for key, value in defaults.items(): - if getattr(user, key) != value: - setattr(user, key, value) - save = True - - if save: - if user.pk: - to_save.append(user) - else: - to_bulk_create.append(user) - - User.objects.bulk_create(to_bulk_create) - for user in to_save: - user.save() - - user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())} - - user_ids = set(user.pk for user in user_by_guid.values()) - - external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)} - - logger.info('Created %s users...', len(to_bulk_create)) - logger.info('Updated %s users...', len(to_save)) - - logger.info('Creating UserExternalId...') - to_bulk_create = [] - to_save = [] - for user in user_by_guid.values(): - ppid = guids[user.uuid]['ppid'] - uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet')) - save = not bool(uei.pk) - if uei.source != 'eudonet': - uei.source = 'eudonet' - save = True - if uei.external_id != ppid: - uei.external_id = ppid - save = True - - if save: - if uei.pk: - to_save.append(uei) - else: - to_bulk_create.append(uei) - - UserExternalId.objects.bulk_create(to_bulk_create) - for uei in to_save: - uei.save() - logger.info('Created %s user external ids...', len(to_bulk_create)) - logger.info('Updated %s user external ids...', len(to_save)) - - - - for eudonet_key, attribute_name in user_attributes_mapping.items(): - attribute = attributes[attribute_name] - serialize = attribute.get_kind()['serialize'] + user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())} to_bulk_create = [] to_save = [] - to_delete = [] - logger.info('Creating attributes %s...', attribute_name) - atvs = {atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute)} - for row in data: - user = user_by_guid[row['guid']] - value = row[eudonet_key] - atv = atvs.get(user.pk, AttributeValue( - content_type=user_ct, - object_id=user.pk, - attribute=attribute, - verified=False, - multiple=False)) - if not value: - if atv.pk: - to_delete.append(atv.pk) - else: - serialized = serialize(value) - if not atv.pk: - atv.content = serialized - to_bulk_create.append(atv) - elif atv.content != serialized: - atv.content = serialized - to_save.append(atv) + logger.info('Creating users...') + for i, row in enumerate(data): + guid = row['guid'] - AttributeValue.objects.bulk_create(to_bulk_create) - for atv in to_save: - atv.save() - AttributeValue.objects.filter(pk__in=to_delete).delete() + user = user_by_guid.get(guid, User(uuid=guid)) - logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name) - logger.info('Updated %s %s attributes ...', len(to_save), attribute_name) - logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name) + save = not bool(user.pk) - if len(sys.argv) < 3 or sys.argv[2] != 'nofake': - raise ValueError('fake') + defaults = { + 'ou': senior_ou, + 'first_name': row['Prenom'], + 'last_name': row['Nom'], + 'email': row['email'] or '', + 'email_verified': bool(row['Email']), + } + for key, value in defaults.items(): + if getattr(user, key) != value: + setattr(user, key, value) + save = True + + if save: + if user.pk: + to_save.append(user) + else: + to_bulk_create.append(user) + + User.objects.bulk_create(to_bulk_create) + for user in to_save: + user.save() + + user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())} + + user_ids = set(user.pk for user in user_by_guid.values()) + + external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)} + + logger.info('Created %s users...', len(to_bulk_create)) + logger.info('Updated %s users...', len(to_save)) + + logger.info('Creating UserExternalId...') + to_bulk_create = [] + to_save = [] + for user in user_by_guid.values(): + ppid = guids[user.uuid]['ppid'] + uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet')) + save = not bool(uei.pk) + if uei.source != 'eudonet': + uei.source = 'eudonet' + save = True + if uei.external_id != ppid: + uei.external_id = ppid + save = True + + if save: + if uei.pk: + to_save.append(uei) + else: + to_bulk_create.append(uei) + + UserExternalId.objects.bulk_create(to_bulk_create) + for uei in to_save: + uei.save() + logger.info('Created %s user external ids...', len(to_bulk_create)) + logger.info('Updated %s user external ids...', len(to_save)) + + for eudonet_key, attribute_name in user_attributes_mapping.items(): + attribute = attributes[attribute_name] + serialize = attribute.get_kind()['serialize'] + + to_bulk_create = [] + to_save = [] + to_delete = [] + + logger.info('Creating attributes %s...', attribute_name) + atvs = { + atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute) + } + for row in data: + user = user_by_guid[row['guid']] + value = row[eudonet_key] + atv = atvs.get(user.pk, AttributeValue( + content_type=user_ct, + object_id=user.pk, + attribute=attribute, + verified=False, + multiple=False)) + if not value: + if atv.pk: + to_delete.append(atv.pk) + else: + serialized = serialize(value) + if not atv.pk: + atv.content = serialized + to_bulk_create.append(atv) + elif atv.content != serialized: + atv.content = serialized + to_save.append(atv) + + AttributeValue.objects.bulk_create(to_bulk_create) + for atv in to_save: + atv.save() + AttributeValue.objects.filter(pk__in=to_delete).delete() + + logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name) + logger.info('Updated %s %s attributes ...', len(to_save), attribute_name) + logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name) + + if len(sys.argv) < 3 or sys.argv[2] != 'nofake': + raise ValueError('fake') +finally: + provisionning.clear() diff --git a/cd06/senior/loader.py b/cd06/senior/loader.py index 2c583df..538979a 100644 --- a/cd06/senior/loader.py +++ b/cd06/senior/loader.py @@ -18,7 +18,7 @@ logger.addHandler(handler) # Adresse_Principale # Annee_Inscription # Annotations_particulières2 -# Archivé +# Contact_Archivé # Autorise_Photos # Batiment_Residence # CP @@ -68,7 +68,7 @@ expected_keys = set([ 'Lien_relationnel', 'Annotations_particulières2', 'Profil_Contact', - 'Archivé', + 'Contact_Archivé', 'Téléphone_fixe', 'NO_Voie', 'Batiment_Residence', @@ -89,11 +89,21 @@ def telephone(row, key): mobile = row[key] if mobile: mobile = mobile.strip() - if mobile == 'NULL': + if re.match(r'PAS.*(TEL|FIXE)', mobile) or re.match(r'n[ée]ant', mobile.strip().lower()): + mobile = '' + elif mobile == 'NULL': mobile = '' else: mobile = re.sub(r'[\s.-]', '', mobile).strip() - if not mobile.isascii() or not mobile.isdigit() or len(mobile) not in (9, 10): + def check_mobile(): + if not mobile.isascii(): + return False + if not mobile.isdigit(): + return False + if not mobile.startswith('00') and len(mobile) not in (9, 10): + return False + return True + if not check_mobile(): logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid {key} {row[key]}') mobile = '' if len(mobile) == 9: @@ -123,7 +133,7 @@ def normalize(row, ppid_index): assert row['ppid'] not in ppids, 'duplicate ppid line %s' % line ppids.add(row['ppid']) assert row['Civilite'] in (None, 'Monsieur', 'Madame') - + # email email = row['Email'] if email: @@ -134,18 +144,23 @@ def normalize(row, ppid_index): # CP cp = row['CP'] - if cp == '0600': - cp = '06000' - if re.match(r'6\s\d\d\d', cp): - cp = re.sub(r'\s', '', cp) - if re.match(r'6\d\d\d', cp): - cp = '0' + cp - if cp != row['CP']: - row['CP'] = cp + if cp is None: + pass + else: + if cp == '0600': + cp = '06000' + if re.match(r'6\s\d\d\d', cp): + cp = re.sub(r'\s', '', cp) + if re.match(r'6\d\d\d', cp): + cp = '0' + cp + if len(cp) != 5: + logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid CP {row["CP"]}') + if cp != row['CP']: + row['CP'] = cp - # Archivé - if row['Archivé'] == 'OUI': - logger.warning(f'line {line} ppid {row["ppid"]} : Archivé==OUI') + # Contact_Archivé + if row['Contact_Archivé'] == 'OUI': + logger.warning(f'line {line} ppid {row["ppid"]} : Contact_Archivé==OUI') row['import'] = False # UUID @@ -218,7 +233,7 @@ def load(filename, number_of_rows=None): ppid_index = {row['ppid'].strip(): i for i, row in enumerate(data)} for i, row in enumerate(data): - assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {row.keys()} != {keys}' + assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {set(row.keys()) - expected_keys} and {expected_keys - set(row.keys())}' error = False for i, row in enumerate(data): @@ -230,4 +245,6 @@ def load(filename, number_of_rows=None): error = True assert not error + print('archivé:', len([d for d in data if d['Contact_Archivé'] == 'OUI'])) + return reader._fieldnames, [row for row in data if row['import']]