cd06/senior: improve loader and prevent provisionning
This commit is contained in:
parent
4759fd6b0f
commit
d55d9fba55
|
@ -1,6 +1,5 @@
|
|||
import sys
|
||||
import loader
|
||||
import re
|
||||
|
||||
keys, data = loader.load(sys.argv[1])
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import sys
|
||||
import loader
|
||||
import logging
|
||||
|
||||
from django.db.transaction import atomic
|
||||
|
||||
|
@ -8,149 +8,157 @@ from authentic2.a2_rbac.models import OrganizationalUnit as OU
|
|||
from django.contrib.contenttypes.models import ContentType
|
||||
from authentic2.models import UserExternalId, Attribute, AttributeValue
|
||||
|
||||
user_ct = ContentType.objects.get_for_model(User)
|
||||
|
||||
user_attributes_mapping = {
|
||||
'Prenom': 'first_name',
|
||||
'Nom': 'last_name',
|
||||
'Nom_JF': 'nom_de_naissance',
|
||||
'Civilite': 'title',
|
||||
'birthdate': 'birthdate',
|
||||
'Lieu_Naissance': 'lieu_de_naissance',
|
||||
'tel_mobile': 'mobile',
|
||||
'tel_fixe': 'phone',
|
||||
'NO_Voie': 'address',
|
||||
'Batiment_Residence': 'complement_d_adresse',
|
||||
'CP': 'zipcode',
|
||||
'Ville': 'city',
|
||||
}
|
||||
from hobo.agent.authentic2.provisionning import provisionning
|
||||
|
||||
attributes = {attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values())}
|
||||
assert set(attributes.keys()) == set(user_attributes_mapping.values())
|
||||
from . import loader
|
||||
|
||||
logger = loader.logger
|
||||
try:
|
||||
user_ct = ContentType.objects.get_for_model(User)
|
||||
|
||||
keys, data = loader.load(sys.argv[1])
|
||||
user_attributes_mapping = {
|
||||
'Prenom': 'first_name',
|
||||
'Nom': 'last_name',
|
||||
'Nom_JF': 'nom_de_naissance',
|
||||
'Civilite': 'title',
|
||||
'birthdate': 'birthdate',
|
||||
'Lieu_Naissance': 'lieu_de_naissance',
|
||||
'tel_mobile': 'mobile',
|
||||
'tel_fixe': 'phone',
|
||||
'NO_Voie': 'address',
|
||||
'Batiment_Residence': 'complement_d_adresse',
|
||||
'CP': 'zipcode',
|
||||
'Ville': 'city',
|
||||
}
|
||||
|
||||
logger.handlers[0].level = logging.INFO
|
||||
attributes = {
|
||||
attribute.name: attribute for attribute in Attribute.objects.filter(name__in=user_attributes_mapping.values())
|
||||
}
|
||||
assert set(attributes.keys()) == set(user_attributes_mapping.values())
|
||||
|
||||
with atomic():
|
||||
senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'})
|
||||
logger = loader.logger
|
||||
|
||||
guids = {row['guid']: row for row in data}
|
||||
keys, data = loader.load(sys.argv[1])
|
||||
|
||||
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
|
||||
logger.handlers[0].level = logging.INFO
|
||||
|
||||
with atomic():
|
||||
senior_ou, created = OU.objects.update_or_create(slug='senior', defaults={'name': 'Sénior'})
|
||||
|
||||
to_bulk_create = []
|
||||
to_save = []
|
||||
guids = {row['guid']: row for row in data}
|
||||
|
||||
logger.info('Creating users...')
|
||||
for i, row in enumerate(data):
|
||||
guid = row['guid']
|
||||
|
||||
user = user_by_guid.get(guid, User(uuid=guid))
|
||||
|
||||
save = not bool(user.pk)
|
||||
|
||||
defaults = {
|
||||
'ou': senior_ou,
|
||||
'first_name': row['Prenom'],
|
||||
'last_name': row['Nom'],
|
||||
'email': row['email'] or '',
|
||||
'email_verified': bool(row['Email']),
|
||||
}
|
||||
for key, value in defaults.items():
|
||||
if getattr(user, key) != value:
|
||||
setattr(user, key, value)
|
||||
save = True
|
||||
|
||||
if save:
|
||||
if user.pk:
|
||||
to_save.append(user)
|
||||
else:
|
||||
to_bulk_create.append(user)
|
||||
|
||||
User.objects.bulk_create(to_bulk_create)
|
||||
for user in to_save:
|
||||
user.save()
|
||||
|
||||
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
|
||||
|
||||
user_ids = set(user.pk for user in user_by_guid.values())
|
||||
|
||||
external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)}
|
||||
|
||||
logger.info('Created %s users...', len(to_bulk_create))
|
||||
logger.info('Updated %s users...', len(to_save))
|
||||
|
||||
logger.info('Creating UserExternalId...')
|
||||
to_bulk_create = []
|
||||
to_save = []
|
||||
for user in user_by_guid.values():
|
||||
ppid = guids[user.uuid]['ppid']
|
||||
uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet'))
|
||||
save = not bool(uei.pk)
|
||||
if uei.source != 'eudonet':
|
||||
uei.source = 'eudonet'
|
||||
save = True
|
||||
if uei.external_id != ppid:
|
||||
uei.external_id = ppid
|
||||
save = True
|
||||
|
||||
if save:
|
||||
if uei.pk:
|
||||
to_save.append(uei)
|
||||
else:
|
||||
to_bulk_create.append(uei)
|
||||
|
||||
UserExternalId.objects.bulk_create(to_bulk_create)
|
||||
for uei in to_save:
|
||||
uei.save()
|
||||
logger.info('Created %s user external ids...', len(to_bulk_create))
|
||||
logger.info('Updated %s user external ids...', len(to_save))
|
||||
|
||||
|
||||
|
||||
for eudonet_key, attribute_name in user_attributes_mapping.items():
|
||||
attribute = attributes[attribute_name]
|
||||
serialize = attribute.get_kind()['serialize']
|
||||
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
|
||||
|
||||
to_bulk_create = []
|
||||
to_save = []
|
||||
to_delete = []
|
||||
|
||||
logger.info('Creating attributes %s...', attribute_name)
|
||||
atvs = {atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute)}
|
||||
for row in data:
|
||||
user = user_by_guid[row['guid']]
|
||||
value = row[eudonet_key]
|
||||
atv = atvs.get(user.pk, AttributeValue(
|
||||
content_type=user_ct,
|
||||
object_id=user.pk,
|
||||
attribute=attribute,
|
||||
verified=False,
|
||||
multiple=False))
|
||||
if not value:
|
||||
if atv.pk:
|
||||
to_delete.append(atv.pk)
|
||||
else:
|
||||
serialized = serialize(value)
|
||||
if not atv.pk:
|
||||
atv.content = serialized
|
||||
to_bulk_create.append(atv)
|
||||
elif atv.content != serialized:
|
||||
atv.content = serialized
|
||||
to_save.append(atv)
|
||||
logger.info('Creating users...')
|
||||
for i, row in enumerate(data):
|
||||
guid = row['guid']
|
||||
|
||||
AttributeValue.objects.bulk_create(to_bulk_create)
|
||||
for atv in to_save:
|
||||
atv.save()
|
||||
AttributeValue.objects.filter(pk__in=to_delete).delete()
|
||||
user = user_by_guid.get(guid, User(uuid=guid))
|
||||
|
||||
logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name)
|
||||
logger.info('Updated %s %s attributes ...', len(to_save), attribute_name)
|
||||
logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name)
|
||||
save = not bool(user.pk)
|
||||
|
||||
if len(sys.argv) < 3 or sys.argv[2] != 'nofake':
|
||||
raise ValueError('fake')
|
||||
defaults = {
|
||||
'ou': senior_ou,
|
||||
'first_name': row['Prenom'],
|
||||
'last_name': row['Nom'],
|
||||
'email': row['email'] or '',
|
||||
'email_verified': bool(row['Email']),
|
||||
}
|
||||
for key, value in defaults.items():
|
||||
if getattr(user, key) != value:
|
||||
setattr(user, key, value)
|
||||
save = True
|
||||
|
||||
if save:
|
||||
if user.pk:
|
||||
to_save.append(user)
|
||||
else:
|
||||
to_bulk_create.append(user)
|
||||
|
||||
User.objects.bulk_create(to_bulk_create)
|
||||
for user in to_save:
|
||||
user.save()
|
||||
|
||||
user_by_guid = {user.uuid: user for user in User.objects.filter(uuid__in=guids.keys())}
|
||||
|
||||
user_ids = set(user.pk for user in user_by_guid.values())
|
||||
|
||||
external_id_by_user = {uei.user_id: uei for uei in UserExternalId.objects.filter(user_id__in=user_ids)}
|
||||
|
||||
logger.info('Created %s users...', len(to_bulk_create))
|
||||
logger.info('Updated %s users...', len(to_save))
|
||||
|
||||
logger.info('Creating UserExternalId...')
|
||||
to_bulk_create = []
|
||||
to_save = []
|
||||
for user in user_by_guid.values():
|
||||
ppid = guids[user.uuid]['ppid']
|
||||
uei = external_id_by_user.get(user.pk, UserExternalId(user=user, source='eudonet'))
|
||||
save = not bool(uei.pk)
|
||||
if uei.source != 'eudonet':
|
||||
uei.source = 'eudonet'
|
||||
save = True
|
||||
if uei.external_id != ppid:
|
||||
uei.external_id = ppid
|
||||
save = True
|
||||
|
||||
if save:
|
||||
if uei.pk:
|
||||
to_save.append(uei)
|
||||
else:
|
||||
to_bulk_create.append(uei)
|
||||
|
||||
UserExternalId.objects.bulk_create(to_bulk_create)
|
||||
for uei in to_save:
|
||||
uei.save()
|
||||
logger.info('Created %s user external ids...', len(to_bulk_create))
|
||||
logger.info('Updated %s user external ids...', len(to_save))
|
||||
|
||||
for eudonet_key, attribute_name in user_attributes_mapping.items():
|
||||
attribute = attributes[attribute_name]
|
||||
serialize = attribute.get_kind()['serialize']
|
||||
|
||||
to_bulk_create = []
|
||||
to_save = []
|
||||
to_delete = []
|
||||
|
||||
logger.info('Creating attributes %s...', attribute_name)
|
||||
atvs = {
|
||||
atv.object_id: atv for atv in AttributeValue.objects.filter(object_id__in=user_ids, attribute=attribute)
|
||||
}
|
||||
for row in data:
|
||||
user = user_by_guid[row['guid']]
|
||||
value = row[eudonet_key]
|
||||
atv = atvs.get(user.pk, AttributeValue(
|
||||
content_type=user_ct,
|
||||
object_id=user.pk,
|
||||
attribute=attribute,
|
||||
verified=False,
|
||||
multiple=False))
|
||||
if not value:
|
||||
if atv.pk:
|
||||
to_delete.append(atv.pk)
|
||||
else:
|
||||
serialized = serialize(value)
|
||||
if not atv.pk:
|
||||
atv.content = serialized
|
||||
to_bulk_create.append(atv)
|
||||
elif atv.content != serialized:
|
||||
atv.content = serialized
|
||||
to_save.append(atv)
|
||||
|
||||
AttributeValue.objects.bulk_create(to_bulk_create)
|
||||
for atv in to_save:
|
||||
atv.save()
|
||||
AttributeValue.objects.filter(pk__in=to_delete).delete()
|
||||
|
||||
logger.info('Created %s %s attributes ...', len(to_bulk_create), attribute_name)
|
||||
logger.info('Updated %s %s attributes ...', len(to_save), attribute_name)
|
||||
logger.info('Deleted %s %s attributes ...', len(to_delete), attribute_name)
|
||||
|
||||
if len(sys.argv) < 3 or sys.argv[2] != 'nofake':
|
||||
raise ValueError('fake')
|
||||
finally:
|
||||
provisionning.clear()
|
||||
|
|
|
@ -18,7 +18,7 @@ logger.addHandler(handler)
|
|||
# Adresse_Principale
|
||||
# Annee_Inscription
|
||||
# Annotations_particulières2
|
||||
# Archivé
|
||||
# Contact_Archivé
|
||||
# Autorise_Photos
|
||||
# Batiment_Residence
|
||||
# CP
|
||||
|
@ -68,7 +68,7 @@ expected_keys = set([
|
|||
'Lien_relationnel',
|
||||
'Annotations_particulières2',
|
||||
'Profil_Contact',
|
||||
'Archivé',
|
||||
'Contact_Archivé',
|
||||
'Téléphone_fixe',
|
||||
'NO_Voie',
|
||||
'Batiment_Residence',
|
||||
|
@ -89,11 +89,21 @@ def telephone(row, key):
|
|||
mobile = row[key]
|
||||
if mobile:
|
||||
mobile = mobile.strip()
|
||||
if mobile == 'NULL':
|
||||
if re.match(r'PAS.*(TEL|FIXE)', mobile) or re.match(r'n[ée]ant', mobile.strip().lower()):
|
||||
mobile = ''
|
||||
elif mobile == 'NULL':
|
||||
mobile = ''
|
||||
else:
|
||||
mobile = re.sub(r'[\s.-]', '', mobile).strip()
|
||||
if not mobile.isascii() or not mobile.isdigit() or len(mobile) not in (9, 10):
|
||||
def check_mobile():
|
||||
if not mobile.isascii():
|
||||
return False
|
||||
if not mobile.isdigit():
|
||||
return False
|
||||
if not mobile.startswith('00') and len(mobile) not in (9, 10):
|
||||
return False
|
||||
return True
|
||||
if not check_mobile():
|
||||
logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid {key} {row[key]}')
|
||||
mobile = ''
|
||||
if len(mobile) == 9:
|
||||
|
@ -123,7 +133,7 @@ def normalize(row, ppid_index):
|
|||
assert row['ppid'] not in ppids, 'duplicate ppid line %s' % line
|
||||
ppids.add(row['ppid'])
|
||||
assert row['Civilite'] in (None, 'Monsieur', 'Madame')
|
||||
|
||||
|
||||
# email
|
||||
email = row['Email']
|
||||
if email:
|
||||
|
@ -134,18 +144,23 @@ def normalize(row, ppid_index):
|
|||
|
||||
# CP
|
||||
cp = row['CP']
|
||||
if cp == '0600':
|
||||
cp = '06000'
|
||||
if re.match(r'6\s\d\d\d', cp):
|
||||
cp = re.sub(r'\s', '', cp)
|
||||
if re.match(r'6\d\d\d', cp):
|
||||
cp = '0' + cp
|
||||
if cp != row['CP']:
|
||||
row['CP'] = cp
|
||||
if cp is None:
|
||||
pass
|
||||
else:
|
||||
if cp == '0600':
|
||||
cp = '06000'
|
||||
if re.match(r'6\s\d\d\d', cp):
|
||||
cp = re.sub(r'\s', '', cp)
|
||||
if re.match(r'6\d\d\d', cp):
|
||||
cp = '0' + cp
|
||||
if len(cp) != 5:
|
||||
logger.warning(f'line {row["line"]} ppid {row["ppid"]} : invalid CP {row["CP"]}')
|
||||
if cp != row['CP']:
|
||||
row['CP'] = cp
|
||||
|
||||
# Archivé
|
||||
if row['Archivé'] == 'OUI':
|
||||
logger.warning(f'line {line} ppid {row["ppid"]} : Archivé==OUI')
|
||||
# Contact_Archivé
|
||||
if row['Contact_Archivé'] == 'OUI':
|
||||
logger.warning(f'line {line} ppid {row["ppid"]} : Contact_Archivé==OUI')
|
||||
row['import'] = False
|
||||
|
||||
# UUID
|
||||
|
@ -218,7 +233,7 @@ def load(filename, number_of_rows=None):
|
|||
ppid_index = {row['ppid'].strip(): i for i, row in enumerate(data)}
|
||||
|
||||
for i, row in enumerate(data):
|
||||
assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {row.keys()} != {keys}'
|
||||
assert set(row.keys()) == expected_keys, f'row {i+1} keys differ: {set(row.keys()) - expected_keys} and {expected_keys - set(row.keys())}'
|
||||
|
||||
error = False
|
||||
for i, row in enumerate(data):
|
||||
|
@ -230,4 +245,6 @@ def load(filename, number_of_rows=None):
|
|||
error = True
|
||||
assert not error
|
||||
|
||||
print('archivé:', len([d for d in data if d['Contact_Archivé'] == 'OUI']))
|
||||
|
||||
return reader._fieldnames, [row for row in data if row['import']]
|
||||
|
|
Loading…
Reference in New Issue