From 010abd53b80012a780d46e1a632203d55e55cf8e Mon Sep 17 00:00:00 2001 From: Emmanuel Cazenave Date: Tue, 15 Jun 2021 16:19:28 +0200 Subject: [PATCH] venissieux-technocarte: split wcs and a2 operations in two scripts --- venissieux-technocarte/a2_run.py | 68 +++++++++ venissieux-technocarte/{run.py => wcs_run.py} | 132 +++--------------- 2 files changed, 91 insertions(+), 109 deletions(-) create mode 100644 venissieux-technocarte/a2_run.py rename venissieux-technocarte/{run.py => wcs_run.py} (71%) diff --git a/venissieux-technocarte/a2_run.py b/venissieux-technocarte/a2_run.py new file mode 100644 index 0000000..c466a20 --- /dev/null +++ b/venissieux-technocarte/a2_run.py @@ -0,0 +1,68 @@ +""" +Usage : +authentic2-multitenant-manage tenant_command runscript venissieux-technocarte/a2_run.py reset -d authentic.dev.publik.love + +authentic2-multitenant-manage tenant_command runscript venissieux-technocarte/a2_run.py import --filepath ~/notes/clients/venissieux/famille/technocarte.csv -d authentic.dev.publik.love +""" + +import argparse +import collections +import csv + +from django.contrib.auth import get_user_model + + +User = get_user_model() + + +def get_rows(args): + with open(args.filepath) as csvfile: + reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') + for i, row in enumerate(reader): + if args.mode != 'full' and i > args.sample_numlines: + break + yield row + + +def build_email(first_name, last_name): + return 'vpf-%s-%s@yopmail.com' % (first_name, last_name) + + +def import_data(args): + familles_adulte_count = collections.defaultdict(int) + for row in get_rows(args): + if row['ADULTE'] != '1': + continue + + code_famille = row['CODFAM'] + if familles_adulte_count[code_famille] >= 2: + # gestion des doublons + continue + + data = { + 'last_name': row['NOMENF'], + 'first_name': row['PREENF'], + 'email': build_email(row['PREENF'], row['NOMENF']) + } + + User.objects.create(**data) + + +def reset(args): + User.objects.filter(email__startswith='vpf').filter(email__endswith='yopmail.com').delete() + + +parser = argparse.ArgumentParser() +subparsers = parser.add_subparsers() + +parser_import = subparsers.add_parser('import') +parser_import.set_defaults(func=import_data) +parser_import.add_argument('--filepath') +parser_import.add_argument('--mode', default='sample', choices=('sample', 'full')) +parser_import.add_argument('--sample-numlines', default=100, type=int) + +parser_reset = subparsers.add_parser('reset') +parser_reset.set_defaults(func=reset) + +args = parser.parse_args() +args.func(args) diff --git a/venissieux-technocarte/run.py b/venissieux-technocarte/wcs_run.py similarity index 71% rename from venissieux-technocarte/run.py rename to venissieux-technocarte/wcs_run.py index b7e6384..94247bd 100644 --- a/venissieux-technocarte/run.py +++ b/venissieux-technocarte/wcs_run.py @@ -1,7 +1,8 @@ """ Usage : -wcs-manage runscript -d wcs.dev.publik.love run.py wcs-reset -wcs-manage runscript -d wcs.dev.publik.love run.py wcs-import --filepath ~/notes/clients/venissieux/famille/technocarte.csv --api-url=https://wcs.dev.publik.love/api/ --email=admin@localhost --orig=wcs.dev.publik.love --key=c25fc1f82bd56b01e7cf62d785ae4410b0f7fdbebfb92a986d3fd6a150f3ba0e +wcs-manage runscript -d wcs.dev.publik.love wcs_run.py reset + +wcs-manage runscript -d wcs.dev.publik.love wcs_run.py import --filepath ~/notes/clients/venissieux/famille/technocarte.csv --api-url=https://wcs.dev.publik.love/api/ --email=admin@localhost --orig=wcs.dev.publik.love --key=c25fc1f82bd56b01e7cf62d785ae4410b0f7fdbebfb92a986d3fd6a150f3ba0e """ import argparse @@ -17,7 +18,6 @@ import time import urllib.parse import requests -from requests.auth import HTTPBasicAuth from wcs.carddef import CardDef CARD_ADULTE_SLUG = 'adultes' @@ -59,7 +59,7 @@ def sign_string(s, key, algo='sha256', timedelta=30): return hash.digest() -def wcs_api_call(url, args, data=None, user_email=None, method='post'): +def api_call(url, args, data=None, user_email=None, method='post'): url = sign_url(url + '?email=%s' % args.email, args.key, orig=args.orig) if method == 'post': post_data = {'data': data} @@ -72,26 +72,6 @@ def wcs_api_call(url, args, data=None, user_email=None, method='post'): return resp -def a2_api_call(url, args, data=None, method='post'): - if method == 'post': - resp = requests.post(url, json=data, auth=HTTPBasicAuth(args.api_user, args.api_pass)) - elif method == 'get': - resp = requests.get(url, auth=HTTPBasicAuth(args.api_user, args.api_pass)) - elif method == 'delete': - resp = requests.delete(url, auth=HTTPBasicAuth(args.api_user, args.api_pass)) - resp.raise_for_status() - return resp - - -def get_field_value(carddata, varname): - field = None - for field in carddata.formdef.get_all_fields(): - if field.varname == varname: - break - assert field is not None - return carddata.data.get(field.id) - - def get_rows(args): with open(args.filepath) as csvfile: reader = csv.DictReader(csvfile, delimiter=';', quotechar='"') @@ -163,7 +143,7 @@ def create_adultes(args): if civilite_clean: data['civilite'] = civilite_clean - resp = wcs_api_call(url, args, data=data, user_email=email) + resp = api_call(url, args, data=data, user_email=email) adulte = { 'technocarte_id': row['CODENF'], @@ -208,7 +188,7 @@ def create_enfants(args, familles_adultes): user_email = adulte['email'] break - resp = wcs_api_call(url, args, data, user_email=user_email) + resp = api_call(url, args, data, user_email=user_email) enfant = { 'technocarte_id': row['CODENF'], @@ -249,13 +229,13 @@ def create_familles(args, adultes, familles_enfants): data['numero_allocataire_caf'] = numero_allocataire_caf url = args.api_url + 'cards/%s/submit' % CARD_FAMILLE_SLUG - resp = wcs_api_call(url, args, data, user_email=adulte['email']) + resp = api_call(url, args, data, user_email=adulte['email']) familles_map[technocarte_famille_id] = str(resp.json()['data']['id']) else: url = args.api_url + 'cards/%s/%s/' % ( CARD_FAMILLE_SLUG, familles_map[technocarte_famille_id] ) - card = wcs_api_call(url, args, method='get').json() + card = api_call(url, args, method='get').json() if card['fields']['adulte2']: # gestion des doublons, s'il ya déjà un deuxième adulte, on zappe continue @@ -267,7 +247,7 @@ def create_familles(args, adultes, familles_enfants): # les gens ne se marient plus data['famille'] = "%s %s" % (card['fields']['famille'], adulte['nom']) - wcs_api_call(url, args, data) + api_call(url, args, data) return familles_map @@ -291,107 +271,41 @@ def create_gardes(args, enfants, familles, familles_adultes): user_email = adulte['email'] break - resp = wcs_api_call(url, args, data, user_email=user_email) + resp = api_call(url, args, data, user_email=user_email) gardes[publik_id] = str(resp.json()['data']['id']) return gardes -def wcs_import_data(args): +def import_data(args): adultes, familles_adultes = create_adultes(args) enfants, familles_enfants = create_enfants(args, familles_adultes) familles = create_familles(args, adultes, familles_enfants) create_gardes(args, enfants, familles, familles_adultes) -def a2_import_data(args): - familles_adulte_count = collections.defaultdict(int) - url = args.api_url + 'users/' - for row in get_rows(args): - if row['ADULTE'] != '1': - continue - - code_famille = row['CODFAM'] - if familles_adulte_count[code_famille] >= 2: - # gestion des doublons - continue - - data = { - 'last_name': row['NOMENF'], - 'first_name': row['PREENF'], - 'email': build_email(row['PREENF'], row['NOMENF']), - 'send_registration_email': False - } - codsex = row['CODSEX'] - if codsex == 'M': - data['gender'] = 1 - elif codsex == 'F': - data['gender'] = 2 - - a2_api_call(url, args, data) - - -def a2_reset(args): - to_delete = [] - url = args.api_url + 'users/' - - next_ = True - while next_: - data = a2_api_call(url, args, method='get').json() - next_ = data['next'] - for user in data['results']: - email, uuid = user['email'], user['uuid'] - if email.startswith('vpf-') and email.endswith('yopmail.com'): - to_delete.append(uuid) - - for uuid in to_delete: - url = args.api_url + 'users/%s/' % uuid - a2_api_call(url, args, method='delete') - - -def wcs_reset(args): +def reset(args): CardDef.get_by_urlname(CARD_ENFANT_SLUG).data_class().wipe() CardDef.get_by_urlname(CARD_ADULTE_SLUG).data_class().wipe() CardDef.get_by_urlname(CARD_FAMILLE_SLUG).data_class().wipe() CardDef.get_by_urlname(CARD_GARDE_SLUG).data_class().wipe() -def add_import_parser_args(parser): - parser.add_argument('--filepath') - parser.add_argument('--mode', default='sample', choices=('sample', 'full')) - parser.add_argument('--sample-numlines', default=100, type=int) - return parser - - -def add_a2_parser_args(parser): - parser.add_argument('--api-url') - parser.add_argument('--api-user') - parser.add_argument('--api-pass') - return parser - - parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() -wcs_parser_import = subparsers.add_parser('wcs-import') -wcs_parser_import.set_defaults(func=wcs_import_data) -wcs_parser_import = add_import_parser_args(wcs_parser_import) -wcs_parser_import.add_argument('--api-url') -wcs_parser_import.add_argument('--email') -wcs_parser_import.add_argument('--orig') -wcs_parser_import.add_argument('--key') +parser_import = subparsers.add_parser('import') +parser_import.set_defaults(func=import_data) +parser_import.add_argument('--filepath') +parser_import.add_argument('--mode', default='sample', choices=('sample', 'full')) +parser_import.add_argument('--sample-numlines', default=100, type=int) +parser_import.add_argument('--api-url') +parser_import.add_argument('--email') +parser_import.add_argument('--orig') +parser_import.add_argument('--key') -wcs_parser_reset = subparsers.add_parser('wcs-reset') -wcs_parser_reset.set_defaults(func=wcs_reset) - -a2_parser_import = subparsers.add_parser('a2-import') -a2_parser_import.set_defaults(func=a2_import_data) -a2_parser_import = add_import_parser_args(a2_parser_import) -a2_parser_import = add_a2_parser_args(a2_parser_import) - -a2_parser_reset = subparsers.add_parser('a2-reset') -a2_parser_reset.set_defaults(func=a2_reset) -a2_parser_reset = add_a2_parser_args(a2_parser_reset) +parser_reset = subparsers.add_parser('reset') +parser_reset.set_defaults(func=reset) args = parser.parse_args() args.func(args)