venissieux-technocarte: split wcs and a2 operations in two scripts

This commit is contained in:
Emmanuel Cazenave 2021-06-15 16:19:28 +02:00
parent 40610a86b6
commit 010abd53b8
2 changed files with 91 additions and 109 deletions

View File

@ -0,0 +1,68 @@
"""
Usage :
authentic2-multitenant-manage tenant_command runscript venissieux-technocarte/a2_run.py reset -d authentic.dev.publik.love
authentic2-multitenant-manage tenant_command runscript venissieux-technocarte/a2_run.py import --filepath ~/notes/clients/venissieux/famille/technocarte.csv -d authentic.dev.publik.love
"""
import argparse
import collections
import csv
from django.contrib.auth import get_user_model
User = get_user_model()
def get_rows(args):
with open(args.filepath) as csvfile:
reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
for i, row in enumerate(reader):
if args.mode != 'full' and i > args.sample_numlines:
break
yield row
def build_email(first_name, last_name):
return 'vpf-%s-%s@yopmail.com' % (first_name, last_name)
def import_data(args):
familles_adulte_count = collections.defaultdict(int)
for row in get_rows(args):
if row['ADULTE'] != '1':
continue
code_famille = row['CODFAM']
if familles_adulte_count[code_famille] >= 2:
# gestion des doublons
continue
data = {
'last_name': row['NOMENF'],
'first_name': row['PREENF'],
'email': build_email(row['PREENF'], row['NOMENF'])
}
User.objects.create(**data)
def reset(args):
User.objects.filter(email__startswith='vpf').filter(email__endswith='yopmail.com').delete()
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
parser_import = subparsers.add_parser('import')
parser_import.set_defaults(func=import_data)
parser_import.add_argument('--filepath')
parser_import.add_argument('--mode', default='sample', choices=('sample', 'full'))
parser_import.add_argument('--sample-numlines', default=100, type=int)
parser_reset = subparsers.add_parser('reset')
parser_reset.set_defaults(func=reset)
args = parser.parse_args()
args.func(args)

View File

@ -1,7 +1,8 @@
"""
Usage :
wcs-manage runscript -d wcs.dev.publik.love run.py wcs-reset
wcs-manage runscript -d wcs.dev.publik.love run.py wcs-import --filepath ~/notes/clients/venissieux/famille/technocarte.csv --api-url=https://wcs.dev.publik.love/api/ --email=admin@localhost --orig=wcs.dev.publik.love --key=c25fc1f82bd56b01e7cf62d785ae4410b0f7fdbebfb92a986d3fd6a150f3ba0e
wcs-manage runscript -d wcs.dev.publik.love wcs_run.py reset
wcs-manage runscript -d wcs.dev.publik.love wcs_run.py import --filepath ~/notes/clients/venissieux/famille/technocarte.csv --api-url=https://wcs.dev.publik.love/api/ --email=admin@localhost --orig=wcs.dev.publik.love --key=c25fc1f82bd56b01e7cf62d785ae4410b0f7fdbebfb92a986d3fd6a150f3ba0e
"""
import argparse
@ -17,7 +18,6 @@ import time
import urllib.parse
import requests
from requests.auth import HTTPBasicAuth
from wcs.carddef import CardDef
CARD_ADULTE_SLUG = 'adultes'
@ -59,7 +59,7 @@ def sign_string(s, key, algo='sha256', timedelta=30):
return hash.digest()
def wcs_api_call(url, args, data=None, user_email=None, method='post'):
def api_call(url, args, data=None, user_email=None, method='post'):
url = sign_url(url + '?email=%s' % args.email, args.key, orig=args.orig)
if method == 'post':
post_data = {'data': data}
@ -72,26 +72,6 @@ def wcs_api_call(url, args, data=None, user_email=None, method='post'):
return resp
def a2_api_call(url, args, data=None, method='post'):
if method == 'post':
resp = requests.post(url, json=data, auth=HTTPBasicAuth(args.api_user, args.api_pass))
elif method == 'get':
resp = requests.get(url, auth=HTTPBasicAuth(args.api_user, args.api_pass))
elif method == 'delete':
resp = requests.delete(url, auth=HTTPBasicAuth(args.api_user, args.api_pass))
resp.raise_for_status()
return resp
def get_field_value(carddata, varname):
field = None
for field in carddata.formdef.get_all_fields():
if field.varname == varname:
break
assert field is not None
return carddata.data.get(field.id)
def get_rows(args):
with open(args.filepath) as csvfile:
reader = csv.DictReader(csvfile, delimiter=';', quotechar='"')
@ -163,7 +143,7 @@ def create_adultes(args):
if civilite_clean:
data['civilite'] = civilite_clean
resp = wcs_api_call(url, args, data=data, user_email=email)
resp = api_call(url, args, data=data, user_email=email)
adulte = {
'technocarte_id': row['CODENF'],
@ -208,7 +188,7 @@ def create_enfants(args, familles_adultes):
user_email = adulte['email']
break
resp = wcs_api_call(url, args, data, user_email=user_email)
resp = api_call(url, args, data, user_email=user_email)
enfant = {
'technocarte_id': row['CODENF'],
@ -249,13 +229,13 @@ def create_familles(args, adultes, familles_enfants):
data['numero_allocataire_caf'] = numero_allocataire_caf
url = args.api_url + 'cards/%s/submit' % CARD_FAMILLE_SLUG
resp = wcs_api_call(url, args, data, user_email=adulte['email'])
resp = api_call(url, args, data, user_email=adulte['email'])
familles_map[technocarte_famille_id] = str(resp.json()['data']['id'])
else:
url = args.api_url + 'cards/%s/%s/' % (
CARD_FAMILLE_SLUG, familles_map[technocarte_famille_id]
)
card = wcs_api_call(url, args, method='get').json()
card = api_call(url, args, method='get').json()
if card['fields']['adulte2']:
# gestion des doublons, s'il ya déjà un deuxième adulte, on zappe
continue
@ -267,7 +247,7 @@ def create_familles(args, adultes, familles_enfants):
# les gens ne se marient plus
data['famille'] = "%s %s" % (card['fields']['famille'], adulte['nom'])
wcs_api_call(url, args, data)
api_call(url, args, data)
return familles_map
@ -291,107 +271,41 @@ def create_gardes(args, enfants, familles, familles_adultes):
user_email = adulte['email']
break
resp = wcs_api_call(url, args, data, user_email=user_email)
resp = api_call(url, args, data, user_email=user_email)
gardes[publik_id] = str(resp.json()['data']['id'])
return gardes
def wcs_import_data(args):
def import_data(args):
adultes, familles_adultes = create_adultes(args)
enfants, familles_enfants = create_enfants(args, familles_adultes)
familles = create_familles(args, adultes, familles_enfants)
create_gardes(args, enfants, familles, familles_adultes)
def a2_import_data(args):
familles_adulte_count = collections.defaultdict(int)
url = args.api_url + 'users/'
for row in get_rows(args):
if row['ADULTE'] != '1':
continue
code_famille = row['CODFAM']
if familles_adulte_count[code_famille] >= 2:
# gestion des doublons
continue
data = {
'last_name': row['NOMENF'],
'first_name': row['PREENF'],
'email': build_email(row['PREENF'], row['NOMENF']),
'send_registration_email': False
}
codsex = row['CODSEX']
if codsex == 'M':
data['gender'] = 1
elif codsex == 'F':
data['gender'] = 2
a2_api_call(url, args, data)
def a2_reset(args):
to_delete = []
url = args.api_url + 'users/'
next_ = True
while next_:
data = a2_api_call(url, args, method='get').json()
next_ = data['next']
for user in data['results']:
email, uuid = user['email'], user['uuid']
if email.startswith('vpf-') and email.endswith('yopmail.com'):
to_delete.append(uuid)
for uuid in to_delete:
url = args.api_url + 'users/%s/' % uuid
a2_api_call(url, args, method='delete')
def wcs_reset(args):
def reset(args):
CardDef.get_by_urlname(CARD_ENFANT_SLUG).data_class().wipe()
CardDef.get_by_urlname(CARD_ADULTE_SLUG).data_class().wipe()
CardDef.get_by_urlname(CARD_FAMILLE_SLUG).data_class().wipe()
CardDef.get_by_urlname(CARD_GARDE_SLUG).data_class().wipe()
def add_import_parser_args(parser):
parser.add_argument('--filepath')
parser.add_argument('--mode', default='sample', choices=('sample', 'full'))
parser.add_argument('--sample-numlines', default=100, type=int)
return parser
def add_a2_parser_args(parser):
parser.add_argument('--api-url')
parser.add_argument('--api-user')
parser.add_argument('--api-pass')
return parser
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
wcs_parser_import = subparsers.add_parser('wcs-import')
wcs_parser_import.set_defaults(func=wcs_import_data)
wcs_parser_import = add_import_parser_args(wcs_parser_import)
wcs_parser_import.add_argument('--api-url')
wcs_parser_import.add_argument('--email')
wcs_parser_import.add_argument('--orig')
wcs_parser_import.add_argument('--key')
parser_import = subparsers.add_parser('import')
parser_import.set_defaults(func=import_data)
parser_import.add_argument('--filepath')
parser_import.add_argument('--mode', default='sample', choices=('sample', 'full'))
parser_import.add_argument('--sample-numlines', default=100, type=int)
parser_import.add_argument('--api-url')
parser_import.add_argument('--email')
parser_import.add_argument('--orig')
parser_import.add_argument('--key')
wcs_parser_reset = subparsers.add_parser('wcs-reset')
wcs_parser_reset.set_defaults(func=wcs_reset)
a2_parser_import = subparsers.add_parser('a2-import')
a2_parser_import.set_defaults(func=a2_import_data)
a2_parser_import = add_import_parser_args(a2_parser_import)
a2_parser_import = add_a2_parser_args(a2_parser_import)
a2_parser_reset = subparsers.add_parser('a2-reset')
a2_parser_reset.set_defaults(func=a2_reset)
a2_parser_reset = add_a2_parser_args(a2_parser_reset)
parser_reset = subparsers.add_parser('reset')
parser_reset.set_defaults(func=reset)
args = parser.parse_args()
args.func(args)