scripts: add import_rs2.py
This commit is contained in:
parent
26a5dbc02c
commit
1c2fd868a3
|
@ -0,0 +1,391 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import csv
|
||||
|
||||
from datetime import datetime, date
|
||||
|
||||
import calebasse.settings
|
||||
import django.core.management
|
||||
|
||||
django.core.management.setup_environ(calebasse.settings)
|
||||
|
||||
from django.db import transaction
|
||||
from calebasse.agenda.models import EventWithAct
|
||||
from calebasse.dossiers.models import PatientRecord
|
||||
from calebasse.personnes.models import Worker
|
||||
from calebasse.ressources.models import Service
|
||||
from calebasse.ressources.models import ActType
|
||||
from calebasse.actes.models import Act, ActValidationState
|
||||
|
||||
from import_dossiers import map_cs
|
||||
|
||||
# Configuration
|
||||
db_path = "./scripts/20130104-213225"
|
||||
|
||||
dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
|
||||
# dbs = ["F_ST_ETIENNE_SESSAD_TED"]
|
||||
|
||||
def _to_datetime(str_date):
|
||||
if not str_date:
|
||||
return None
|
||||
return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")
|
||||
|
||||
def _to_date(str_date):
|
||||
dt = _to_datetime(str_date)
|
||||
return dt and dt.date()
|
||||
|
||||
def _to_time(str_date):
|
||||
dt = _to_datetime(str_date)
|
||||
return dt and dt.time()
|
||||
|
||||
def _to_duration(str_date):
|
||||
dt = _to_datetime(str_date)
|
||||
if dt is None:
|
||||
return None
|
||||
return dt - datetime(1900, 01, 01, 0, 0)
|
||||
|
||||
def _to_int(str_int):
|
||||
if not str_int:
|
||||
return None
|
||||
return int(str_int)
|
||||
|
||||
def _get_dict(cols, line):
|
||||
""""""
|
||||
res = {}
|
||||
for i, data in enumerate(line):
|
||||
res[cols[i]] = data.decode('utf-8')
|
||||
return res
|
||||
|
||||
def batch_delete(qs, limit):
|
||||
count = qs.count()
|
||||
i = 0
|
||||
while i < count:
|
||||
ids = qs[i:i+limit].values_list('pk', flat=True)
|
||||
qs.filter(pk__in=ids).delete()
|
||||
i += limit
|
||||
|
||||
|
||||
PERIOD_FAURE_NOUS = {1 : 1,
|
||||
2 : 2,
|
||||
3 : 3,
|
||||
4 : 4,
|
||||
5: 6,
|
||||
6: 7,
|
||||
7: 8,
|
||||
8: 9,
|
||||
9: None,
|
||||
10: 10,
|
||||
12: 11,
|
||||
13: 12,
|
||||
}
|
||||
|
||||
JOURS = {1: 'lundi',
|
||||
2: 'mardi',
|
||||
3: 'mercredi',
|
||||
4: 'jeudi',
|
||||
5: 'vendredi'
|
||||
}
|
||||
|
||||
dic_worker = {}
|
||||
|
||||
def load_csv(db, name, offset=0, limit=9999999, id_column=0, year=None):
|
||||
records = []
|
||||
idx = {}
|
||||
|
||||
csvfile = open(os.path.join(db_path, db, name + '.csv'), 'rb')
|
||||
csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
|
||||
cols = csvlines.next()
|
||||
i = 0
|
||||
for line in csvlines:
|
||||
if not (offset <= int(line[id_column]) < offset+limit):
|
||||
continue
|
||||
data = _get_dict(cols, line)
|
||||
if year and not (('date_rdv' in data and data['date_rdv'].startswith(str(year))) \
|
||||
or ('date_debut' in data and data['date_debut'].startswith(str(year)))):
|
||||
continue
|
||||
records.append(data)
|
||||
idx[data['id']] = i
|
||||
i += 1
|
||||
csvfile.close()
|
||||
return records, idx, cols
|
||||
|
||||
def add_invalid(d, reason):
|
||||
d.setdefault('invalid', '')
|
||||
if d['invalid']:
|
||||
d['invalid'] += ', '
|
||||
d['invalid'] += reason
|
||||
|
||||
@transaction.commit_on_success
|
||||
def main():
|
||||
""" """
|
||||
|
||||
workers = Worker.objects.all()
|
||||
for db in dbs:
|
||||
workers_idx = {}
|
||||
act_types_idx = {}
|
||||
if "F_ST_ETIENNE_CMPP" == db:
|
||||
service = Service.objects.get(name="CMPP")
|
||||
elif "F_ST_ETIENNE_CAMSP" == db:
|
||||
service = Service.objects.get(name="CAMSP")
|
||||
elif "F_ST_ETIENNE_SESSAD_TED" == db:
|
||||
service = Service.objects.get(name="SESSAD TED")
|
||||
elif "F_ST_ETIENNE_SESSAD" == db:
|
||||
service = Service.objects.get(name="SESSAD DYS")
|
||||
|
||||
print '===', service.name, '==='
|
||||
print datetime.now()
|
||||
|
||||
for i, worker in enumerate(workers):
|
||||
if service.name == 'CMPP':
|
||||
j = worker.old_cmpp_id
|
||||
elif service.name == 'CAMSP':
|
||||
j = worker.old_camsp_id
|
||||
elif service.name == 'SESSAD DYS':
|
||||
j = worker.old_sessad_dys_id
|
||||
elif service.name == 'SESSAD TED':
|
||||
j = worker.old_sessad_ted_id
|
||||
else:
|
||||
print "service inconnu!!!"
|
||||
exit(0)
|
||||
if j:
|
||||
workers_idx[j] = worker
|
||||
|
||||
act_types = ActType.objects.for_service(service)
|
||||
act_type_id_not_found = set()
|
||||
for i, act_type in enumerate(act_types):
|
||||
j = act_type.old_id
|
||||
if j:
|
||||
act_types_idx[j] = act_type
|
||||
else:
|
||||
act_type_id_not_found.add(act_type)
|
||||
print "%s - Nombre de types d'actes : %d" % (service.name, len(act_types))
|
||||
print "%s - Liste des types d'actes sans id : %s" % (service.name, str(act_type_id_not_found))
|
||||
|
||||
invalid_rs = set()
|
||||
invalid_rr = set()
|
||||
new_rs = []
|
||||
new_rr = []
|
||||
invalid_rs_csv = open('./scripts/invalid_rs_%s.csv' % (service.name, ), 'wb+')
|
||||
invalid_rs_writer = csv.writer(invalid_rs_csv, delimiter=',',
|
||||
quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||
invalid_rr_csv = open('./scripts/invalid_rr_%s.csv' % (service.name, ), 'wb+')
|
||||
invalid_rr_writer = csv.writer(invalid_rr_csv, delimiter=',',
|
||||
quotechar='"', quoting=csv.QUOTE_MINIMAL)
|
||||
for year in range(2011, 2014):
|
||||
print service.name, ' - Year', year
|
||||
rs_data, rs_idx, rs_cols = load_csv(db, 'rs', year=year)
|
||||
rr_data, rr_idx, rr_cols = load_csv(db, 'rr', year=year)
|
||||
print service.name, year, 'read', len(rs_data), 'rs and', len(rr_data), 'rr'
|
||||
rs_cols.extend(['workers', 'invalid'])
|
||||
rr_cols.extend(['workers', 'invalid'])
|
||||
invalid_rs_writer.writerow(map(lambda x: x.encode('utf-8'), rs_cols))
|
||||
invalid_rr_writer.writerow(map(lambda x: x.encode('utf-8'), rr_cols))
|
||||
|
||||
single_rdv_idx = dict(EventWithAct.objects.filter(services=service,
|
||||
start_datetime__year=year,
|
||||
recurrence_periodicity__isnull=True).values_list('old_rs_id', 'id'))
|
||||
multi_rdv_idx = dict(EventWithAct.objects.filter(services=service,
|
||||
start_datetime__year=year,
|
||||
recurrence_periodicity__isnull=False).values_list('old_rr_id', 'id'))
|
||||
|
||||
rs_details_data, _, _ = load_csv(db, 'details_rs')
|
||||
rr_details_data, _, _ = load_csv(db, 'details_rr')
|
||||
|
||||
def set_act_type(row, not_found=None):
|
||||
act_type_id = row['type_acte']
|
||||
if act_type_id == '0':
|
||||
add_invalid(row, 'no act_id=>not importable')
|
||||
elif act_type_id in act_types_idx:
|
||||
row['act_type'] = act_types_idx[act_type_id]
|
||||
else:
|
||||
add_invalid(row, 'act_type not found %s' % act_type_id)
|
||||
if not_found:
|
||||
not_found.add(act_type_id)
|
||||
|
||||
def handle_details(data, idx, details, id_key):
|
||||
not_found = set()
|
||||
for detail in details:
|
||||
i = detail[id_key]
|
||||
thera_id = detail['thera_id']
|
||||
if i not in idx:
|
||||
continue
|
||||
row = data[idx[i]]
|
||||
if thera_id in workers_idx:
|
||||
workers = row.setdefault('workers', set())
|
||||
workers.add(workers_idx[thera_id])
|
||||
else:
|
||||
# add_invalid(row, 'unknown thera_id %s' % thera_id)
|
||||
not_found.add(thera_id)
|
||||
print "%s - Liste des worker not found : %s" % (service.name, str(set(not_found)))
|
||||
|
||||
print ' - Traitement rs_detail....'
|
||||
handle_details(rs_data, rs_idx, rs_details_data, 'rs_id')
|
||||
print ' - Traitement rr_detail....'
|
||||
handle_details(rr_data, rr_idx, rr_details_data, 'rr_id')
|
||||
|
||||
enfant_idx = {}
|
||||
for enfant in PatientRecord.objects.filter(service=service):
|
||||
enfant_idx[enfant.old_id] = enfant
|
||||
|
||||
def set_enfant(row, not_found=None):
|
||||
# connect enfant
|
||||
enfant_id = row['enfant_id']
|
||||
if enfant_id == '0':
|
||||
add_invalid(row, 'no enfant_id=>not an appointment')
|
||||
row['event'] = True
|
||||
elif enfant_id in enfant_idx:
|
||||
row['enfant'] = enfant_idx[enfant_id]
|
||||
else:
|
||||
add_invalid(row, 'enfant_id not found %s' % enfant_id)
|
||||
if not_found:
|
||||
not_found.add(enfant_id)
|
||||
|
||||
enfant_not_found = set()
|
||||
rr_not_found = set()
|
||||
rs_without_act_type = set()
|
||||
unknown_act_type_id = set()
|
||||
for rs in rs_data:
|
||||
if rs['id'] in single_rdv_idx:
|
||||
continue
|
||||
rs.setdefault('workers', set())
|
||||
rs['exception'] = False
|
||||
rs['event'] = False
|
||||
rs['date'] = _to_date(rs['date_rdv'])
|
||||
# connect enfant
|
||||
set_enfant(rs)
|
||||
set_act_type(rs)
|
||||
if rs['event']:
|
||||
continue
|
||||
# connect rr
|
||||
rr_id = rs['rr_ev_id']
|
||||
if rr_id:
|
||||
if rr_id.startswith('ev_'):
|
||||
continue
|
||||
rs['time'] = _to_time(rs['heure'])
|
||||
rs['duration'] = _to_duration(rs['duree'])
|
||||
rs['start_datetime'] = datetime.combine(rs['date'], rs['time'])
|
||||
rs['end_datetime'] = rs['start_datetime'] + rs['duration']
|
||||
if len(rs['workers']) == 0:
|
||||
add_invalid(rs, 'aucun participant')
|
||||
if rs.get('invalid') and not rs['event']:
|
||||
invalid_rs.add(rs['id'])
|
||||
invalid_rs_writer.writerow([ unicode(rs[col]).encode('utf-8') for col in rs_cols ])
|
||||
else:
|
||||
new_rs.append(rs)
|
||||
|
||||
|
||||
print "%s - Liste des enfants not found : %s" % (service.name, str(enfant_not_found))
|
||||
print "%s - Liste des RR not found : %s" % (service.name, str(rr_not_found))
|
||||
print "%s - Liste des RS sans type d'acte : %s" % (service.name, str(rs_without_act_type))
|
||||
print "%s - Liste des types d'actes inconnus : %s" % (service.name, str(unknown_act_type_id))
|
||||
print
|
||||
|
||||
enfant_not_found = set()
|
||||
rr_not_found = set()
|
||||
rs_without_act_type = set()
|
||||
unknown_act_type_id = set()
|
||||
for rr in rr_data:
|
||||
if rr['id'] in multi_rdv_idx:
|
||||
continue
|
||||
rr['event'] = False
|
||||
rs.setdefault('workers', set())
|
||||
rr.setdefault('exceptions', [])
|
||||
rr['start_date'] = _to_date(rr['date_debut'])
|
||||
rr['end_date'] = _to_date(rr['date_fin'])
|
||||
if rr['end_date'] and rr['start_date'] > rr['end_date']:
|
||||
add_invalid(rr, 'date_fin < date_debut')
|
||||
rr['time'] = _to_time(rr['heure'])
|
||||
rr['duration'] = _to_duration(rr['duree'])
|
||||
rr['start_datetime'] = datetime.combine(rr['start_date'], rr['time'])
|
||||
rr['end_datetime'] = rr['start_datetime'] + rr['duration']
|
||||
# connect rythme
|
||||
rr['rythme'] = int(rr['rythme'])
|
||||
if PERIOD_FAURE_NOUS.get(rr['rythme']):
|
||||
rr['recurrence_periodicity'] = PERIOD_FAURE_NOUS[rr['rythme']]
|
||||
else:
|
||||
add_invalid(rr, 'rythme not found %s' % rr['rythme'])
|
||||
set_enfant(rr)
|
||||
set_act_type(rr)
|
||||
if rr['event']:
|
||||
continue
|
||||
if len(rr['workers']) == 0:
|
||||
add_invalid(rr, 'aucun participant')
|
||||
if rr.get('invalid'):
|
||||
invalid_rr.add(rr['id'])
|
||||
invalid_rr_writer.writerow([ unicode(rr[col]).encode('utf-8') for col in rr_cols ])
|
||||
else:
|
||||
new_rr.append(rr)
|
||||
print service.name, 'found', len(new_rs), 'individual rdv not imported'
|
||||
print service.name, 'found', len(invalid_rs), 'individual rdv invalid'
|
||||
print service.name, 'found', len(new_rr), 'multiple rdv not imported'
|
||||
print service.name, 'found', len(invalid_rr), 'multiple rdv invalid'
|
||||
|
||||
for row in new_rs:
|
||||
act_id = row['base_id']
|
||||
if act_id == '0':
|
||||
continue
|
||||
acts = Act.objects.filter(old_id=act_id)
|
||||
row['act'] = None
|
||||
if len(acts) == 0:
|
||||
print service.name, 'act not imported', act_id, 'linked to rs', row['id']
|
||||
elif len(acts) > 1:
|
||||
print service.name, 'more than one act', acts.values_list('id', flat=True), 'linked to rs', row['id']
|
||||
else:
|
||||
row['act'] = acts[0]
|
||||
|
||||
|
||||
# RR
|
||||
for row in new_rr:
|
||||
event = EventWithAct.objects.create(
|
||||
patient=row['enfant'],
|
||||
start_datetime=row['start_datetime'],
|
||||
end_datetime=row['end_datetime'],
|
||||
act_type=row['act_type'],
|
||||
old_rr_id=row['id'],
|
||||
title=row['enfant'].display_name,
|
||||
description=row['texte'],
|
||||
recurrence_periodicity=row['recurrence_periodicity'],
|
||||
recurrence_end_date=row['end_date'])
|
||||
event.services = [ service ]
|
||||
event.participants = row['workers']
|
||||
print 'created', event
|
||||
for row in new_rs:
|
||||
if row['exception'] or row.get('invalid'):
|
||||
continue
|
||||
exception_to = None
|
||||
exception_date = None
|
||||
if row['rr_ev_id']:
|
||||
_, rr_id = row['rr_ev_id'].split('_')
|
||||
e = EventWithAct.objects.filter(old_rr_id=rr_id)
|
||||
if len(e) == 0:
|
||||
pass
|
||||
elif len(e) > 1:
|
||||
add_invalid(row, 'more than one recurring event %s' % e.values_list('id', flat=True))
|
||||
invalid_rs_writer.writerow([ unicode(row[col]).encode('utf-8') for col in rs_cols ])
|
||||
continue
|
||||
else:
|
||||
exception_to = e[0]
|
||||
exception_date = row['start_datetime'].date()
|
||||
if exception_to and EventWithAct.objects.filter(exception_to=exception_to, exception_date=exception_date):
|
||||
print 'exception existante', EventWithAct.objects.filter(exception_to=exception_to, exception_date=exception_date), ' ligne ', row, 'ignore'
|
||||
continue
|
||||
|
||||
|
||||
event = EventWithAct.objects.create(patient=row['enfant'],
|
||||
start_datetime=row['start_datetime'],
|
||||
end_datetime=row['end_datetime'],
|
||||
act_type=row['act_type'],
|
||||
old_rs_id=row['id'],
|
||||
title=row['enfant'].display_name,
|
||||
description=row['texte'],
|
||||
exception_to=exception_to,
|
||||
exception_date=exception_date)
|
||||
event.services = [ service ]
|
||||
event.participants = row['workers']
|
||||
print 'created', event
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Reference in New Issue