entrouvert
/
synchro-orleans
Archived
17
0
Fork 0
Code Issues Pull Requests Releases Activity

data import improved by using bulk creation

This commit is contained in:
Serghei Mihai 2014-04-16 18:13:54 +02:00
parent 2bc843c68c
commit ff31005d87
1 changed files with 19 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
synchro_orleans/data/models.py
View File

@ -49,7 +49,7 @@ class PersonneManager(models.Manager):
csv_reader = reader(source, delimiter = ';')
caption = csv_reader.next()
attrs = map(lambda e: self.attribute_mappings[e], caption)
ids = ()
objs = ()
for row in csv_reader:
params = dict(zip(attrs, row))
@ -58,21 +58,9 @@ class PersonneManager(models.Manager):
for key, value in params.iteritems():
params[key] = value.decode('utf-8')
obj_id = params['id']
ids += (obj_id, )
try:
obj = self.get(pk = obj_id)
obj_id = params.pop('id')
updated = False
for attr, value in params.iteritems():
if getattr(obj, attr) != value:
setattr(obj, attr, value)
updated = True
if updated:
obj.save()
except models.ObjectDoesNotExist:
self.create(**params)
self.exclude(id__in=ids).delete()
objs += (Personne(**params), )
self.all().delete()
self.bulk_create(objs)
class FamilleManager(models.Manager):
@ -172,7 +160,7 @@ class FactureManager(models.Manager):
csv_reader = reader(source, delimiter = ';')
caption = csv_reader.next()
attrs = map(lambda e: self.attribute_mappings[e], caption)
ids = ()
objs = ()
for row in csv_reader:
params = dict(zip(attrs, row))
@ -192,20 +180,10 @@ class FactureManager(models.Manager):
params['active'] = path.exists(settings.INVOICES_LOCATION_PATTERN.format(invoice_id = params['id']))
family = Famille.objects.get(pk = params.pop('famille'))
params['famille'] = family
objs += (Facture(**params), )
try:
obj = self.get(pk = params['id'])
ids += (params.pop('id'), )
updated = False
for attr, value in params.iteritems():
if getattr(obj, attr) != value:
setattr(obj, attr, value)
updated = True
if updated:
obj.save()
except models.ObjectDoesNotExist:
obj = self.create(**params)
self.exclude(id__in=ids).delete()
self.all().delete()
self.bulk_create(objs)
class KidManager(models.Manager):
@ -224,42 +202,35 @@ class KidManager(models.Manager):
csv_reader = reader(source, delimiter = ';')
caption = csv_reader.next()
attrs = map(lambda e: self.attribute_mappings[e], caption)
ids = ()
objs = ()
for row in csv_reader:
params = dict(zip(attrs, row))
# transform all inputs to unicode
for key, value in params.iteritems():
params[key] = value.decode('utf-8')
family_id = params.pop('famille')
try:
params['date_naissance'] = datetime.strptime(params['date_naissance'],
INPUT_DATE_FORMAT)
except ValueError:
params.pop('date_naissance')
obj_id = params['id']
ids += (obj_id, )
family_id = params.pop('famille')
try:
family = Famille.objects.get(pk = family_id)
except Famille.DoesNotExist:
print "Famille %s for kid %s doesn't exist(%s)" % (family_id, obj_id, row)
print "Famille %s for kid %s doesn't exist(%s)" % (family_id, params['id'], row)
continue
params['famille'] = family
try:
obj = self.get(pk = obj_id)
obj_id = params.pop('id')
updated = False
for attr, value in params.iteritems():
if getattr(obj, attr) != value:
setattr(obj, attr, value)
updated = True
if updated:
obj.save()
except models.ObjectDoesNotExist:
obj = self.create(**params)
self.exclude(id__in=ids).delete()
objs += (Enfant(**params), )
self.all().delete()
self.bulk_create(objs)
class Personne(models.Model):
id = models.AutoField(primary_key = True)