227 lines
8.2 KiB
Python
227 lines
8.2 KiB
Python
# Passerelle - uniform access to data and services
|
|
# Copyright (C) 2017 Entr'ouvert
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU Affero General Public License as published
|
|
# by the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import os
|
|
import csv
|
|
from decimal import Decimal
|
|
|
|
from django.utils.timezone import make_aware, datetime
|
|
from django.utils import six, timezone
|
|
from django.utils.encoding import force_text
|
|
from django.core.exceptions import ValidationError
|
|
from django.utils.translation import ugettext_lazy as _
|
|
from django.core.files.storage import DefaultStorage
|
|
|
|
from ..models import Family, Adult, Child, Invoice
|
|
from ..models import dict_cherry_pick, get_datetime, get_date
|
|
|
|
|
|
def normalize_adult(adult):
|
|
sex = adult['typ_sexe']
|
|
if sex == 'G':
|
|
sex = 'M'
|
|
return {
|
|
'external_id': adult['id_per'],
|
|
'first_name': adult['lib_prenom_personne'],
|
|
'last_name': adult['lib_nom_personne'],
|
|
'sex': sex,
|
|
'street_number': adult['num_voie_adr'] + adult['lib_nom_btq'],
|
|
'street_name': adult['lib_nom_rue'],
|
|
'address_complement': adult['lib_comp_adr'],
|
|
'zipcode': adult['cod_postal'],
|
|
'city': adult['lib_commune'],
|
|
'country': adult['lib_pays'],
|
|
'phone': adult['num_teldom_per'],
|
|
'cellphone': adult['num_telport_per'],
|
|
}
|
|
|
|
|
|
def normalize_family(family, adults):
|
|
return {
|
|
'external_id': family['id_fam'],
|
|
'adults': [
|
|
adults[family[id]] for id in ('id_per1', 'id_per2') if family[id] and adults.get(family[id])
|
|
],
|
|
'children': [],
|
|
'invoices': [],
|
|
'login': family['id_fam'],
|
|
'password': family['cod_secret_fam'],
|
|
'family_quotient': family['qf_vo'],
|
|
'street_number': family['num_voie_adr'] + family['lib_nom_btq'],
|
|
'street_name': family['lib_nom_rue'],
|
|
'address_complement': family['lib_comp_adr'],
|
|
'zipcode': family['cod_postal_adr'],
|
|
'city': family['lib_commune_adr'],
|
|
}
|
|
|
|
|
|
def normalize_child(child):
|
|
sex = child['typ_sexe_per']
|
|
if sex == 'G':
|
|
sex = 'M'
|
|
return {
|
|
'external_id': child['id_per'],
|
|
'first_name': child['lib_prenom_per'],
|
|
'last_name': child['lib_nom_per'],
|
|
'sex': sex,
|
|
'birthdate': get_date(child['dat_naissance']),
|
|
}
|
|
|
|
|
|
def normalize_invoice(i):
|
|
invoice = {
|
|
'external_id': i['id_fac'],
|
|
'label': i['id_fac'],
|
|
'total_amount': Decimal(i['mnt_facture_fac']),
|
|
'amount': Decimal(i['mnt_solde_fac']),
|
|
'issue_date': i['dat_generation_fac'],
|
|
'pay_limit_date': get_date(i['dat_limitepaie_fac']),
|
|
'autobilling': i['on_prelevauto_ins'] == 'O',
|
|
'online_payment': True,
|
|
'payment_date': get_datetime(i['dat_reglement']),
|
|
'litigation_date': get_date(i['dat_perception_fac']),
|
|
'paid': Decimal(i['mnt_solde_fac']) == 0,
|
|
}
|
|
return invoice
|
|
|
|
|
|
class Dialect(csv.Dialect):
|
|
'''Because sometimes it cannot be sniffed by csv.Sniffer'''
|
|
|
|
delimiter = ';'
|
|
doublequote = False
|
|
escapechar = None
|
|
lineterminator = '\n'
|
|
quoting = csv.QUOTE_NONE
|
|
skipinitialspace = False
|
|
|
|
|
|
class Loader(object):
|
|
def __init__(self, connector):
|
|
self.connector = connector
|
|
|
|
def clean(self, archive):
|
|
for filename in (
|
|
'extract_prcit_personne.csv',
|
|
'extract_prcit_famille.csv',
|
|
'extract_prcit_enfant.csv',
|
|
'extract_prcit_facture.csv',
|
|
):
|
|
if not filename in archive.namelist():
|
|
raise ValidationError(_('Missing %(filename)s file in zip.') % {'filename': filename})
|
|
|
|
def csvread(self, filename):
|
|
fd = self.archive.open(filename)
|
|
if six.PY3:
|
|
import io
|
|
|
|
fd = io.TextIOWrapper(fd, 'iso-8859-15')
|
|
|
|
reader = csv.reader(fd, Dialect)
|
|
titles = [x.lower() for x in next(reader)]
|
|
for row in reader:
|
|
row = [force_text(x, 'iso-8859-15') for x in row]
|
|
yield dict(zip(titles, row))
|
|
|
|
def build_families(self):
|
|
families = {}
|
|
adults = {}
|
|
for adult in self.csvread('extract_prcit_personne.csv'):
|
|
adult = normalize_adult(adult)
|
|
adults[adult['external_id']] = adult
|
|
|
|
for family in self.csvread('extract_prcit_famille.csv'):
|
|
family = normalize_family(family, adults)
|
|
families[family['external_id']] = family
|
|
|
|
for child in self.csvread('extract_prcit_enfant.csv'):
|
|
families[child['id_fam']]['children'].append(normalize_child(child))
|
|
|
|
for invoice in self.csvread('extract_prcit_facture.csv'):
|
|
families[invoice['id_fam']]['invoices'].append(normalize_invoice(invoice))
|
|
return families
|
|
|
|
def load(self, archive):
|
|
self.archive = archive
|
|
|
|
try:
|
|
families = self.build_families()
|
|
except Exception as e:
|
|
self.connector.logger.error('Error occured while building families: %s', e)
|
|
return
|
|
|
|
import_start_timestamp = timezone.now()
|
|
try:
|
|
for family_data in families.values():
|
|
data = dict_cherry_pick(
|
|
family_data,
|
|
(
|
|
'login',
|
|
'password',
|
|
'family_quotient',
|
|
'zipcode',
|
|
'street_number',
|
|
'street_name',
|
|
'address_complement',
|
|
'city',
|
|
),
|
|
)
|
|
family, created = Family.objects.update_or_create(
|
|
external_id=family_data['external_id'], resource=self.connector, defaults=data
|
|
)
|
|
|
|
for adult_data in family_data.get('adults') or []:
|
|
Adult.objects.update_or_create(
|
|
family=family, external_id=adult_data['external_id'], defaults=adult_data
|
|
)
|
|
|
|
for child_data in family_data.get('children') or []:
|
|
Child.objects.get_or_create(
|
|
family=family, external_id=child_data['external_id'], defaults=child_data
|
|
)
|
|
|
|
for invoice_data in family_data.get('invoices') or []:
|
|
storage = DefaultStorage()
|
|
invoices_dir = storage.path('family-%s/invoices' % self.connector.id)
|
|
invoice_filename = '%s.pdf' % invoice_data['external_id']
|
|
invoice_path = os.path.join(invoices_dir, invoice_filename)
|
|
# create invoice object only if associated pdf exists
|
|
if os.path.exists(invoice_path):
|
|
invoice, created = Invoice.objects.update_or_create(
|
|
resource=self.connector,
|
|
family=family,
|
|
external_id=invoice_data['external_id'],
|
|
defaults=invoice_data,
|
|
)
|
|
|
|
except Exception as e:
|
|
self.connector.logger.error('Error occured while importing data: %s', e)
|
|
|
|
Family.objects.filter(resource=self.connector, update_timestamp__lte=import_start_timestamp).delete()
|
|
Adult.objects.filter(
|
|
family__resource=self.connector, update_timestamp__lte=import_start_timestamp
|
|
).delete()
|
|
Child.objects.filter(
|
|
family__resource=self.connector, update_timestamp__lte=import_start_timestamp
|
|
).delete()
|
|
# remove obsolete invoices and their pdfs
|
|
for invoice in Invoice.objects.filter(
|
|
resource=self.connector, update_timestamp__lte=import_start_timestamp
|
|
):
|
|
if invoice.has_pdf:
|
|
os.unlink(invoice.pdf_filename())
|
|
invoice.delete()
|