From 96f6b658016e02713e8bc4c314f844d48f5b3329 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Thu, 17 Aug 2017 18:02:40 +0200 Subject: [PATCH] fix data extraction mechanism --- concerto_2_chrono_format.py | 62 ++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/concerto_2_chrono_format.py b/concerto_2_chrono_format.py index afb4552..3c677c2 100755 --- a/concerto_2_chrono_format.py +++ b/concerto_2_chrono_format.py @@ -17,13 +17,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os import re import csv import sys import json -import zipfile -import datetime import logging +import zipfile +import tempfile +import datetime from logging.config import dictConfig from itertools import groupby @@ -108,13 +110,11 @@ def json_write_data(json_data, output): def extract_zip_to_tmp(filename): with zipfile.ZipFile(filename, 'r') as zipf: - zipf.extractall('/tmp/') + zipf.extractall(tempfile.gettempdir()) class ConcertoFormatter(object): name = 'concerto' - output_file = '/tmp/concerto.json' - expected_file = '/tmp/extract_prcit_planning_creche.csv' fieldnames = [ 'ID_LIE', 'DAT_JOUR_TMP', 'HEU_DEBUT_TMP', 'HEU_FIN_TMP', 'NB_PLACESPOSS_TMP', 'NB_PLACESOCC_TMP', 'NB_PLACESRESFUT_TMP', @@ -124,29 +124,41 @@ class ConcertoFormatter(object): def __init__(self, filename, **kwargs): self.filename = filename + @property + def output_file(self): + return os.path.join(tempfile.gettempdir(), 'concerto.json') + + @property + def expected_file(self): + return os.path.join(tempfile.gettempdir(), 'extract_prcit_planning_creche.csv') + def format(self): - agendas = [] + agendas = {} extract_zip_to_tmp(self.filename) data = csv_get_dict_data(self.expected_file, fieldnames=self.fieldnames) - for nursery, events in groupby(data, lambda x: x['LIB_NOM_LIE']): - nursery_slug = slugify(nursery) - agenda = {'slug': nursery_slug, 'label': unicode(nursery, errors='replace'), - 'events': [], 'kind': 'events', 'minimal_booking_delay': 0, 'maximal_booking_delay': 30} - agenda['permissions'] = {'edit': None, 'view': None} - for event in events: - event_day = datetime.datetime.strptime(event['DAT_JOUR_TMP'], '%Y%m%d') - event_time = datetime.datetime.strptime(event['HEU_DEBUT_TMP'], '%H%M') - event_datetime = datetime.datetime.combine(event_day, event_time.time()) - # XXX: value such as '-' OR '-76,24' - try: - event_places = int(event['NB_PLACESLIBRES_TMP'].split(',')[0]) - except ValueError: - event_places = 0 - if event_places < 0: - event_places = 0 - agenda['events'].append({'start_datetime': event_datetime.strftime(DATETIME_FORMAT), 'places': event_places}) - agendas.append(agenda) - return json_write_data({'agendas': agendas}, self.output_file) + for event in data: + agenda_id = event['ID_LIE'] + if not agenda_id in agendas: + nursery_slug = agenda_id + nursery = event['LIB_NOM_LIE'] + agenda = {'slug': nursery_slug, 'label': unicode(nursery, errors='replace'), + 'events': [], 'kind': 'events', 'minimal_booking_delay': 0, 'maximal_booking_delay': 30} + agenda['permissions'] = {'edit': None, 'view': None} + else: + agenda = agendas[agenda_id] + event_day = datetime.datetime.strptime(event['DAT_JOUR_TMP'], '%Y%m%d') + event_time = datetime.datetime.strptime(event['HEU_DEBUT_TMP'], '%H%M') + event_datetime = datetime.datetime.combine(event_day, event_time.time()) + # XXX: value such as '-' OR '-76,24' + try: + event_places = int(event['NB_PLACESLIBRES_TMP'].split(',')[0]) + except ValueError: + event_places = 0 + if event_places < 0: + event_places = 0 + agenda['events'].append({'start_datetime': event_datetime.strftime(DATETIME_FORMAT), 'places': event_places}) + agendas[agenda_id] = agenda + return json_write_data({'agendas': agendas.values()}, self.output_file) if __name__ == "__main__":