#!/usr/bin/env python # -*- coding: utf-8 -*- # vi:filetype=python # chrono - agendas system # Copyright (C) 2016-2017 Entr'ouvert # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License as published # by the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import re import csv import sys import json import datetime import logging from logging.config import dictConfig from itertools import groupby LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'handlers': { 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', }, 'syslog': { 'level': 'DEBUG', 'address': '/dev/log', 'class': 'logging.handlers.SysLogHandler', 'formatter': 'syslog', }, }, 'formatters': { 'syslog': { 'format': '%(levelname)s %(name)s.%(funcName)s: %(message)s', }, }, 'loggers': { '': { 'handlers': ['console', 'syslog'], 'level': 'DEBUG', 'propagate': True, } } } dictConfig(LOGGING) logger = logging.getLogger('hobo-loaders') _slugify_strip_re = re.compile(r'[^\w\s-]') _slugify_hyphenate_re = re.compile(r'[-\s]+') DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' def slugify(value): import unicodedata if not isinstance(value, unicode): value = unicode(value, errors='ignore') value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') value = unicode(_slugify_strip_re.sub('', value).strip().lower(), errors='ignore') return _slugify_hyphenate_re.sub('-', value) def csv_get_dialect(filename): with open(filename, 'rb') as fd: content = fd.read() content = content.decode('utf-8-sig', 'replace').encode('utf-8') dialect = csv.Sniffer().sniff(content) return dialect def csv_get_dict_data(filename, fieldnames=[], skip_header=True): with open(filename, 'rb') as fd: kwargs = {'fieldnames': fieldnames} kwargs['dialect'] = csv_get_dialect(filename) reader = csv.DictReader(fd, **kwargs) if skip_header: reader.next() return list(reader) return False def json_write_data(json_data, output): """Write data into a json file """ with open(output, 'w') as f: json.dump(json_data, f, indent=4, encoding='utf-8', sort_keys=True) return True return False class ConcertoFormatter(object): name = 'concerto' output_file = 'concerto.json' fieldnames = [ 'ID_LIE', 'DAT_JOUR_TMP', 'HEU_DEBUT_TMP', 'HEU_FIN_TMP', 'NB_PLACESPOSS_TMP', 'NB_PLACESOCC_TMP', 'NB_PLACESRESFUT_TMP', 'NB_PLACESRES_TMP', 'NB_PLACESLIBRES_TMP', 'NB_DUREE_TMP', 'LIB_NOM_LIE', 'NB_PLACESRESFUTPOND_TMP'] def __init__(self, filename, **kwargs): self.filename = filename def format(self): agendas = [] data = csv_get_dict_data(self.filename, fieldnames=self.fieldnames) for nursery, events in groupby(data, lambda x: x['LIB_NOM_LIE']): nursery_slug = slugify(nursery) agenda = {'slug': nursery_slug, 'label': unicode(nursery, errors='replace'), 'events': [], 'kind': 'events'} agenda['permissions'] = {'edit': None, 'view': None} for event in events: event_day = datetime.datetime.strptime(event['DAT_JOUR_TMP'], '%Y%m%d') event_time = datetime.datetime.strptime(event['HEU_DEBUT_TMP'], '%H%M') event_datetime = datetime.datetime.combine(event_day, event_time.time()) # XXX: value such as '-' OR '-76,24' try: event_places = int(event['NB_PLACESLIBRES_TMP'].split(',')[0]) except ValueError: event_places = 0 if event_places < 0: event_places = 0 agenda['events'].append({'start_datetime': event_datetime.strftime(DATETIME_FORMAT), 'places': event_places}) agendas.append(agenda) return json_write_data({'agendas': agendas}, self.output_file) if __name__ == "__main__": if len(sys.argv) < 2: logger.error('Invalid agrument number: concerto_2_chrono_format.py ') sys.exit(1) filename = sys.argv[1] concerto = ConcertoFormatter(filename) concerto.format() sys.exit(0)