From cdb90482c3c38d81f03ca21aa2c9ca37bc57be0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 25 Mar 2019 18:17:06 +0100
Subject: [PATCH] manager: allow CSV file to be iso-8859-15 (#25984)

---
 chrono/manager/forms.py | 20 +++++++++++++++++---
 tests/test_manager.py   | 22 ++++++++++++++++++++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/chrono/manager/forms.py b/chrono/manager/forms.py
index 2ac53118..24e080d0 100644
--- a/chrono/manager/forms.py
+++ b/chrono/manager/forms.py
@@ -169,15 +169,29 @@ class ImportEventsForm(forms.Form):
         if b'\0' in content:
             raise ValidationError(_('Invalid file format.'))
 
-        if six.PY3:
-            content = content.decode('utf-8')
+        for charset in ('utf-8', 'iso-8859-15'):
+            try:
+                content = content.decode(charset)
+                break
+            except UnicodeDecodeError:
+                continue
+        # all byte-sequences are ok for iso-8859-15 so we will always reach
+        # this line with content being a unicode string.
+
         try:
             dialect = csv.Sniffer().sniff(content)
         except csv.Error:
             dialect = None
 
+        if six.PY3:
+            utf_8_encoder = lambda x: x
+        else:
+            def utf_8_encoder(unicode_csv_data):
+                for line in unicode_csv_data:
+                    yield line.encode('utf-8')
+
         events = []
-        for i, csvline in enumerate(csv.reader(content.splitlines(), dialect=dialect)):
+        for i, csvline in enumerate(csv.reader(utf_8_encoder(content.splitlines()), dialect=dialect)):
             if not csvline:
                 continue
             if len(csvline) < 3:
diff --git a/tests/test_manager.py b/tests/test_manager.py
index fd43a100..41031532 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -581,6 +581,28 @@ def test_import_events(app, admin_user):
     assert Event.objects.all()[0].label == u'éléphant'
     Event.objects.all().delete()
 
+    resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
+    resp.form['events_csv_file'] = Upload('t.csv',
+            u'2016-09-16,18:00,10,5,éléphant'.encode('iso-8859-15'), 'text/csv')
+    resp = resp.form.submit(status=302)
+    assert Event.objects.count() == 1
+    assert Event.objects.all()[0].start_datetime == make_aware(datetime.datetime(2016, 9, 16, 18, 0))
+    assert Event.objects.all()[0].places == 10
+    assert Event.objects.all()[0].waiting_list_places == 5
+    assert Event.objects.all()[0].label == u'éléphant'
+    Event.objects.all().delete()
+
+    resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
+    resp.form['events_csv_file'] = Upload('t.csv',
+            u'2016-09-16,18:00,10,5,éléphant'.encode('eucjp'), 'text/csv')
+    resp = resp.form.submit(status=302)
+    assert Event.objects.count() == 1
+    assert Event.objects.all()[0].start_datetime == make_aware(datetime.datetime(2016, 9, 16, 18, 0))
+    assert Event.objects.all()[0].places == 10
+    assert Event.objects.all()[0].waiting_list_places == 5
+    assert Event.objects.all()[0].label == u'\x8f«±l\x8f«±phant'  # eucjp interpreted as iso-8859-15
+    Event.objects.all().delete()
+
     resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
     resp.form['events_csv_file'] = Upload('t.csv', b'date,time,etc.\n'
                                                    b'2016-09-16,18:00,10,5,bla bla bla\n'