csv_import: ignore BOM (#43627)

src/authentic2/manager/forms.py

@@ -664,7 +664,7 @@ class SiteImportForm(forms.Form):
 
 
 ENCODINGS = [
-    ('utf-8', _('Unicode (UTF-8)')),
+    ('utf-8-sig', _('Unicode (UTF-8)')),
     ('cp1252', _('Western Europe (Windows-1252)')),
     ('iso-8859-15', _('Western Europe (ISO-8859-15)')),
 ]

tests/test_csv_import.py

@@ -20,6 +20,7 @@ from __future__ import unicode_literals
 import pytest
 
 import io
+import codecs
 
 from django.core import mail
 
@@ -134,6 +135,14 @@ def test_too_many_key_columns_error(profile, user_csv_importer_factory):
     assert importer.errors == [Error('too-many-key-columns')]
 
 
+def test_bom_character(profile, user_csv_importer_factory):
+    content = codecs.BOM_UTF8 + 'email key,first_name\ntest@entrouvert.org,hop'.encode('utf-8')
+    file_content = io.BytesIO(content)
+    importer = UserCsvImporter()
+    assert importer.run(file_content, 'utf-8-sig')
+    assert not importer.has_errors
+
+
 def test_run(profile, user_csv_importer_factory):
     assert User.objects.count() == 0
     content = '''email key,first_name,last_name,phone update

tests/test_user_manager.py

@@ -342,7 +342,7 @@ def test_user_table(app, admin, user_ou1, ou1):
     assert response.pyquery('td.username')
 
 
-@pytest.mark.parametrize('encoding', ['utf-8', 'cp1252', 'iso-8859-15'])
+@pytest.mark.parametrize('encoding', ['utf-8-sig', 'cp1252', 'iso-8859-15'])
 def test_user_import(encoding, transactional_db, app, admin, ou1, admin_ou1, media):
     Attribute.objects.create(name='phone', kind='phone_number', label='Numéro de téléphone')
 
@@ -482,7 +482,7 @@ def import_csv(csv_content, app):
     response.forms[index].set(
         'import_file',
         Upload('users.csv', csv_content.encode('utf-8'), 'application/octet-stream'))
-    response.forms[index].set('encoding', 'utf-8')
+    response.forms[index].set('encoding', 'utf-8-sig')
     response.forms[index].set('ou', str(get_default_ou().pk))
     response = response.forms[index].submit().follow()
     response = response.forms['action-form'].submit(name='execute').follow()