Import CSV, rapporter toutes les erreurs d'un coup (#70523) #130

Merged
vdeniaud merged 2 commits from wip/70523-Import-CSV-rapporter-toutes-les- into main 2023-08-16 09:46:26 +02:00
2 changed files with 158 additions and 128 deletions

View File

@ -1206,14 +1206,7 @@ class ImportEventsForm(forms.Form):
super().__init__(**kwargs)
def clean_events_csv_file(self):
class ValidationErrorWithOrdinal(ValidationError):
line_offset = 1
def __init__(self, message, event_no):
super().__init__(message)
self.message = format_html(message, event_no=mark_safe(ordinal(event_no + self.line_offset)))
exclude_from_validation = ['desk', 'meeting_type', 'primary_event']
self.exclude_from_validation = ['desk', 'meeting_type', 'primary_event']
content = self.cleaned_data['events_csv_file'].read()
if b'\0' in content:
@ -1234,49 +1227,118 @@ class ImportEventsForm(forms.Form):
except csv.Error:
dialect = None
events = []
warnings = {}
events_by_slug = {x.slug: x for x in Event.objects.filter(agenda=self.agenda.pk)}
event_ids_with_bookings = set(
errors = []
self.events = []
self.warnings = {}
self.events_by_slug = {x.slug: x for x in Event.objects.filter(agenda=self.agenda.pk)}
self.event_ids_with_bookings = set(
Booking.objects.filter(
event__agenda=self.agenda.pk, cancellation_datetime__isnull=True
).values_list('event_id', flat=True)
)
seen_slugs = set(events_by_slug.keys())
self.seen_slugs = set(self.events_by_slug.keys())
line_offset = 1
for i, csvline in enumerate(csv.reader(StringIO(content), dialect=dialect)):
if not csvline:
continue
if len(csvline) < 3:
raise ValidationErrorWithOrdinal(_('Invalid file format. ({event_no} event)'), i)
if i == 0 and csvline[0].strip('#') in ('date', 'Date', _('date'), _('Date')):
ValidationErrorWithOrdinal.line_offset = 0
line_offset = 0
continue
# label needed to generate a slug
label = None
if len(csvline) >= 5:
label = force_str(csvline[4])
try:
event = self.parse_csvline(csvline)
except ValidationError as e:
for error in getattr(e, 'error_list', [e]):
errors.append(
format_html(
'{message} ({event_no} event)',
message=error.message,
event_no=mark_safe(ordinal(i + line_offset)),
)
)
else:
self.events.append(event)
# get or create event
event = None
slug = None
if len(csvline) >= 6:
slug = force_str(csvline[5]) if csvline[5] else None
# get existing event if relevant
if slug and slug in seen_slugs:
event = events_by_slug[slug]
# update label
event.label = label
if event is None:
# new event
event = Event(agenda_id=self.agenda.pk, label=label)
# generate a slug if not provided
event.slug = slug or generate_slug(event, seen_slugs=seen_slugs, agenda=self.agenda.pk)
# maintain caches
seen_slugs.add(event.slug)
events_by_slug[event.slug] = event
if errors:
errors = [_('Invalid file format:')] + errors
raise ValidationError(errors)
def parse_csvline(self, csvline):
if len(csvline) < 3:
raise ValidationError(_('Not enough columns.'))
# label needed to generate a slug
label = None
if len(csvline) >= 5:
label = force_str(csvline[4])
# get or create event
event = None
slug = None
if len(csvline) >= 6:
slug = force_str(csvline[5]) if csvline[5] else None
# get existing event if relevant
if slug and slug in self.seen_slugs:
event = self.events_by_slug[slug]
# update label
event.label = label
if event is None:
# new event
event = Event(agenda_id=self.agenda.pk, label=label)
# generate a slug if not provided
event.slug = slug or generate_slug(event, seen_slugs=self.seen_slugs, agenda=self.agenda.pk)
# maintain caches
self.seen_slugs.add(event.slug)
self.events_by_slug[event.slug] = event
for datetime_fmt in (
'%Y-%m-%d %H:%M',
'%d/%m/%Y %H:%M',
'%d/%m/%Y %Hh%M',
'%Y-%m-%d %H:%M:%S',
'%d/%m/%Y %H:%M:%S',
):
try:
event_datetime = make_aware(
datetime.datetime.strptime('%s %s' % tuple(csvline[:2]), datetime_fmt)
)
except ValueError:
continue
if (
event.pk is not None
and event.start_datetime != event_datetime
and event.start_datetime > now()
and event.pk in self.event_ids_with_bookings
and event.pk not in self.warnings
):
# event start datetime has changed, event is not past and has not cancelled bookings
# => warn the user
self.warnings[event.pk] = event
event.start_datetime = event_datetime
break
else:
raise ValidationError(_('Wrong start date/time format.'))
try:
event.places = int(csvline[2])
except ValueError:
raise ValidationError(_('Number of places must be an integer.'))
if len(csvline) >= 4:
try:
event.waiting_list_places = int(csvline[3])
except ValueError:
raise ValidationError(_('Number of places in waiting list must be an integer.'))
column_index = 7
for more_attr in ('description', 'pricing', 'url'):
if len(csvline) >= column_index:
setattr(event, more_attr, csvline[column_index - 1])
column_index += 1
if len(csvline) >= 10 and csvline[9]: # publication date is optional
for datetime_fmt in (
'%Y-%m-%d',
'%d/%m/%Y',
'%Y-%m-%d %H:%M',
'%d/%m/%Y %H:%M',
'%d/%m/%Y %Hh%M',
@ -1284,93 +1346,39 @@ class ImportEventsForm(forms.Form):
'%d/%m/%Y %H:%M:%S',
):
try:
event_datetime = make_aware(
datetime.datetime.strptime('%s %s' % tuple(csvline[:2]), datetime_fmt)
event.publication_datetime = make_aware(
datetime.datetime.strptime(csvline[9], datetime_fmt)
)
break
except ValueError:
continue
if (
event.pk is not None
and event.start_datetime != event_datetime
and event.start_datetime > now()
and event.pk in event_ids_with_bookings
and event.pk not in warnings
):
# event start datetime has changed, event is not past and has not cancelled bookings
# => warn the user
warnings[event.pk] = event
event.start_datetime = event_datetime
break
else:
raise ValidationErrorWithOrdinal(
_('Invalid file format. (date/time format, {event_no} event)'), i
)
try:
event.places = int(csvline[2])
except ValueError:
raise ValidationError(_('Invalid file format. (number of places, {event_no} event)'), i)
if len(csvline) >= 4:
raise ValidationError(_('Wrong publication date/time format.'))
if self.agenda.partial_bookings:
if len(csvline) < 11 or not csvline[10]:
raise ValidationError(_('Missing end_time.'))
event.end_time = csvline[10]
else:
self.exclude_from_validation.append('end_time')
if len(csvline) >= 11 and csvline[10]: # duration is optional
try:
event.waiting_list_places = int(csvline[3])
event.duration = int(csvline[10])
except ValueError:
raise ValidationError(
_('Invalid file format. (number of places in waiting list, {event_no} event)'), i
)
raise ValidationError(_('Duration must be an integer.'))
column_index = 7
for more_attr in ('description', 'pricing', 'url'):
if len(csvline) >= column_index:
setattr(event, more_attr, csvline[column_index - 1])
column_index += 1
try:
event.full_clean(exclude=self.exclude_from_validation)
except ValidationError as e:
errors = []
for label, field_errors in e.message_dict.items():
label_name = self.get_verbose_name(label)
msg = _('%s: ') % label_name if label_name else ''
msg += ', '.join(field_errors)
errors.append(msg)
raise ValidationError(errors)
if len(csvline) >= 10 and csvline[9]: # publication date is optional
for datetime_fmt in (
'%Y-%m-%d',
'%d/%m/%Y',
'%Y-%m-%d %H:%M',
'%d/%m/%Y %H:%M',
'%d/%m/%Y %Hh%M',
'%Y-%m-%d %H:%M:%S',
'%d/%m/%Y %H:%M:%S',
):
try:
event.publication_datetime = make_aware(
datetime.datetime.strptime(csvline[9], datetime_fmt)
)
break
except ValueError:
continue
else:
raise ValidationError(_('Invalid file format. (date/time format, {event_no} event)'), i)
if self.agenda.partial_bookings:
if len(csvline) < 11 or not csvline[10]:
raise ValidationError(_('Invalid file format. (missing end_time, {event_no} event)'), i)
event.end_time = csvline[10]
else:
exclude_from_validation.append('end_time')
if len(csvline) >= 11 and csvline[10]: # duration is optional
try:
event.duration = int(csvline[10])
except ValueError:
raise ValidationError(_('Invalid file format. (duration, {event_no} event)'), i)
try:
event.full_clean(exclude=exclude_from_validation)
except ValidationError as e:
errors = [_('Invalid file format:\n')]
for label, field_errors in e.message_dict.items():
label_name = self.get_verbose_name(label)
msg = _('%s: ') % label_name if label_name else ''
msg += _('%(errors)s (line %(line)d)') % {
'errors': ', '.join(field_errors),
'line': i + 1,
}
errors.append(msg)
raise ValidationError(errors)
events.append(event)
self.events = events
self.warnings = warnings
return event
@staticmethod
def get_verbose_name(field_name):

View File

@ -798,7 +798,8 @@ def test_import_events(app, admin_user):
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
resp.form['events_csv_file'] = Upload('t.csv', b'xx', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format.' in resp.text
assert 'Invalid file format:' in resp.text
assert 'Wrong start date/time format. (1st event)' in resp.text
resp.form['events_csv_file'] = Upload('t.csv', b'xxxx\0\0xxxx', 'text/csv')
resp = resp.form.submit(status=200)
@ -806,15 +807,16 @@ def test_import_events(app, admin_user):
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format.' in resp.text
assert 'Invalid file format:' in resp.text
assert 'Not enough columns. (1st event)' in resp.text
resp.form['events_csv_file'] = Upload('t.csv', b'date,time,etc.\n2016-14-16,18:00,10', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format. (date/time format, 1st event)' in resp.text
assert 'Wrong start date/time format. (1st event)' in resp.text
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00,10', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format. (date/time format, 1st event)' in resp.text
assert 'Wrong start date/time format. (1st event)' in resp.text
with override_settings(LANGUAGE_CODE='fr-fr'):
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00,10', 'text/csv')
@ -824,11 +826,11 @@ def test_import_events(app, admin_user):
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,blah', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format. (number of places,' in resp.text
assert 'Number of places must be an integer. (1st event)' in resp.text
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,blah', 'text/csv')
resp = resp.form.submit(status=200)
assert 'Invalid file format. (number of places in waiting list,' in resp.text
assert 'Number of places in waiting list must be an integer. (1st event)' in resp.text
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,5,' + b'x' * 151, 'text/csv')
resp = resp.form.submit(status=200)
@ -988,7 +990,7 @@ def test_import_events(app, admin_user):
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,foobar', 'text/csv'
)
resp = resp.form.submit(status=200)
assert 'Invalid file format. (date/time format' in resp.text
assert 'Wrong publication date/time format. (1st event)' in resp.text
# duration bad format
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
@ -996,7 +998,7 @@ def test_import_events(app, admin_user):
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,2016-09-16,foobar', 'text/csv'
)
resp = resp.form.submit(status=200)
assert 'Invalid file format. (duration' in resp.text
assert 'Duration must be an integer. (1st event)' in resp.text
# import events with empty slugs
Event.objects.all().delete()
@ -1027,8 +1029,7 @@ def test_import_events(app, admin_user):
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,5,label,1234', 'text/csv')
resp = resp.form.submit(status=200)
assert 'value cannot be a number' in resp.text
assert 'Identifier:' in resp.text # verbose_name is shown, not field name ('slug:')
assert 'Identifier: This value cannot be a number. (1st event)' in resp.text
def test_import_event_nested_quotes(app, admin_user):
@ -1192,7 +1193,7 @@ def test_import_events_partial_bookings(app, admin_user):
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,2016-09-16', 'text/csv'
)
resp = resp.form.submit(status=200)
assert 'Invalid file format. (missing end_time' in resp.text
assert 'Missing end_time.' in resp.text
# invalid end time
resp = app.get('/manage/agendas/%s/import-events' % agenda.pk)
@ -1203,6 +1204,27 @@ def test_import_events_partial_bookings(app, admin_user):
assert '“xxx” value has an invalid format' in resp.text
def test_import_events_multiple_errors(app, admin_user):
agenda = Agenda.objects.create(label='Foo bar', kind='events')
Desk.objects.create(agenda=agenda, slug='_exceptions_holder')
app = login(app)
resp = app.get('/manage/agendas/%s/import-events' % agenda.pk)
resp.form['events_csv_file'] = Upload(
't.csv',
b'2016-09-17,18:00,10,5,label,slug\n' # valid event
b'2016-09-17,19:00,xxx,5,label2,slug2\n' # invalid places
b'2016-09-17,20:00,10,5,,1234\n', # invalid slug
'text/csv',
)
resp = resp.form.submit(status=200)
assert [x.text for x in resp.pyquery('.errorlist li')] == [
'Invalid file format:',
'Number of places must be an integer. (2nd event)',
'Identifier: This value cannot be a number. (3rd event)',
]
@pytest.mark.freeze_time('2022-05-24')
def test_event_detail(app, admin_user):
agenda = Agenda.objects.create(label='Events', kind='events')