Import CSV, rapporter toutes les erreurs d'un coup (#70523) #130
|
@ -1206,14 +1206,7 @@ class ImportEventsForm(forms.Form):
|
|||
super().__init__(**kwargs)
|
||||
|
||||
def clean_events_csv_file(self):
|
||||
class ValidationErrorWithOrdinal(ValidationError):
|
||||
line_offset = 1
|
||||
|
||||
def __init__(self, message, event_no):
|
||||
super().__init__(message)
|
||||
self.message = format_html(message, event_no=mark_safe(ordinal(event_no + self.line_offset)))
|
||||
|
||||
exclude_from_validation = ['desk', 'meeting_type', 'primary_event']
|
||||
self.exclude_from_validation = ['desk', 'meeting_type', 'primary_event']
|
||||
|
||||
content = self.cleaned_data['events_csv_file'].read()
|
||||
if b'\0' in content:
|
||||
|
@ -1234,49 +1227,118 @@ class ImportEventsForm(forms.Form):
|
|||
except csv.Error:
|
||||
dialect = None
|
||||
|
||||
events = []
|
||||
warnings = {}
|
||||
events_by_slug = {x.slug: x for x in Event.objects.filter(agenda=self.agenda.pk)}
|
||||
event_ids_with_bookings = set(
|
||||
errors = []
|
||||
self.events = []
|
||||
self.warnings = {}
|
||||
self.events_by_slug = {x.slug: x for x in Event.objects.filter(agenda=self.agenda.pk)}
|
||||
self.event_ids_with_bookings = set(
|
||||
Booking.objects.filter(
|
||||
event__agenda=self.agenda.pk, cancellation_datetime__isnull=True
|
||||
).values_list('event_id', flat=True)
|
||||
)
|
||||
seen_slugs = set(events_by_slug.keys())
|
||||
self.seen_slugs = set(self.events_by_slug.keys())
|
||||
line_offset = 1
|
||||
for i, csvline in enumerate(csv.reader(StringIO(content), dialect=dialect)):
|
||||
if not csvline:
|
||||
continue
|
||||
if len(csvline) < 3:
|
||||
raise ValidationErrorWithOrdinal(_('Invalid file format. ({event_no} event)'), i)
|
||||
|
||||
if i == 0 and csvline[0].strip('#') in ('date', 'Date', _('date'), _('Date')):
|
||||
ValidationErrorWithOrdinal.line_offset = 0
|
||||
line_offset = 0
|
||||
continue
|
||||
|
||||
# label needed to generate a slug
|
||||
label = None
|
||||
if len(csvline) >= 5:
|
||||
label = force_str(csvline[4])
|
||||
try:
|
||||
event = self.parse_csvline(csvline)
|
||||
except ValidationError as e:
|
||||
for error in getattr(e, 'error_list', [e]):
|
||||
errors.append(
|
||||
format_html(
|
||||
'{message} ({event_no} event)',
|
||||
message=error.message,
|
||||
event_no=mark_safe(ordinal(i + line_offset)),
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.events.append(event)
|
||||
|
||||
# get or create event
|
||||
event = None
|
||||
slug = None
|
||||
if len(csvline) >= 6:
|
||||
slug = force_str(csvline[5]) if csvline[5] else None
|
||||
# get existing event if relevant
|
||||
if slug and slug in seen_slugs:
|
||||
event = events_by_slug[slug]
|
||||
# update label
|
||||
event.label = label
|
||||
if event is None:
|
||||
# new event
|
||||
event = Event(agenda_id=self.agenda.pk, label=label)
|
||||
# generate a slug if not provided
|
||||
event.slug = slug or generate_slug(event, seen_slugs=seen_slugs, agenda=self.agenda.pk)
|
||||
# maintain caches
|
||||
seen_slugs.add(event.slug)
|
||||
events_by_slug[event.slug] = event
|
||||
if errors:
|
||||
errors = [_('Invalid file format:')] + errors
|
||||
raise ValidationError(errors)
|
||||
|
||||
def parse_csvline(self, csvline):
|
||||
if len(csvline) < 3:
|
||||
raise ValidationError(_('Not enough columns.'))
|
||||
|
||||
# label needed to generate a slug
|
||||
label = None
|
||||
if len(csvline) >= 5:
|
||||
label = force_str(csvline[4])
|
||||
|
||||
# get or create event
|
||||
event = None
|
||||
slug = None
|
||||
if len(csvline) >= 6:
|
||||
slug = force_str(csvline[5]) if csvline[5] else None
|
||||
# get existing event if relevant
|
||||
if slug and slug in self.seen_slugs:
|
||||
event = self.events_by_slug[slug]
|
||||
# update label
|
||||
event.label = label
|
||||
if event is None:
|
||||
# new event
|
||||
event = Event(agenda_id=self.agenda.pk, label=label)
|
||||
# generate a slug if not provided
|
||||
event.slug = slug or generate_slug(event, seen_slugs=self.seen_slugs, agenda=self.agenda.pk)
|
||||
# maintain caches
|
||||
self.seen_slugs.add(event.slug)
|
||||
self.events_by_slug[event.slug] = event
|
||||
|
||||
for datetime_fmt in (
|
||||
'%Y-%m-%d %H:%M',
|
||||
'%d/%m/%Y %H:%M',
|
||||
'%d/%m/%Y %Hh%M',
|
||||
'%Y-%m-%d %H:%M:%S',
|
||||
'%d/%m/%Y %H:%M:%S',
|
||||
):
|
||||
try:
|
||||
event_datetime = make_aware(
|
||||
datetime.datetime.strptime('%s %s' % tuple(csvline[:2]), datetime_fmt)
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
if (
|
||||
event.pk is not None
|
||||
and event.start_datetime != event_datetime
|
||||
and event.start_datetime > now()
|
||||
and event.pk in self.event_ids_with_bookings
|
||||
and event.pk not in self.warnings
|
||||
):
|
||||
# event start datetime has changed, event is not past and has not cancelled bookings
|
||||
# => warn the user
|
||||
self.warnings[event.pk] = event
|
||||
event.start_datetime = event_datetime
|
||||
break
|
||||
else:
|
||||
raise ValidationError(_('Wrong start date/time format.'))
|
||||
try:
|
||||
event.places = int(csvline[2])
|
||||
except ValueError:
|
||||
raise ValidationError(_('Number of places must be an integer.'))
|
||||
if len(csvline) >= 4:
|
||||
try:
|
||||
event.waiting_list_places = int(csvline[3])
|
||||
except ValueError:
|
||||
raise ValidationError(_('Number of places in waiting list must be an integer.'))
|
||||
|
||||
column_index = 7
|
||||
for more_attr in ('description', 'pricing', 'url'):
|
||||
if len(csvline) >= column_index:
|
||||
setattr(event, more_attr, csvline[column_index - 1])
|
||||
column_index += 1
|
||||
|
||||
if len(csvline) >= 10 and csvline[9]: # publication date is optional
|
||||
for datetime_fmt in (
|
||||
'%Y-%m-%d',
|
||||
'%d/%m/%Y',
|
||||
'%Y-%m-%d %H:%M',
|
||||
'%d/%m/%Y %H:%M',
|
||||
'%d/%m/%Y %Hh%M',
|
||||
|
@ -1284,93 +1346,39 @@ class ImportEventsForm(forms.Form):
|
|||
'%d/%m/%Y %H:%M:%S',
|
||||
):
|
||||
try:
|
||||
event_datetime = make_aware(
|
||||
datetime.datetime.strptime('%s %s' % tuple(csvline[:2]), datetime_fmt)
|
||||
event.publication_datetime = make_aware(
|
||||
datetime.datetime.strptime(csvline[9], datetime_fmt)
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
if (
|
||||
event.pk is not None
|
||||
and event.start_datetime != event_datetime
|
||||
and event.start_datetime > now()
|
||||
and event.pk in event_ids_with_bookings
|
||||
and event.pk not in warnings
|
||||
):
|
||||
# event start datetime has changed, event is not past and has not cancelled bookings
|
||||
# => warn the user
|
||||
warnings[event.pk] = event
|
||||
event.start_datetime = event_datetime
|
||||
break
|
||||
else:
|
||||
raise ValidationErrorWithOrdinal(
|
||||
_('Invalid file format. (date/time format, {event_no} event)'), i
|
||||
)
|
||||
try:
|
||||
event.places = int(csvline[2])
|
||||
except ValueError:
|
||||
raise ValidationError(_('Invalid file format. (number of places, {event_no} event)'), i)
|
||||
if len(csvline) >= 4:
|
||||
raise ValidationError(_('Wrong publication date/time format.'))
|
||||
|
||||
if self.agenda.partial_bookings:
|
||||
if len(csvline) < 11 or not csvline[10]:
|
||||
raise ValidationError(_('Missing end_time.'))
|
||||
event.end_time = csvline[10]
|
||||
else:
|
||||
self.exclude_from_validation.append('end_time')
|
||||
if len(csvline) >= 11 and csvline[10]: # duration is optional
|
||||
try:
|
||||
event.waiting_list_places = int(csvline[3])
|
||||
event.duration = int(csvline[10])
|
||||
except ValueError:
|
||||
raise ValidationError(
|
||||
_('Invalid file format. (number of places in waiting list, {event_no} event)'), i
|
||||
)
|
||||
raise ValidationError(_('Duration must be an integer.'))
|
||||
|
||||
column_index = 7
|
||||
for more_attr in ('description', 'pricing', 'url'):
|
||||
if len(csvline) >= column_index:
|
||||
setattr(event, more_attr, csvline[column_index - 1])
|
||||
column_index += 1
|
||||
try:
|
||||
event.full_clean(exclude=self.exclude_from_validation)
|
||||
except ValidationError as e:
|
||||
errors = []
|
||||
for label, field_errors in e.message_dict.items():
|
||||
label_name = self.get_verbose_name(label)
|
||||
msg = _('%s: ') % label_name if label_name else ''
|
||||
msg += ', '.join(field_errors)
|
||||
errors.append(msg)
|
||||
raise ValidationError(errors)
|
||||
|
||||
if len(csvline) >= 10 and csvline[9]: # publication date is optional
|
||||
for datetime_fmt in (
|
||||
'%Y-%m-%d',
|
||||
'%d/%m/%Y',
|
||||
'%Y-%m-%d %H:%M',
|
||||
'%d/%m/%Y %H:%M',
|
||||
'%d/%m/%Y %Hh%M',
|
||||
'%Y-%m-%d %H:%M:%S',
|
||||
'%d/%m/%Y %H:%M:%S',
|
||||
):
|
||||
try:
|
||||
event.publication_datetime = make_aware(
|
||||
datetime.datetime.strptime(csvline[9], datetime_fmt)
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
raise ValidationError(_('Invalid file format. (date/time format, {event_no} event)'), i)
|
||||
|
||||
if self.agenda.partial_bookings:
|
||||
if len(csvline) < 11 or not csvline[10]:
|
||||
raise ValidationError(_('Invalid file format. (missing end_time, {event_no} event)'), i)
|
||||
event.end_time = csvline[10]
|
||||
else:
|
||||
exclude_from_validation.append('end_time')
|
||||
if len(csvline) >= 11 and csvline[10]: # duration is optional
|
||||
try:
|
||||
event.duration = int(csvline[10])
|
||||
except ValueError:
|
||||
raise ValidationError(_('Invalid file format. (duration, {event_no} event)'), i)
|
||||
|
||||
try:
|
||||
event.full_clean(exclude=exclude_from_validation)
|
||||
except ValidationError as e:
|
||||
errors = [_('Invalid file format:\n')]
|
||||
for label, field_errors in e.message_dict.items():
|
||||
label_name = self.get_verbose_name(label)
|
||||
msg = _('%s: ') % label_name if label_name else ''
|
||||
msg += _('%(errors)s (line %(line)d)') % {
|
||||
'errors': ', '.join(field_errors),
|
||||
'line': i + 1,
|
||||
}
|
||||
errors.append(msg)
|
||||
raise ValidationError(errors)
|
||||
events.append(event)
|
||||
self.events = events
|
||||
self.warnings = warnings
|
||||
return event
|
||||
|
||||
@staticmethod
|
||||
def get_verbose_name(field_name):
|
||||
|
|
|
@ -798,7 +798,8 @@ def test_import_events(app, admin_user):
|
|||
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'xx', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format.' in resp.text
|
||||
assert 'Invalid file format:' in resp.text
|
||||
assert 'Wrong start date/time format. (1st event)' in resp.text
|
||||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'xxxx\0\0xxxx', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
|
@ -806,15 +807,16 @@ def test_import_events(app, admin_user):
|
|||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format.' in resp.text
|
||||
assert 'Invalid file format:' in resp.text
|
||||
assert 'Not enough columns. (1st event)' in resp.text
|
||||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'date,time,etc.\n2016-14-16,18:00,10', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (date/time format, 1st event)' in resp.text
|
||||
assert 'Wrong start date/time format. (1st event)' in resp.text
|
||||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00,10', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (date/time format, 1st event)' in resp.text
|
||||
assert 'Wrong start date/time format. (1st event)' in resp.text
|
||||
|
||||
with override_settings(LANGUAGE_CODE='fr-fr'):
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-14-16,18:00,10', 'text/csv')
|
||||
|
@ -824,11 +826,11 @@ def test_import_events(app, admin_user):
|
|||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,blah', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (number of places,' in resp.text
|
||||
assert 'Number of places must be an integer. (1st event)' in resp.text
|
||||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,blah', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (number of places in waiting list,' in resp.text
|
||||
assert 'Number of places in waiting list must be an integer. (1st event)' in resp.text
|
||||
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,5,' + b'x' * 151, 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
|
@ -988,7 +990,7 @@ def test_import_events(app, admin_user):
|
|||
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,foobar', 'text/csv'
|
||||
)
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (date/time format' in resp.text
|
||||
assert 'Wrong publication date/time format. (1st event)' in resp.text
|
||||
|
||||
# duration bad format
|
||||
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
|
||||
|
@ -996,7 +998,7 @@ def test_import_events(app, admin_user):
|
|||
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,2016-09-16,foobar', 'text/csv'
|
||||
)
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (duration' in resp.text
|
||||
assert 'Duration must be an integer. (1st event)' in resp.text
|
||||
|
||||
# import events with empty slugs
|
||||
Event.objects.all().delete()
|
||||
|
@ -1027,8 +1029,7 @@ def test_import_events(app, admin_user):
|
|||
resp = app.get('/manage/agendas/%s/import-events' % agenda.id, status=200)
|
||||
resp.form['events_csv_file'] = Upload('t.csv', b'2016-09-16,18:00,10,5,label,1234', 'text/csv')
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'value cannot be a number' in resp.text
|
||||
assert 'Identifier:' in resp.text # verbose_name is shown, not field name ('slug:')
|
||||
assert 'Identifier: This value cannot be a number. (1st event)' in resp.text
|
||||
|
||||
|
||||
def test_import_event_nested_quotes(app, admin_user):
|
||||
|
@ -1192,7 +1193,7 @@ def test_import_events_partial_bookings(app, admin_user):
|
|||
't.csv', b'2016-09-16,18:00,10,5,label,slug,description,pricing,url,2016-09-16', 'text/csv'
|
||||
)
|
||||
resp = resp.form.submit(status=200)
|
||||
assert 'Invalid file format. (missing end_time' in resp.text
|
||||
assert 'Missing end_time.' in resp.text
|
||||
|
||||
# invalid end time
|
||||
resp = app.get('/manage/agendas/%s/import-events' % agenda.pk)
|
||||
|
@ -1203,6 +1204,27 @@ def test_import_events_partial_bookings(app, admin_user):
|
|||
assert '“xxx” value has an invalid format' in resp.text
|
||||
|
||||
|
||||
def test_import_events_multiple_errors(app, admin_user):
|
||||
agenda = Agenda.objects.create(label='Foo bar', kind='events')
|
||||
Desk.objects.create(agenda=agenda, slug='_exceptions_holder')
|
||||
|
||||
app = login(app)
|
||||
resp = app.get('/manage/agendas/%s/import-events' % agenda.pk)
|
||||
resp.form['events_csv_file'] = Upload(
|
||||
't.csv',
|
||||
b'2016-09-17,18:00,10,5,label,slug\n' # valid event
|
||||
b'2016-09-17,19:00,xxx,5,label2,slug2\n' # invalid places
|
||||
b'2016-09-17,20:00,10,5,,1234\n', # invalid slug
|
||||
'text/csv',
|
||||
)
|
||||
resp = resp.form.submit(status=200)
|
||||
assert [x.text for x in resp.pyquery('.errorlist li')] == [
|
||||
'Invalid file format:',
|
||||
'Number of places must be an integer. (2nd event)',
|
||||
'Identifier: This value cannot be a number. (3rd event)',
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.freeze_time('2022-05-24')
|
||||
def test_event_detail(app, admin_user):
|
||||
agenda = Agenda.objects.create(label='Events', kind='events')
|
||||
|
|
Loading…
Reference in New Issue