api: add import-csv endpoint to create cards from a CSV (#48210)

tests/backoffice_pages/test_all.py

@@ -6339,6 +6339,10 @@ def test_backoffice_cards_import_data_from_csv(pub, studio):
             "value,"
             "value\r\n" % (pub.get_default_position(), today))
 
+    # missing file
+    resp = resp.forms[0].submit()
+    assert '>required field<' in resp
+
     resp.forms[0]['file'] = Upload('test.csv', b'\0', 'text/csv')
     resp = resp.forms[0].submit()
     assert 'Invalid file format.' in resp

tests/test_api.py

@@ -3384,6 +3384,36 @@ def test_cards(pub, local_user):
     assert resp.json['fields'][0]['varname'] == 'foo'
 
 
+def test_cards_import_csv(pub, local_user):
+    Role.wipe()
+    role = Role(name='test')
+    role.store()
+    local_user.roles = [role.id]
+    local_user.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='0', label='foobar', varname='foo'),
+        fields.StringField(id='1', label='foobar2', varname='foo2'),
+    ]
+    carddef.workflow_roles = {'_viewer': role.id}
+    carddef.backoffice_submission_roles = [role.id]
+    carddef.digest_template = 'bla {{ form_var_foo }} xxx'
+    carddef.store()
+
+    carddef.data_class().wipe()
+
+    get_app(pub).get(sign_uri('/api/cards/test/import-csv'), status=405)
+    get_app(pub).put(sign_uri('/api/cards/test/import-csv'), status=403)
+    get_app(pub).put(sign_uri('/api/cards/test/import-csv', user=local_user),
+            params=b'foobar;foobar2\nfirst entry;plop\nsecond entry;plop\n',
+            headers={'content-type': 'text/csv'})
+    assert carddef.data_class().count() == 2
+    assert set([x.data['0'] for x in carddef.data_class().select()]) == {'first entry', 'second entry'}
+
+
 def test_api_invalid_http_basic_auth(pub, local_user, admin_user, ics_data):
     app = get_app(pub)
     app.get('/api/forms/test/ics/foobar?email=%s' % local_user.email,

wcs/api.py

@@ -19,6 +19,7 @@ import re
 import time
 
 from quixote import get_request, get_publisher, get_response, get_session
+from quixote.errors import MethodNotAllowedError, QueryError
 from quixote.directory import Directory
 
 from django.utils.encoding import force_text
@@ -42,6 +43,7 @@ from wcs.api_utils import sign_url_auto_orig, is_url_signed, get_user_from_api_q
 
 from .backoffice.management import FormPage as BackofficeFormPage
 from .backoffice.management import ManagementDirectory
+from .backoffice.data_management import CardPage as BackofficeCardPage
 
 
 def posted_json_data_to_formdata_data(formdef, data):
@@ -168,10 +170,7 @@ class ApiFormdataPage(FormStatusPage):
             raise AccessForbiddenError('unsufficient roles')
 
 
-class ApiFormPage(BackofficeFormPage):
-    _q_exports = [('list', 'json'), 'geojson', 'ods']  # restrict to API endpoints
-    formdef_class = FormDef
-
+class ApiFormPageMixin:
     def __init__(self, component):
         try:
             self.formdef = self.formdef_class.get_by_urlname(component)
@@ -227,17 +226,40 @@ class ApiFormPage(BackofficeFormPage):
             # webhooks have their own access checks, request cannot be blocked
             # at this point.
             self.is_webhook = bool(path[1] == 'hooks')
-        return super(ApiFormPage, self)._q_traverse(path)
+        return super()._q_traverse(path)
 
 
-class ApiCardPage(ApiFormPage):
-    formdef_class = CardDef
-    _q_exports = [('list', 'json'), 'geojson', 'ods', ('@schema', 'schema')]  # restrict to API endpoints
+class ApiFormPage(ApiFormPageMixin, BackofficeFormPage):
+    _q_exports = [('list', 'json'), 'geojson', 'ods']  # restrict to API endpoints
+
+
+class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
+    _q_exports = [  # restricted to API endpoints
+            ('list', 'json'),
+            ('import-csv', 'import_csv'),
+            'geojson',
+            'ods',
+            ('@schema', 'schema'),
+    ]
 
     def schema(self):
         get_response().set_content_type('application/json')
         return self.formdef.export_to_json(anonymise=not is_url_signed())
 
+    def import_csv(self):
+        if get_request().get_method() != 'PUT':
+            raise MethodNotAllowedError(allowed_methods=['PUT'])
+        get_request()._user = get_user_from_api_query_string()
+        if not (get_request()._user and self.can_user_add_cards()):
+            raise AccessForbiddenError('cannot import cards')
+
+        get_response().set_content_type('application/json')
+        try:
+            self.import_csv_submit(get_request().stdin, afterjob=False)
+        except ValueError as e:
+            return json.dumps({'err': 1, 'err_desc': str(e)})
+        return json.dumps({'err': 0})
+
 
 class ApiFormsDirectory(Directory):
     _q_exports = ['', 'geojson']

wcs/backoffice/data_management.py

@@ -174,7 +174,7 @@ class CardPage(FormPage):
             context['job'] = job
         except KeyError:
             form = Form(enctype='multipart/form-data', use_tokens=False)
-            form.add(FileWidget, 'file', title=_('File'), required=False)
+            form.add(FileWidget, 'file', title=_('File'), required=True)
             form.add_submit('submit', _('Submit'))
             form.add_submit('cancel', _('Cancel'))
             if form.get_widget('cancel').parse():
@@ -182,7 +182,7 @@ class CardPage(FormPage):
 
             if form.is_submitted() and not form.has_errors():
                 try:
-                    return self.import_csv_submit(form)
+                    return self.import_csv_submit(form.get_widget('file').parse().fp)
                 except ValueError as e:
                     form.set_error('file', e)
             context['form'] = form
@@ -202,13 +202,10 @@ class CardPage(FormPage):
             templates=['wcs/backoffice/card-data-import-form.html'],
             context=context)
 
-    def import_csv_submit(self, form):
-        if form.get_widget('file').parse():
-            content = form.get_widget('file').parse().fp.read()
-            if b'\0' in content:
-                raise ValueError(_('Invalid file format.'))
-        else:
-            raise ValueError(_('You have to enter a file.'))
+    def import_csv_submit(self, fd, afterjob=True):
+        content = fd.read()
+        if b'\0' in content:
+            raise ValueError(_('Invalid file format.'))
 
         for charset in ('utf-8', 'iso-8859-15'):
             try:
@@ -275,7 +272,7 @@ class CardPage(FormPage):
                 self.data_class = data_class
                 self.lines = lines
 
-            def execute(self, job):
+            def execute(self, job=None):
                 for item in self.lines:
                     data_instance = self.data_class()
                     data_instance.data = item
@@ -284,10 +281,13 @@ class CardPage(FormPage):
                     data_instance.perform_workflow()
 
         action = ImportAction(self.formdef.data_class(), data_lines)
-        job = get_response().add_after_job(N_('Importing data into cards'),
-                                           action.execute)
-        job.store()
-        return redirect('import-csv?job=%s' % job.id)
+        if afterjob:
+            job = get_response().add_after_job(N_('Importing data into cards'),
+                                               action.execute)
+            job.store()
+            return redirect('import-csv?job=%s' % job.id)
+        else:
+            action.execute()
 
     def _q_lookup(self, component):
 

wcs/qommon/publisher.py

@@ -413,6 +413,7 @@ class QommonPublisher(Publisher, object):
         self.config.display_exceptions = debug_cfg.get('display_exceptions')
         self.config.form_tokens = True
         self.config.session_cookie_httponly = True
+        self.config.allowed_methods = ['GET', 'HEAD', 'POST', 'PUT']
 
         if request:
             if request.get_scheme() == 'https':