cvsdatasource: add pagination to data and query endpoints (#32121)

passerelle/apps/csvdatasource/models.py

@@ -400,6 +400,23 @@ class CsvDataSource(BaseResource):
         # force rendition of iterator as list
         data = list(data)
 
+        if 'limit' in request.GET:
+            try:
+                limit = int(request.GET['limit'])
+            except ValueError:
+                raise APIError('invalid limit parameter')
+            if limit < 1:
+                raise APIError('invalid limit parameter')
+            try:
+                offset = int(request.GET.get('offset') or 0)
+            except ValueError:
+                raise APIError('invalid offset parameter')
+            if offset < 0:
+                raise APIError('invalid offset parameter')
+
+            # paginate data
+            data = data[offset:offset+limit]
+
         if query.structure == 'array':
             return {'data': [[row[t] for t in titles] for row in data]}
         elif query.structure == 'dict':

tests/test_csv_datasource.py

@@ -28,6 +28,7 @@ from django.core.urlresolvers import reverse
 from django.contrib.contenttypes.models import ContentType
 from django.test import Client
 from django.core.management import call_command
+from django.utils.six.moves.urllib.parse import urlencode
 
 from passerelle.base.models import ApiUser, AccessRight
 from passerelle.apps.csvdatasource.models import CsvDataSource, Query, TableRow
@@ -764,3 +765,61 @@ def test_change_csv_command(setup):
     call_command('change-csv', 'test', os.path.join(TEST_BASE_DIR, 'data.ods'), sheet_name='Feuille2')
     csv.refresh_from_db()
     assert list(csv.get_rows()) != []
+
+
+@pytest.mark.parametrize('payload,expected', [
+    ({}, 20),
+    ({'limit': 10}, 10),
+    ({'limit': 10, 'offset': 0}, 10),
+    ({'limit': 10, 'offset': 15}, 5),
+    ({'limit': 10, 'offset': 42}, 0),
+])
+def test_pagination(app, setup, filetype, payload, expected):
+    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype,
+                         data=get_file_content(filetype))
+
+    # data endpoint
+    response = app.get(url + '?' + urlencode(payload))
+    assert len(response.json['data']) == expected
+
+    # query endpoint
+    url = reverse('generic-endpoint', kwargs={
+        'connector': 'csvdatasource',
+        'slug': csvdata.slug,
+        'endpoint': 'query/query-1_/',
+    })
+    query = Query(slug='query-1_', resource=csvdata, structure='array')
+    query.projections = '\n'.join(['id:int(id)', 'text:prenom'])
+    query.save()
+    response = app.get(url + '?' + urlencode(payload))
+    assert len(response.json['data']) == expected
+
+
+@pytest.mark.parametrize('payload,expected_error', [
+    ({'limit': 'bla'}, 'invalid limit parameter'),
+    ({'limit': 0}, 'invalid limit parameter'),
+    ({'limit': -1}, 'invalid limit parameter'),
+    ({'limit': 10, 'offset': 'bla'}, 'invalid offset parameter'),
+    ({'limit': 10, 'offset': -1}, 'invalid offset parameter'),
+])
+def test_pagination_error(app, setup, filetype, payload, expected_error):
+    csvdata, url = setup('id,whatever,nom,prenom,sexe', filename=filetype,
+                         data=get_file_content(filetype))
+
+    # data endpoint
+    response = app.get(url + '?' + urlencode(payload))
+    assert response.json['err'] == 1
+    assert response.json['err_desc'] == expected_error
+
+    # query endpoint
+    url = reverse('generic-endpoint', kwargs={
+        'connector': 'csvdatasource',
+        'slug': csvdata.slug,
+        'endpoint': 'query/query-1_/',
+    })
+    query = Query(slug='query-1_', resource=csvdata, structure='array')
+    query.projections = '\n'.join(['id:int(id)', 'text:prenom'])
+    query.save()
+    response = app.get(url + '?' + urlencode(payload))
+    assert response.json['err'] == 1
+    assert response.json['err_desc'] == expected_error