tests/admin_pages/test_form.py

@@ -13,7 +13,7 @@ from webtest import Upload
 from wcs import fields
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
-from wcs.categories import Category, DataSourceCategory, WorkflowCategory
+from wcs.categories import CardDefCategory, Category, DataSourceCategory, WorkflowCategory
 from wcs.data_sources import NamedDataSource
 from wcs.formdef import FormDef
 from wcs.qommon.errors import ConnectionError
@@ -2252,6 +2252,8 @@ def test_form_edit_item_field_data_source_with_categories(pub):
     CardDef.wipe()
     DataSourceCategory.wipe()
     NamedDataSource.wipe()
+    pub.custom_view_class.wipe()
+
     data_source = NamedDataSource(name='test')
     data_source.store()
 
@@ -2262,6 +2264,22 @@ def test_form_edit_item_field_data_source_with_categories(pub):
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/forms/1/fields/1/')
+    assert len(resp.pyquery('select[name="data_source$type"] optgroup')) == 2
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).attr['label']
+        == 'Manually Configured Data Sources'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).find('option').text() == 'test'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).attr['label']
+        == 'Generic Data Sources'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).find('option').text()
+        == 'JSON URL JSONP URL Python Expression (deprecated) JSON Expression'
+    )
     assert [o[0] for o in resp.form['data_source$type'].options] == [
         'None',
         'test',
@@ -2286,6 +2304,32 @@ def test_form_edit_item_field_data_source_with_categories(pub):
     data_source.store()
 
     resp = app.get('/backoffice/forms/1/fields/1/')
+    assert len(resp.pyquery('select[name="data_source$type"] optgroup')) == 4
+    assert PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).attr['label'] == 'Cat A'
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).find('option').text()
+        == 'foo baz'
+    )
+    assert PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).attr['label'] == 'Cat B'
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).find('option').text()
+        == 'bar foo foo bar'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[2]).attr['label']
+        == 'Without category'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[2]).find('option').text() == 'test'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[3]).attr['label']
+        == 'Generic Data Sources'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[3]).find('option').text()
+        == 'JSON URL JSONP URL Python Expression (deprecated) JSON Expression'
+    )
     assert [o[0] for o in resp.form['data_source$type'].options] == [
         'None',
         'foo_baz',
@@ -2299,6 +2343,102 @@ def test_form_edit_item_field_data_source_with_categories(pub):
     ]
 
 
+def test_form_edit_item_field_data_source_with_carddef_categories(pub):
+    create_superuser(pub)
+    create_role(pub)
+
+    FormDef.wipe()
+    CardDefCategory.wipe()
+    CardDef.wipe()
+    DataSourceCategory.wipe()
+    NamedDataSource.wipe()
+    pub.custom_view_class.wipe()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [fields.ItemField(id='1', label='1st field', type='item')]
+    formdef.store()
+
+    carddef = CardDef()
+    carddef.name = 'Baz'
+    carddef.digest_templates = {'default': 'plop'}
+    carddef.store()
+    custom_view = pub.custom_view_class()
+    custom_view.title = 'card view'
+    custom_view.formdef = carddef
+    custom_view.visibility = 'datasource'
+    custom_view.store()
+
+    carddef2 = CardDef()
+    carddef2.name = 'Bar'
+    carddef2.digest_templates = {'default': 'plop'}
+    carddef2.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/forms/1/fields/1/')
+    assert len(resp.pyquery('select[name="data_source$type"] optgroup')) == 2
+    assert PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).attr['label'] == 'Cards'
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).find('option').text()
+        == 'Bar Baz Baz - card view'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).attr['label']
+        == 'Generic Data Sources'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).find('option').text()
+        == 'JSON URL JSONP URL Python Expression (deprecated) JSON Expression'
+    )
+    assert [o[0] for o in resp.form['data_source$type'].options] == [
+        'None',
+        'carddef:bar',
+        'carddef:baz',
+        'carddef:baz:card-view',
+        'json',
+        'jsonp',
+        'python',
+        'jsonvalue',
+    ]
+
+    category = CardDefCategory(name='Foobar')
+    category.store()
+    carddef.category = category
+    carddef.store()
+    resp = app.get('/backoffice/forms/1/fields/1/')
+    assert len(resp.pyquery('select[name="data_source$type"] optgroup')) == 3
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).attr['label'] == 'Cards - Foobar'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[0]).find('option').text()
+        == 'Baz Baz - card view'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).attr['label']
+        == 'Cards - Uncategorised'
+    )
+    assert PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[1]).find('option').text() == 'Bar'
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[2]).attr['label']
+        == 'Generic Data Sources'
+    )
+    assert (
+        PyQuery(resp.pyquery('select[name="data_source$type"] optgroup')[2]).find('option').text()
+        == 'JSON URL JSONP URL Python Expression (deprecated) JSON Expression'
+    )
+    assert [o[0] for o in resp.form['data_source$type'].options] == [
+        'None',
+        'carddef:baz',
+        'carddef:baz:card-view',
+        'carddef:bar',
+        'json',
+        'jsonp',
+        'python',
+        'jsonvalue',
+    ]
+
+
 def test_form_edit_item_field_geojson_data_source(pub, http_requests):
     NamedDataSource.wipe()
     create_superuser(pub)

tests/backoffice_pages/test_applications.py

@@ -996,6 +996,7 @@ def test_datasources(pub, application_with_icon, application_without_icon, icon)
     NamedDataSource.wipe()
     DataSourceCategory.wipe()
     CardDef.wipe()
+    CardDefCategory.wipe()
 
     if icon:
         application = application_with_icon
@@ -1156,8 +1157,14 @@ def test_datasources(pub, application_with_icon, application_without_icon, icon)
     ApplicationElement.update_or_create_for_object(application, cat)
     datasource2.category = cat
     datasource2.store()
+    cat = CardDefCategory()
+    cat.name = 'card cat'
+    cat.store()
+    ApplicationElement.update_or_create_for_object(application, cat)
+    carddef2.category = cat
+    carddef2.store()
     resp = app.get('/backoffice/forms/data-sources/')
-    assert len(resp.pyquery('.section')) == 4
+    assert len(resp.pyquery('.section')) == 5
     assert PyQuery(resp.pyquery('.section')[0]).find('h2').text() == 'Users Data Sources'
     assert len(PyQuery(resp.pyquery('.section')[0]).find('ul.objects-list li')) == 2
     assert (
@@ -1186,11 +1193,16 @@ def test_datasources(pub, application_with_icon, application_without_icon, icon)
     )
     assert (
         PyQuery(resp.pyquery('.section')[3]).find('h2').text()
-        == 'Data Sources from Card Models - automatically configured'
+        == 'Data Sources from Card Models - automatically configured - card cat'
     )
-    assert len(PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li')) == 2
-    assert PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li:nth-child(1)').text() == 'card1'
-    assert PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li:nth-child(2)').text() == 'card2'
+    assert len(PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li')) == 1
+    assert PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li:nth-child(1)').text() == 'card2'
+    assert (
+        PyQuery(resp.pyquery('.section')[4]).find('h2').text()
+        == 'Data Sources from Card Models - automatically configured - Uncategorised'
+    )
+    assert len(PyQuery(resp.pyquery('.section')[4]).find('ul.objects-list li')) == 1
+    assert PyQuery(resp.pyquery('.section')[4]).find('ul.objects-list li:nth-child(1)').text() == 'card1'
 
     # check application view
     resp = resp.click(href='application/%s/' % application.slug)
@@ -1215,7 +1227,7 @@ def test_datasources(pub, application_with_icon, application_without_icon, icon)
     )
     assert (
         PyQuery(resp.pyquery('.section')[3]).find('h2').text()
-        == 'Data Sources from Card Models - automatically configured'
+        == 'Data Sources from Card Models - automatically configured - card cat'
     )
     assert len(PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li')) == 1
     assert PyQuery(resp.pyquery('.section')[3]).find('ul.objects-list li:nth-child(1)').text() == 'card2'

wcs/admin/data_sources.py

@@ -23,7 +23,7 @@ from wcs.admin.categories import DataSourceCategoriesDirectory, get_categories
 from wcs.backoffice.applications import ApplicationsDirectory
 from wcs.backoffice.snapshots import SnapshotsDirectory
 from wcs.carddef import CardDef
-from wcs.categories import DataSourceCategory
+from wcs.categories import CardDefCategory, DataSourceCategory
 from wcs.data_sources import (
     DataSourceSelectionWidget,
     NamedDataSource,
@@ -564,12 +564,21 @@ class NamedDataSourcesDirectory(Directory):
             generated_data_sources = [g for g in generated_data_sources if g[0] in carddefs]
         else:
             Application.populate_objects([g[0] for g in generated_data_sources])
+        carddef_categories = CardDefCategory.select()
+        CardDefCategory.sort_by_position(carddef_categories)
+        if carddef_categories:
+            carddef_categories.append(CardDefCategory(pgettext('categories', 'Uncategorised')))
+            for carddef_category in carddef_categories:
+                carddef_category.generated_data_sources = [
+                    x for x in generated_data_sources if x[0].category_id == carddef_category.id
+                ]
         return {
             'data_sources': data_sources,
             'categories': categories,
             'user_data_sources': user_data_sources,
             'agenda_data_sources': agenda_data_sources,
             'generated_data_sources': generated_data_sources,
+            'carddef_categories': carddef_categories,
         }
 
     def _new(self, url, breadcrumb, title, ds_type=None):

wcs/data_sources.py

@@ -30,7 +30,7 @@ from quixote.html import TemplateIO
 
 from .api_utils import sign_url_auto_orig
 from .categories import DataSourceCategory
-from .qommon import _, force_str, get_logger, misc
+from .qommon import _, get_logger, misc, pgettext
 from .qommon.afterjobs import AfterJob
 from .qommon.cron import CronJob
 from .qommon.form import CompositeWidget, OptGroup, SingleSelectWidget, StringWidget
@@ -76,20 +76,33 @@ class DataSourceSelectionWidget(CompositeWidget):
 
         if 'cards' in allowed_source_types:
             from wcs.carddef import CardDef
+            from wcs.categories import CardDefCategory
 
             user = get_request().user
             cards_options = []
             for ds in CardDef.get_carddefs_as_data_source():
-                option = [ds[2], ds[1], ds[2]]
+                option = [ds[2], ds[1], ds[2], {'carddef': ds[0]}]
                 if ds[3] and (user.is_admin or ds[0].is_of_concern_for_user(user)):
-                    option.append({'data-goto-url': '%s%s' % (ds[0].get_url(), ds[3].get_url_slug())})
+                    option[3].update({'data-goto-url': '%s%s' % (ds[0].get_url(), ds[3].get_url_slug())})
                 elif get_publisher().get_backoffice_root().is_accessible('cards'):
-                    option.append({'data-goto-url': ds[0].get_admin_url()})
+                    option[3].update({'data-goto-url': ds[0].get_admin_url()})
                 cards_options.append(option)
             cards_options.sort(key=lambda x: misc.simplify(x[1]))
             if cards_options:
-                options.append(OptGroup(_('Cards')))
-                options.extend(cards_options)
+                carddef_categories = CardDefCategory.select()
+                CardDefCategory.sort_by_position(carddef_categories)
+                if carddef_categories:
+                    carddef_categories.append(CardDefCategory(pgettext('categories', 'Uncategorised')))
+                    for carddef_category in carddef_categories:
+                        carddef_category.cards_options = [
+                            x for x in cards_options if x[3]['carddef'].category_id == carddef_category.id
+                        ]
+                        if carddef_category.cards_options:
+                            options.append(OptGroup('%s - %s' % (_('Cards'), carddef_category.name)))
+                            options.extend(carddef_category.cards_options)
+                else:
+                    options.append(OptGroup(_('Cards')))
+                    options.extend(cards_options)
 
         if 'named' in allowed_source_types:
             admin_accessible = NamedDataSource.is_admin_accessible()

wcs/templates/wcs/backoffice/includes/data-sources.html

@@ -59,25 +59,45 @@
   {% endif %}
 {% endif %}
 
-<div class="section foldable">
-  <h2>{% trans "Data Sources from Card Models" %}  - {% trans "automatically configured" %}</h2>
-  {% if generated_data_sources %}
-    <ul class="objects-list single-links">
-      {% for data_source in generated_data_sources %}
-        <li>
-          <a href="{{ data_source.0.get_url }}{% if data_source.3 %}{{ data_source.3.get_url_slug }}/{% endif %}">
-            {% if not application %}{% include 'wcs/backoffice/includes/application_icons.html' with object=data_source.0 %}{% endif %}
-            {{ data_source.1 }}
-          </a>
-        </li>
-      {% endfor %}
-    </ul>
-  {% else %}
-    <div>
-      {% trans "There are no data sources from card models." %}
-    </div>
-  {% endif %}
-</div>
+{% if carddef_categories %}
+  {% for carddef_category in carddef_categories %}
+    {% if carddef_category.generated_data_sources %}
+      <div class="section foldable">
+        <h2>{% trans "Data Sources from Card Models" %} - {% trans "automatically configured" %} - {{ carddef_category.name }}</h2>
+        <ul class="objects-list single-links">
+          {% for data_source in carddef_category.generated_data_sources %}
+            <li>
+              <a href="{{ data_source.0.get_url }}{% if data_source.3 %}{{ data_source.3.get_url_slug }}/{% endif %}">
+                {% if not application %}{% include 'wcs/backoffice/includes/application_icons.html' with object=data_source.0 %}{% endif %}
+                {{ data_source.1 }}
+              </a>
+            </li>
+          {% endfor %}
+        </ul>
+      </div>
+    {% endif %}
+  {% endfor %}
+{% else %}
+  <div class="section foldable">
+    <h2>{% trans "Data Sources from Card Models" %} - {% trans "automatically configured" %}</h2>
+    {% if generated_data_sources %}
+      <ul class="objects-list single-links">
+        {% for data_source in generated_data_sources %}
+          <li>
+            <a href="{{ data_source.0.get_url }}{% if data_source.3 %}{{ data_source.3.get_url_slug }}/{% endif %}">
+              {% if not application %}{% include 'wcs/backoffice/includes/application_icons.html' with object=data_source.0 %}{% endif %}
+              {{ data_source.1 }}
+            </a>
+          </li>
+        {% endfor %}
+      </ul>
+    {% else %}
+      <div>
+        {% trans "There are no data sources from card models." %}
+      </div>
+    {% endif %}
+  </div>
+{% endif %}
 
 {% if has_chrono %}
   <div class="section foldable">