api: export/import, get wcs dependencies (#86291)

lingo/export_import/api_views.py

@@ -34,6 +34,7 @@ from lingo.invoicing.models import Payer, Regie
 from lingo.invoicing.utils import import_site as invoicing_import_site
 from lingo.pricing.models import CriteriaCategory, Pricing
 from lingo.pricing.utils import import_site as pricing_import_site
+from lingo.utils.wcs import WCSError
 
 klasses = {
     klass.application_component_type: klass
@@ -188,7 +189,10 @@ class ComponentDependencies(GenericAPIView):
                 return element
             return get_component_bundle_entry(request, element)
 
-        dependencies = [dependency_dict(x) for x in component.get_dependencies() if x]
+        try:
+            dependencies = [dependency_dict(x) for x in component.get_dependencies() if x]
+        except WCSError as e:
+            return Response({'err': 1, 'err_desc': str(e)}, status=400)
         return Response({'err': 0, 'data': dependencies})
 
 

lingo/invoicing/models.py

@@ -42,7 +42,13 @@ from lingo.agendas.chrono import ChronoError, lock_events_check
 from lingo.agendas.models import Agenda
 from lingo.utils.fields import RichTextField
 from lingo.utils.misc import LingoImportError, clean_import_data, generate_slug
-from lingo.utils.wcs import get_wcs_json, get_wcs_matching_card_model, get_wcs_services
+from lingo.utils.wcs import (
+    WCSError,
+    get_wcs_dependencies_from_template,
+    get_wcs_json,
+    get_wcs_matching_card_model,
+    get_wcs_services,
+)
 
 
 class RegieImportError(Exception):
@@ -117,12 +123,14 @@ class Payer(models.Model):
                 parts = self.carddef_reference.split(':')
                 wcs_key, card_slug = parts[:2]
                 wcs_site = get_wcs_services().get(wcs_key)
-                card_schema = get_wcs_json(wcs_site, 'api/cards/%s/@schema' % card_slug, log_errors='warn')
-
-                if not card_schema:
+                try:
+                    card_schema = get_wcs_json(
+                        wcs_site, 'api/cards/%s/@schema' % card_slug, log_errors='warn'
+                    )
+                except WCSError:
                     return
 
-                if card_schema.get('err') == 1:
+                if not card_schema:
                     return
 
                 self.cached_carddef_json = card_schema
@@ -135,7 +143,23 @@ class Payer(models.Model):
         return slugify(self.label)
 
     def get_dependencies(self):
-        return []
+        if self.carddef_reference:
+            parts = self.carddef_reference.split(':')
+            wcs_key, card_slug = parts[:2]
+            wcs_site_url = get_wcs_services().get(wcs_key)['url']
+            urls = {
+                'export': f'{wcs_site_url}api/export-import/cards/{card_slug}/',
+                'dependencies': f'{wcs_site_url}api/export-import/cards/{card_slug}/dependencies/',
+                'redirect': f'{wcs_site_url}api/export-import/cards/{card_slug}/redirect/',
+            }
+            yield {
+                'type': 'cards',
+                'id': card_slug,
+                'text': self.cached_carddef_json.get('name'),
+                'urls': urls,
+            }
+        yield from get_wcs_dependencies_from_template(self.payer_external_id_template)
+        yield from get_wcs_dependencies_from_template(self.payer_external_id_from_nameid_template)
 
     def export_json(self):
         return {

lingo/pricing/models.py

@@ -26,6 +26,7 @@ from django.utils.translation import gettext_lazy as _
 
 from lingo.agendas.models import Agenda, CheckType
 from lingo.utils.misc import LingoImportError, clean_import_data, generate_slug
+from lingo.utils.wcs import get_wcs_dependencies_from_template
 
 
 class PricingError(Exception):
@@ -330,6 +331,12 @@ class Pricing(models.Model):
     def get_dependencies(self):
         yield from self.agendas.all()
         yield from self.categories.all()
+        for value in sorted(self.extra_variables.values()):
+            yield from get_wcs_dependencies_from_template(value)
+        if self.kind == 'reduction':
+            yield from get_wcs_dependencies_from_template(self.reduction_rate)
+        if self.kind == 'effort':
+            yield from get_wcs_dependencies_from_template(self.effort_rate_target)
 
     def export_json(self):
         return {

lingo/utils/wcs.py

@@ -15,23 +15,40 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
+import re
 
 from django.conf import settings
+from django.utils.translation import gettext_lazy as _
 from requests.exceptions import RequestException
 
 from lingo.utils import requests
 
 
-def is_wcs_enabled(cls):
+class WCSError(Exception):
+    pass
+
+
+def is_wcs_enabled():
     return hasattr(settings, 'KNOWN_SERVICES') and settings.KNOWN_SERVICES.get('wcs')
 
 
 def get_wcs_services():
-    if not is_wcs_enabled(None):
+    if not is_wcs_enabled():
         return {}
     return settings.KNOWN_SERVICES.get('wcs')
 
 
+def get_default_wcs_service_key():
+    services = get_wcs_services()
+
+    for key, service in services.items():
+        if not service.get('secondary', False):
+            # if secondary is not set or not set to True, return this one
+            return key
+
+    return None
+
+
 def get_wcs_json(wcs_site, path, log_errors=True):
     if wcs_site is None:
         # no site specified (probably an import referencing a not yet deployed
@@ -52,8 +69,12 @@ def get_wcs_json(wcs_site, path, log_errors=True):
                 # return json if available (on 404 responses by example)
                 return e.response.json()
             except json.JSONDecodeError:
-                pass
-        return {'err': 1, 'data': None}
+                return {
+                    'err': 1,
+                    'err_desc': 'request-error-status-%s' % e.response.status_code,
+                    'data': None,
+                }
+        return {'err': 1, 'err_desc': 'request-error', 'data': None}
     return response.json()
 
 
@@ -84,3 +105,33 @@ def get_wcs_matching_card_model(ref):
     for carddef_reference, card_label in card_models:
         if carddef_reference == ref:
             return card_label
+
+
+def get_wcs_dependencies_from_template(string):
+    if not is_wcs_enabled():
+        return []
+    service_key = get_default_wcs_service_key()
+    wcs = get_wcs_services().get(service_key)
+    wcs_url = wcs.get('url') or ''
+    response_json = get_wcs_json(wcs, '/api/cards/@list')
+    if response_json.get('err') == 1:
+        raise WCSError(_('Unable to get WCS service (%s)') % response_json.get('err_desc'))
+    if not response_json.get('data'):
+        raise WCSError(_('Unable to get WCS data'))
+    carddef_labels_by_slug = {e['slug']: e['title'] for e in response_json['data']}
+    for carddef_slug in re.findall(r'cards\|objects:"([\w_-]+:?[\w_-]*)"', string):
+        if ':' in carddef_slug:
+            carddef_slug = carddef_slug.split(':')[0]
+        if carddef_slug not in carddef_labels_by_slug:
+            # ignore unknown card model
+            continue
+        yield {
+            'type': 'cards',
+            'id': carddef_slug,
+            'text': carddef_labels_by_slug[carddef_slug],
+            'urls': {
+                'export': f'{wcs_url}api/export-import/cards/{carddef_slug}/',
+                'dependencies': f'{wcs_url}api/export-import/cards/{carddef_slug}/dependencies/',
+                'redirect': f'{wcs_url}api/export-import/cards/{carddef_slug}/redirect/',
+            },
+        }

tests/api/test_applification_api.py

@@ -8,11 +8,14 @@ from unittest import mock
 import pytest
 from django.contrib.auth.models import Group
 from django.contrib.contenttypes.models import ContentType
+from requests.exceptions import ConnectionError
+from requests.models import Response
 
 from lingo.agendas.models import Agenda, CheckType, CheckTypeGroup
 from lingo.export_import.models import Application, ApplicationElement
 from lingo.invoicing.models import Payer, Regie
 from lingo.pricing.models import CriteriaCategory, Pricing
+from tests.invoicing.utils import MockedRequestResponse, mocked_requests_send
 
 pytestmark = pytest.mark.django_db
 
@@ -213,12 +216,17 @@ def test_export_minor_components(app, user):
     assert resp.json['data']['label'] == 'Foo Bar Regie'
 
 
-def test_pricing_dependencies(app, user):
+@mock.patch('requests.Session.send', side_effect=mocked_requests_send)
+def test_pricing_dependencies(mock_send, app, user):
     app.authorization = ('Basic', ('john.doe', 'password'))
     pricing = Pricing.objects.create(
         label='Foo Bar pricing',
         date_start=datetime.date(year=2021, month=9, day=1),
         date_end=datetime.date(year=2022, month=9, day=1),
+        extra_variables={
+            'foo': '{{ cards|objects:"card_model_1" }}',
+            'bar': '{{ cards|objects:"card_model_2:custom-view" }}',
+        },
     )
     category1 = CriteriaCategory.objects.create(label='Cat 1')
     category2 = CriteriaCategory.objects.create(label='Cat 2')
@@ -271,10 +279,139 @@ def test_pricing_dependencies(app, user):
                     'redirect': 'http://testserver/api/export-import/pricing_categories/cat-2/redirect/',
                 },
             },
+            {
+                'type': 'cards',
+                'id': 'card_model_1',
+                'text': 'Card Model 1',
+                'urls': {
+                    'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+                    'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+                    'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+                },
+            },
+            {
+                'type': 'cards',
+                'id': 'card_model_2',
+                'text': 'Card Model 2',
+                'urls': {
+                    'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+                    'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+                    'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+                },
+            },
         ],
         'err': 0,
     }
 
+    pricing.extra_variables = {}
+    pricing.reduction_rate = '{{ cards|objects:"card_model_1" }}'
+    pricing.effort_rate_target = '{{ cards|objects:"card_model_2:custom-view" }}'
+    pricing.save()
+    resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_1',
+        'text': 'Card Model 1',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+        },
+    } not in resp.json['data']
+    assert {
+        'type': 'cards',
+        'id': 'card_model_2',
+        'text': 'Card Model 2',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+        },
+    } not in resp.json['data']
+
+    pricing.kind = 'reduction'
+    pricing.save()
+    resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_1',
+        'text': 'Card Model 1',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+        },
+    } in resp.json['data']
+    assert {
+        'type': 'cards',
+        'id': 'card_model_2',
+        'text': 'Card Model 2',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+        },
+    } not in resp.json['data']
+
+    pricing.kind = 'effort'
+    pricing.save()
+    resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_1',
+        'text': 'Card Model 1',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+        },
+    } not in resp.json['data']
+    assert {
+        'type': 'cards',
+        'id': 'card_model_2',
+        'text': 'Card Model 2',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+        },
+    } in resp.json['data']
+
+    with mock.patch('requests.Session.get') as requests_get:
+        requests_get.side_effect = ConnectionError()
+        resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/', status=400)
+        assert resp.json['err'] == 1
+        assert resp.json['err_desc'] == 'Unable to get WCS service (request-error)'
+
+    with mock.patch('requests.Session.get') as requests_get:
+        mock_resp = Response()
+        mock_resp.status_code = 500
+        requests_get.return_value = mock_resp
+        resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/', status=400)
+        assert resp.json['err'] == 1
+        assert resp.json['err_desc'] == 'Unable to get WCS service (request-error-status-500)'
+
+    with mock.patch('requests.Session.get') as requests_get:
+        mock_resp = Response()
+        mock_resp.status_code = 404
+        requests_get.return_value = mock_resp
+        resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/', status=400)
+        assert resp.json['err'] == 1
+        assert resp.json['err_desc'] == 'Unable to get WCS service (request-error-status-404)'
+
+    with mock.patch('requests.Session.get') as requests_get:
+        requests_get.return_value = MockedRequestResponse(content=json.dumps({'foo': 'bar'}))
+        resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/', status=400)
+        assert resp.json['err'] == 1
+        assert resp.json['err_desc'] == 'Unable to get WCS data'
+
+    data = {'data': []}
+    with mock.patch('requests.Session.get') as requests_get:
+        requests_get.return_value = MockedRequestResponse(content=json.dumps(data))
+        resp = app.get('/api/export-import/pricings/foo-bar-pricing/dependencies/', status=400)
+        assert resp.json['err'] == 1
+        assert resp.json['err_desc'] == 'Unable to get WCS data'
+
 
 def test_agenda_dependencies(app, user):
     app.authorization = ('Basic', ('john.doe', 'password'))
@@ -355,15 +492,77 @@ def test_regie_dependencies(app, user):
     }
 
 
-def test_payer_dependencies(app, user):
+@mock.patch('requests.Session.send', side_effect=mocked_requests_send)
+def test_payer_dependencies(mock_send, app, user):
     app.authorization = ('Basic', ('john.doe', 'password'))
-    Payer.objects.create(label='Foo Bar Payer')
+    payer = Payer.objects.create(
+        label='Foo Bar Payer',
+    )
     resp = app.get('/api/export-import/payers/foo-bar-payer/dependencies/')
     assert resp.json == {
         'data': [],
         'err': 0,
     }
 
+    payer.carddef_reference = 'default:card_model_1'
+    payer.cached_carddef_json = {'name': 'Card Model 1'}
+    payer.save()
+    resp = app.get('/api/export-import/payers/foo-bar-payer/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_1',
+        'text': 'Card Model 1',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+        },
+    } in resp.json['data']
+
+    payer.carddef_reference = 'default:card_model_1:custom-view'
+    payer.save()
+    resp = app.get('/api/export-import/payers/foo-bar-payer/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_1',
+        'text': 'Card Model 1',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_1/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_1/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_1/redirect/',
+        },
+    } in resp.json['data']
+
+    payer.carddef_reference = ''
+    payer.payer_external_id_template = '{{ cards|objects:"card_model_2" }}'
+    payer.save()
+    resp = app.get('/api/export-import/payers/foo-bar-payer/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_2',
+        'text': 'Card Model 2',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+        },
+    } in resp.json['data']
+
+    payer.payer_external_id_template = ''
+    payer.payer_external_id_from_nameid_template = '{{ cards|objects:"card_model_2" }}'
+    payer.save()
+    resp = app.get('/api/export-import/payers/foo-bar-payer/dependencies/')
+    assert {
+        'type': 'cards',
+        'id': 'card_model_2',
+        'text': 'Card Model 2',
+        'urls': {
+            'export': 'http://wcs.example.org/api/export-import/cards/card_model_2/',
+            'dependencies': 'http://wcs.example.org/api/export-import/cards/card_model_2/dependencies/',
+            'redirect': 'http://wcs.example.org/api/export-import/cards/card_model_2/redirect/',
+        },
+    } in resp.json['data']
+
 
 def test_pricing_categories_dependencies(app, user):
     app.authorization = ('Basic', ('john.doe', 'password'))

tests/settings.py

@@ -46,11 +46,11 @@ KNOWN_SERVICES = {
     'wcs': {
         'default': {
             'title': 'test',
-            'url': 'http://example.org/',
+            'url': 'http://wcs.example.org/',
             'secret': 'lingo',
             'orig': 'lingo',
-            'backoffice-menu-url': 'http://example.org/manage/',
-        }
+            'backoffice-menu-url': 'http://wcs.example.org/manage/',
+        },
     },
 }