api: keep local cache of API clients from idp (#88697)

tests/api/test_formdata.py

@@ -3128,6 +3128,14 @@ def test_api_ods_formdata(pub, local_user, user, auth):
     formdef.store()
     get_url('/api/forms/test/ods', status=200)
 
+    if user == 'idp-api-client':
+        # check a single api access object has been created
+        assert ApiAccess.count() == 1
+        api_access = ApiAccess.select()[0]
+        assert api_access.idp_api_client
+        assert api_access.access_identifier == '_idp_test'
+        assert api_access.access_key is None
+
 
 def test_api_global_geojson(pub, local_user):
     pub.role_class.wipe()

wcs/admin/api_access.py

@@ -201,7 +201,7 @@ class ApiAccessDirectory(Directory):
             templates=['wcs/backoffice/api_accesses.html'],
             context={
                 'view': self,
-                'api_accesses': ApiAccess.select(order_by='name'),
+                'api_accesses': [x for x in ApiAccess.select(order_by='name') if not x.idp_api_client],
                 'api_manage_url': api_manage_url,
             },
         )

wcs/api_access.py

@@ -33,6 +33,7 @@ class ApiAccess(XmlStorableObject):
     access_key = None
     description = None
     restrict_to_anonymised_data = False
+    idp_api_client = False
     _roles = None
     _role_ids = Ellipsis
 
@@ -44,6 +45,7 @@ class ApiAccess(XmlStorableObject):
         ('access_key', 'str'),
         ('restrict_to_anonymised_data', 'bool'),
         ('roles', 'roles'),
+        ('idp_api_client', 'bool'),
     ]
 
     @classmethod
@@ -75,7 +77,6 @@ class ApiAccess(XmlStorableObject):
             id = Ellipsis  # make sure it fails all over the place if used
             is_admin = False
             is_api_user = True
-            restrict_to_anonymised_data = False
 
             def __init__(self, api_access):
                 self.api_access = api_access
@@ -99,7 +100,7 @@ class ApiAccess(XmlStorableObject):
     @classmethod
     def get_with_credentials(cls, username, password):
         api_access = cls.get_by_identifier(username)
-        if not api_access or api_access.access_key != password:
+        if not api_access or api_access.access_key != password or api_access.idp_api_client:
             api_access = cls.get_from_idp(username, password)
             if not api_access:
                 raise KeyError
@@ -144,11 +145,18 @@ class ApiAccess(XmlStorableObject):
         if data.get('err', 1) != 0:
             return None
 
-        api_access = cls.volatile()
+        # cache api client locally, it is necessary for serialization for afterjobs
+        # in uwsgi spooler.
+        access_identifier = f'_idp_{username}'
+        api_access = cls.get_by_identifier(access_identifier) or cls()
+        api_access.idp_api_client = True
+        api_access.access_identifier = access_identifier
+        role_class = get_publisher().role_class
         try:
             api_access.restrict_to_anonymised_data = data['data']['restrict_to_anonymised_data']
-            api_access._role_ids = data['data']['roles']
+            api_access.roles = [role_class.get(x, ignore_errors=True) for x in data['data']['roles']]
+            api_access.roles = [x for x in api_access.roles if x is not None]
         except KeyError:
             return None
-
+        api_access.store()
         return api_access

wcs/backoffice/management.py

@@ -4566,7 +4566,13 @@ class CsvExportAfterJob(AfterJob):
     label = _('Exporting to CSV file')
 
     def __init__(self, formdef, **kwargs):
-        kwargs.update(self.serialize_user(kwargs.pop('user', None)))
+        user = kwargs.pop('user', None)
+        if user and user.is_api_user:
+            kwargs['user_is_api_user'] = True
+            kwargs['user_id'] = user.api_access.id
+        else:
+            kwargs['user_is_api_user'] = False
+            kwargs['user_id'] = user.id if user else None
         super().__init__(formdef_class=formdef.__class__, formdef_id=formdef.id, **kwargs)
         self.file_name = '%s.csv' % formdef.url_name
 
@@ -4637,7 +4643,10 @@ class CsvExportAfterJob(AfterJob):
         query = self.kwargs['query']
         criterias = self.kwargs['criterias']
         order_by = self.kwargs['order_by']
-        user = self.unserialize_user(self.kwargs)
+        if self.kwargs['user_is_api_user']:
+            user = ApiAccess.get(self.kwargs['user_id']).get_as_api_user()
+        else:
+            user = get_publisher().user_class.get(self.kwargs['user_id'])
 
         items, total_count = FormDefUI(formdef).get_listing_items(
             fields,

wcs/qommon/afterjobs.py

@@ -84,30 +84,6 @@ class AfterJob(StorableObject):
     def done_action_attributes(self):
         return self.done_button_attributes_arg
 
-    def serialize_user(self, user):
-        obj = {}
-        if user and user.is_api_user:
-            obj['user_is_api_user'] = True
-            # serialize as a volatile object to match idp api clients
-            obj['user_role_ids'] = list(user.roles)
-            obj['user_restrict_to_anonymised_data'] = user.restrict_to_anonymised_data
-        else:
-            obj['user_is_api_user'] = False
-            obj['user_id'] = user.id if user else None
-        return obj
-
-    def unserialize_user(self, obj):
-        if obj['user_is_api_user']:
-            from wcs.api_access import ApiAccess
-
-            api_access = ApiAccess.volatile()
-            api_access._role_ids = obj['user_role_ids']
-            api_access.restrict_to_anonymised_data = obj['user_restrict_to_anonymised_data']
-            user = api_access.get_as_api_user()
-        else:
-            user = get_publisher().user_class.get(obj['user_id'])
-        return user
-
     def increment_count(self, amount=1):
         self.current_count = (self.current_count or 0) + amount
         # delay storage to avoid repeated writes on slow storage

wcs/templates/wcs/backoffice/api_access.html

@@ -3,10 +3,12 @@
 {% block body %}
   <div id="appbar">
     <h2>{% trans "API access" %} - {{ api_access.name }}</h2>
-    <span class="actions">
-      <a href="delete" rel="popup">{% trans "Delete" %}</a>
-      <a href="edit">{% trans "Edit" %}</a>
-    </span>
+    {% if not api_access.idp_api_client %}
+      <span class="actions">
+        <a href="delete" rel="popup">{% trans "Delete" %}</a>
+        <a href="edit">{% trans "Edit" %}</a>
+      </span>
+    {% endif %}
   </div>
 
   {% if api_access.description %}
@@ -16,8 +18,12 @@
   <div class="bo-block">
     <h3>{% trans "Parameters" %}</h3>
     <ul>
-      <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
-      <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% if not api_access.idp_api_client %}
+        <li>{% trans "Access identifier:" %} {{ api_access.access_identifier }}</li>
+        <li>{% trans "Access key:" %} {{ api_access.access_key }}</li>
+      {% else %}
+        <li>{% trans "API client from identity provider, identifier:" %} {{ api_access.access_identifier|removeprefix:"_idp_" }}</li>
+      {% endif %}
       {% if api_access.restrict_to_anonymised_data %}<li>{% trans "Restricted to anonymised data" %}</li>{% endif %}
       {% if api_access.get_roles %}
         <li>{% trans "Roles:" %}