carl: implements Token authentication & sites references (#86683)

@@ -0,0 +132,4 @@
+        ),
+        methods=['get'],
+        parameters={
+            'type': {'description': _('Carl entitie type name'), 'example_value': 'wo'},

@@ -0,0 +138,4 @@
+                'description': _(
+                    'Comma separated list of fieldname pointing on foreign entitie we want data to be merged in the reply'
+                ),
+                'example_value': 'site, NMaddress',

@@ -0,0 +144,4 @@
+                'description': _(
+                    'When 1 given, add a carl_relationships field in reply containing a list of fields referencing another entitie'
+                ),
+                'example_value': '1',

@@ -0,0 +148,4 @@
+            },
+        },
+    )
+    def entitie(self, request, **kwargs):

@@ -0,0 +184,4 @@
+                data={'code': 'not-found', 'url': url},
+            )
+
+        if 'relationships' not in data['data']:

@@ -0,0 +281,4 @@
+                errors[fieldname] = _('Field %s allready in linked fields : cannot create')
+                continue
+            if len(errors):
+                continue  # do not attempt to create anything : allready failed

@@ -0,0 +380,4 @@
+            return {'data': {'status': _('No token in cache')}}
+
+        expiration = tok_infos[1]
+        ts = datetime.datetime.fromtimestamp(expiration, datetime.timezone.utc)

@@ -0,0 +390,4 @@
+
+    @property
+    def token_authentication(self):
+        '''Is True when token authentication wanted'''

@@ -0,0 +436,4 @@
+        TODO : handle paginated results
+        '''
+        filter_params = None
+        if entitie_id is not None:

@@ -0,0 +509,4 @@
+        - e_id : Entitie ID
+        '''
+        url = self.entitie_url(e_type, e_id)
+        try:

@@ -0,0 +595,4 @@
+
+        Handles common error cases.
+        Handles token authentication.
+        Defaults timeout to 2s.

@@ -0,0 +607,4 @@
+        '''
+        return self._json_request(True, http_method, url, **kwargs)
+
+    def _json_request(self, retry_auth, http_method, url, **kwargs):

@@ -0,0 +658,4 @@
+                return data
+        else:
+            err_fmt = _('Got an HTTP error code (%d) from Carl : %s')
+

@@ -0,0 +679,4 @@
+                    src = '(%s)' % src
+                else:
+                    src = ''
+                if error['title'] == error['detail']:

@@ -0,0 +1,1086 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2019 Entr'ouvert

@@ -0,0 +125,4 @@
+    service_url = models.URLField(
+        blank=False,
+        verbose_name=_('Service URL'),
+        help_text=_('Base webservice URL (such as https://example.tld/foo/bar/'),

@@ -0,0 +129,4 @@
+    )
+
+    carl_username = models.CharField(
+        max_length=128, verbose_name=_('Carl token authentication username'), blank=False

@@ -0,0 +132,4 @@
+        max_length=128, verbose_name=_('Carl token authentication username'), blank=False
+    )
+    carl_password = models.CharField(
+        max_length=128, verbose_name=_('Carl token authentication password'), blank=False

@@ -0,0 +143,4 @@
+    @endpoint(
+        name='entity',
+        pattern=r'^by_id$',
+        example_pattern='entity/by_id',

@@ -0,0 +175,4 @@
+                },
+            },
+            'properties': {
+                'carl_relationships': {

@@ -0,0 +183,4 @@
+        },
+    )
+    def entity_by_id(self, request, **kwargs):
+        missing = [param for param in ('type', 'id') if param not in kwargs]

@@ -0,0 +184,4 @@
+    )
+    def entity_by_id(self, request, **kwargs):
+        missing = [param for param in ('type', 'id') if param not in kwargs]
+        if len(missing):

@@ -0,0 +191,4 @@
+
+        carl_type = kwargs.pop('type')
+        carl_id = kwargs.pop('id')
+        want_relationships = kwargs.pop('relationships', False)

@@ -0,0 +193,4 @@
+        carl_id = kwargs.pop('id')
+        want_relationships = kwargs.pop('relationships', False)
+        related = kwargs.pop('related', '')
+        related = [rel for rel in (sp.strip() for sp in related.split(',')) if len(rel) > 0]

@@ -0,0 +195,4 @@
+        related = kwargs.pop('related', '')
+        related = [rel for rel in (sp.strip() for sp in related.split(',')) if len(rel) > 0]
+
+        if len(kwargs):  # unexpected extra parameters

@@ -0,0 +211,4 @@
+
+    @endpoint(
+        name='entity',
+        pattern=r'^by_values$',

@@ -0,0 +296,4 @@
+
+    @endpoint(
+        name='entity',
+        pattern='^create$',

@@ -0,0 +311,4 @@
+                },
+            },
+            'properties': {
+                'carl_id': {'type': 'string'},

@@ -0,0 +316,4 @@
+            },
+        },
+    )
+    def create_entity_endpoint(self, request, post_data):

@@ -0,0 +436,4 @@
+            )
+
+        missing = [field for field in ('id', 'type', 'attributes') if field not in entity['data']]
+        if len(missing) > 0:

@@ -0,0 +526,4 @@
+        )
+
+    @endpoint(
+        name='token_status',

@@ -0,0 +588,4 @@
+
+        Returns a dict suitable to be returned by an endpoint
+        '''
+        related = [] if related is None else related

@@ -0,0 +654,4 @@
+
+        Returns: an external reference in Publik format (
+        https://doc-publik.entrouvert.com/dev/wcs/api-webservices/acces-a-des-referentiels-externe/ )
+        TODO: handle paginated results

@@ -0,0 +695,4 @@
+            {'text': ent2text(ent), 'id': ent2id(ent), 'carl_id': ent['id']}
+            for ent in data['data']
+            if len(query) == 0 or query in ent2text(ent).lower()
+        ]

@@ -0,0 +708,4 @@
+
+        Returns: a dict with 'data' being a list of Carl entities
+
+        NOTE : TODO handle paginated results (if too many entities returned)

@@ -0,0 +722,4 @@
+        '''
+        url = self.entity_url(entity['data']['type'])
+        headers = {'Content-Type': 'application/vnd.api+json'}
+        data = json.dumps(entity)

@@ -0,0 +787,4 @@
+    def filters_to_params(filters):
+        '''Return a dict suitable for requests params argument given filters
+        Arguments:
+        - filters: A dict with fieldname as key and wanted value as value

@@ -0,0 +798,4 @@
+        '''Forge an entity URL of the form
+        SERVICE_URL/api/entities/v1/[entity_type[/entity_id[/related_fieldname]]]
+        '''
+        url = urllib.parse.urljoin(self.service_url, 'api/entities/v1/')

@@ -0,0 +812,4 @@
+                break
+        return url
+
+    def filters_from_qsdict(self, qsdict):

@@ -0,0 +142,4 @@
+
+    @endpoint(
+        name='entity',
+        methods=['post'],

@@ -0,0 +180,4 @@
+                _('Error with specified linked fields: %s')
+                % (', '.join(['%s (%s)' % elt for elt in errors.items()])),
+                http_status=400,
+                data={'code': 'creation-error'},

@@ -0,0 +217,4 @@
+            raise APIError(
+                _('Error looking for linked fields: %s')
+                % (', '.join(['%r (%s)' % elt for elt in errors.items()])),
+                data={'code': 'creation-error'},

@@ -0,0 +255,4 @@
+                'errors': '. '.join([_('Error with field %s (%s)') % e for e in errors.items()]),
+            }
+            if cleanup_err is not None:
+                err_msg += '. %s' % str(cleanup_err)

@@ -0,0 +275,4 @@
+            data = expt.data
+            data.update(cleanup_err.data)
+            raise APIError(
+                _('Error creating entity: %s. Then error during related entities cleanup: %s')

@@ -0,0 +281,4 @@
+                data=data,
+            )
+
+        missing = [field for field in ('id', 'type', 'attributes') if field not in entity['data']]

@@ -0,0 +315,4 @@
+            },
+            'id': {
+                'description': _('Only return the entity with corresponding ID'),
+            },

@@ -0,0 +356,4 @@
+            )
+            result = [result]
+        else:
+            if q is not None:

@@ -0,0 +367,4 @@
+            entities = self.filter_entities(carl_type, filters, filters_like)
+            result = [self.entity_data(ent) for ent in entities['data']]
+
+        text_attrname = 'description' if text_attrname is None else text_attrname

@@ -0,0 +368,4 @@
+            result = [self.entity_data(ent) for ent in entities['data']]
+
+        text_attrname = 'description' if text_attrname is None else text_attrname
+        result = [{**ent, 'text': ent['attributes'].get(text_attrname)} for ent in result]

@@ -0,0 +399,4 @@
+
+        Arguments:
+        - entity_type: The name of the entity type
+        - filters: A dict with fieldname as key and wanted value as value. Values

@@ -0,0 +410,4 @@
+               will not be complete (only first "page" returned)
+        '''
+        url = self.entity_url(entity_type)
+        params = {'filter[%s]' % fname: fval for fname, fval in filters_eq.items()}

@@ -0,0 +411,4 @@
+        '''
+        url = self.entity_url(entity_type)
+        params = {'filter[%s]' % fname: fval for fname, fval in filters_eq.items()}
+        if filters_like is not None:

@@ -0,0 +423,4 @@
+        - related: str ',' separated list of related entities we want data to be fetched
+        - relationships: bool If True add a list of related entities field names
+        '''
+        related = [rel for rel in (sp.strip() for sp in related.split(',')) if rel]

@@ -0,0 +441,4 @@
+            if relattr not in relations:
+                missing.append(relattr)
+                continue
+            if missing:

@@ -0,0 +449,4 @@
+                result['related'] = {}
+            result['related'][relattr] = self.entity_data(rel_data['data'])
+
+        if missing:

@@ -0,0 +463,4 @@
+            )
+        return result
+
+    def entity_data(self, entity):

@@ -0,0 +501,4 @@
+            )
+        if not response.ok:
+            raise self.carlerror_to_apierror(
+                _('HTTP error (%%(status)d) during entitie (%s/%s) deletion: %%(detail)s') % (e_type, e_id),

@@ -0,0 +519,4 @@
+            except APIError as expt:
+                expts.append(expt)
+                not_cleaned.append('%s(%s)' % (e_type, e_id))
+                continue

@@ -0,0 +552,4 @@
+                url = urllib.parse.urljoin(url + '/', urllib.parse.quote(str(part)))
+            else:
+                break
+        return url

@@ -0,0 +627,4 @@
+        Returns: a dict (with 'data' key present) representing decoded JSON
+                  response from Carl
+        '''
+        return self._json_request(True, http_method, url, **kwargs)

@@ -0,0 +676,4 @@
+        if response.status_code in (401, 403) and retry_auth and self.token_authentication:
+            # handling early/unexpected token expiration
+            self.invalidate_token()
+            self._json_request(False, http_method, url, **kwargs)

@@ -0,0 +688,4 @@
+        '''When needed modify given headers by reference adding authentication token
+        Arguments:
+        - headers A dict with HTTP headers or None
+        Returns: The modificated dict or a new instance with authentication token

@@ -0,0 +707,4 @@
+        token_data = cache.get(key)
+        if token_data is not None:
+            token, expiration = token_data
+            if expiration <= utcnow_ts():

@@ -0,0 +712,4 @@
+        if token_data is None:
+            token, expires_in = self.fetch_token()
+            expiration = utcnow_ts() + expires_in
+            cache.set(key, (token, expiration), expires_in)

@@ -0,0 +648,4 @@
+        '''Invalidate token in cache'''
+        cache.delete(self.token_cache_key)
+
+    def fetch_token(self):

@@ -0,0 +662,4 @@
+            response = self.requests.post(url, data=data, timeout=2)
+        except requests.RequestException as expt:
+            raise APIError(
+                _('Unable to send authentication request to Carl: %s') % exception_to_text(expt),

@@ -0,0 +695,4 @@
+            )
+
+        err_fmt = _('Got an invalid token when authentication on Carl: %s')
+        missing = []

@@ -0,0 +13,4 @@
+from passerelle.base.models import AccessRight, ApiUser
+
+APIERROR_CLS = 'passerelle.utils.jsonresponse.APIError'
+SERVICE_URL = 'http://test.carl.notld/'