passerelle/apps/api_impot_particulier/migrations/0001_initial.py

@@ -0,0 +1,56 @@
+# Generated by Django 3.2.18 on 2023-04-14 17:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ('base', '0030_resourcelog_base_resour_appname_298cbc_idx'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Resource',
+            fields=[
+                (
+                    'id',
+                    models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+                ),
+                ('title', models.CharField(max_length=50, verbose_name='Title')),
+                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
+                ('description', models.TextField(verbose_name='Description')),
+                (
+                    'api_url',
+                    models.URLField(
+                        default='https://gwfc.dgfip.finances.gouv.fr/',
+                        max_length=256,
+                        verbose_name='DGFIP API base URL',
+                    ),
+                ),
+                ('oauth_username', models.CharField(max_length=128, verbose_name='DGFIP API Username')),
+                ('oauth_password', models.CharField(max_length=128, verbose_name='DGFIP API Password')),
+                (
+                    'oauth_scopes',
+                    models.CharField(max_length=128, verbose_name='DGFIP API Scopes', blank=True),
+                ),
+                (
+                    'id_teleservice',
+                    models.TextField(max_length=128, verbose_name='DGFIP API ID_Teleservice', blank=True),
+                ),
+                (
+                    'users',
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name='_api_impot_particulier_resource_users_+',
+                        related_query_name='+',
+                        to='base.ApiUser',
+                    ),
+                ),
+            ],
+            options={
+                'verbose_name': 'API Impot Particulier',
+            },
+        ),
+    ]

passerelle/apps/api_impot_particulier/models.py

@@ -0,0 +1,273 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2023  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import datetime
+import hashlib
+import uuid
+from urllib.parse import urljoin
+
+import requests
+from django.core.cache import cache
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+from passerelle.utils.jsonresponse import APIError
+from passerelle.utils.timeout import Timeout
+
+
+class ServiceIsDown(APIError):
+    def __init__(self):
+        super().__init__(_('API Impot Particulier service is unavailable'))
+
+    def __str__(self):
+        if self.__context__:
+            return f'{super().__str__()}: {self.__context__}'
+        return super().__str__()
+
+
+class Resource(BaseResource):
+    api_url = models.URLField(
+        _('DGFIP API base URL'),
+        max_length=256,
+        default='https://gw.dgfip.finances.gouv.fr/impotparticulier/1.0',
+    )
+    oauth_username = models.CharField(_('DGFIP API Username'), max_length=128)
+    oauth_password = models.CharField(_('DGFIP API Password'), max_length=128)
+    oauth_scopes = models.CharField(_('DGFIP API Scopes'), max_length=128, blank=True)
+    id_teleservice = models.TextField(_('DGFIP API ID_Teleservice'), max_length=128, blank=True)
+
+    log_requests_errors = False
+    requests_timeout = 30
+    requests_max_retries = {
+        'total': 3,
+        'backoff_factor': 0.5,
+        'allowed_methods': ['GET', 'POST'],
+        # retry after: 0.5, 1.5 and 3.5 seconds
+        'status_forcelist': [413, 429, 503, 504],
+    }
+
+    category = _('Business Process Connectors')
+
+    class Meta:
+        verbose_name = _('API Impot Particulier')
+
+    @classmethod
+    def parse_numero_fiscal(cls, value):
+        value = value.strip().replace(' ', '')
+        if not (value and value.isascii() and value.isdigit()):
+            raise APIError(_('invalid numero_fiscal'))
+        return value
+
+    @classmethod
+    def parse_annee_de_revenu(cls, value):
+        try:
+            value = int(value)
+        except (TypeError, ValueError):
+            raise APIError(_('invalid annee_de_revenu'))
+        today = datetime.date.today()
+        if not (0 < today.year - value < 10):
+            raise APIError(_('invalid annee_de_revenu'))
+        return value
+
+    @endpoint(
+        name='spi-situations-ir-assiettes-annrev',
+        description=_('Provides revenue tax situation for a specific year.'),
+        parameters={
+            'numero_fiscal': {
+                'description': _('Tax number of the person'),
+            },
+            'annee_de_revenu': {
+                'description': _('Income year'),
+            },
+        },
+    )
+    def spi_situations_ir_assiettes_annrev(self, request, numero_fiscal, annee_de_revenu):
+        numero_fiscal = self.parse_numero_fiscal(numero_fiscal)
+        annee_de_revenu = self.parse_annee_de_revenu(annee_de_revenu)
+        return {
+            'data': self.get_spi_situations_ir_assiettes_annrev(
+                numero_fiscal=numero_fiscal, annee_de_revenu=annee_de_revenu, timeout=Timeout(20)
+            )
+        }
+
+    def get_spi_situations_ir_assiettes_annrev(self, numero_fiscal, annee_de_revenu, timeout=None):
+        return self.call(
+            name='spi-situations-ir-assiettes-deuxans',
+            endpoint_template='spi/{spi}/situations/ir/assiettes/annrev/{annrev}',
+            timeout=timeout,
+            spi=numero_fiscal,
+            annrev=annee_de_revenu,
+        )
+
+    def call(self, name, endpoint_template, timeout=None, **kwargs):
+        correlation_id = str(uuid.uuid4().hex)
+        kwargs_formatted = ', '.join(f'{key}={value}' for key, value in kwargs.items())
+        try:
+            data = self.get_tax_data(
+                session=self.requests,
+                base_url=self.api_url,
+                access_token=self._get_access_token(timeout=timeout),
+                correlation_id=correlation_id,
+                endpoint_template=endpoint_template,
+                id_teleservice=self.id_teleservice,
+                timeout=timeout,
+                **kwargs,
+            )
+        except ServiceIsDown as e:
+            self.logger.warning(
+                '%s(%s) failed: %s',
+                name,
+                kwargs_formatted,
+                e,
+                extra={
+                    'correlation_id': correlation_id,
+                    'id_teleservice': self.id_teleservice,
+                    'kwargs': kwargs,
+                },
+            )
+            raise
+        else:
+            self.logger.warning(
+                '%s(%s) success',
+                name,
+                kwargs_formatted,
+                extra={
+                    'data': data,
+                    'correlation_id': correlation_id,
+                    'id_teleservice': self.id_teleservice,
+                    'kwargs': kwargs,
+                },
+            )
+        return data
+
+    @classmethod
+    def get_tax_data(
+        cls,
+        session,
+        base_url,
+        access_token,
+        correlation_id,
+        endpoint_template,
+        id_teleservice=None,
+        headers=None,
+        timeout=None,
+        **kwargs,
+    ):
+        headers = {
+            **(headers or {}),
+            'Authorization': f'Bearer {access_token}',
+            'X-Correlation-ID': correlation_id,
+            'Accept': 'application/prs.dgfip.part.situations.ir.assiettes.v1+json',
+        }
+        if id_teleservice:
+            headers['ID_Teleservice'] = id_teleservice
+
+        endpoint = endpoint_template.format(**kwargs)
+        if not base_url.endswith('/'):
+            base_url += '/'
+        url = urljoin(base_url, endpoint)
+
+        if timeout is not None:
+            timeout = float(timeout)
+
+        # api-impot-particulier error reporting is byzantine, some errors are
+        # accompanied by a 4xx code, some others with a 20x code, some have a
+        # JSON content, other are only identified by a codeapp header on
+        # the response
+        try:
+            response = session.get(url, headers=headers, timeout=timeout)
+            response.raise_for_status()
+        except requests.HTTPError:
+            try:
+                content = response.json()['erreur']
+            except (ValueError, KeyError):
+                try:
+                    raise APIError(
+                        'api-impot-particulier error', data={'codeapp': response.headers['codeapp']}
+                    )
+                except KeyError:
+                    pass
+                raise ServiceIsDown
+            raise APIError('api-impot-particulier-error', data=content)
+        except requests.RequestException:
+            raise ServiceIsDown
+
+        if response.status_code != 200:
+            try:
+                content = response.json()['erreur']
+            except (ValueError, KeyError):
+                try:
+                    raise APIError(
+                        'api-impot-particulier-error', data={'codeapp': response.headers['codeapp']}
+                    )
+                except KeyError:
+                    raise ServiceIsDown
+            raise APIError('api-impot-particulier error', data=content)
+
+        try:
+            response_data = response.json()
+        except ValueError:
+            raise ServiceIsDown
+        return response_data
+
+    def _get_access_token(self, timeout=None):
+        key = (
+            'dgfip-at-'
+            + hashlib.sha256(
+                f'{self.oauth_username}-{self.oauth_password}-{self.api_url}'.encode()
+            ).hexdigest()
+        )
+
+        access_token = cache.get(key)
+        if not access_token:
+            access_token = self.get_access_token(
+                session=self.requests,
+                base_url=self.api_url,
+                username=self.oauth_username,
+                password=self.oauth_password,
+                scope=self.oauth_scopes,
+                timeout=timeout,
+            )
+            cache.set(key, access_token, 300)
+        return access_token
+
+    @classmethod
+    def get_access_token(cls, session, base_url, username, password, scope, timeout=None):
+        data = {
+            'grant_type': 'client_credentials',
+        }
+        if scope:
+            data['scope'] = scope
+
+        url = urljoin(base_url, '/token')
+
+        if timeout is not None:
+            timeout = float(timeout)
+
+        try:
+            response = session.post(url, data=data, auth=(username, password), timeout=timeout)
+            response.raise_for_status()
+        except requests.RequestException:
+            raise ServiceIsDown
+        try:
+            response_data = response.json()
+            access_token = response_data['access_token']
+            response_data = response.json()
+        except (ValueError, KeyError, TypeError):
+            raise ServiceIsDown
+        return access_token

passerelle/settings.py

@@ -129,6 +129,7 @@ INSTALLED_APPS = (
     'passerelle.apps.airquality',
     'passerelle.apps.api_entreprise',
     'passerelle.apps.api_particulier',
+    'passerelle.apps.api_impot_particulier',
     'passerelle.apps.arcgis',
     'passerelle.apps.arpege_ecp',
     'passerelle.apps.astech',

passerelle/utils/timeout.py

@@ -0,0 +1,28 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2023  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import time
+
+
+class Timeout:
+    '''Wall-time diminshing timeout'''
+
+    def __init__(self, timeout):
+        self.timeout = timeout
+        self.start = time.time()
+
+    def __float__(self):
+        return float(max(self.timeout - (time.time() - self.start), 0.1))

tests/api_impot_particulier/test_misc.py

@@ -0,0 +1,213 @@
+# Passerelle
+
+import datetime
+import re
+from unittest import mock
+
+import pytest
+import requests
+import responses
+
+from passerelle.apps.api_impot_particulier.models import Resource
+from passerelle.utils.jsonresponse import APIError
+
+from ..utils import make_resource
+
+last_year = str(datetime.date.today().year - 1)
+
+
+@responses.activate
+def test_get_access_token_with_scope():
+    responses.add(responses.POST, 'https://dgfip/token', json={'access_token': '1234'})
+
+    assert (
+        Resource.get_access_token(requests, 'https://dgfip/impotparticulier/1.0', 'user', 'pass', 'scope')
+        == '1234'
+    )
+    assert len(responses.calls) == 1
+    request = responses.calls[0].request
+    assert request.body == 'grant_type=client_credentials&scope=scope'
+    assert request.headers['authorization'] == 'Basic dXNlcjpwYXNz'
+
+
+@responses.activate
+def test_get_access_token_without_scope():
+    responses.add(responses.POST, 'https://dgfip/token', json={'access_token': '1234'})
+    assert (
+        Resource.get_access_token(requests, 'https://dgfip/impotparticulier/1.0', 'user', 'pass', '')
+        == '1234'
+    )
+    assert len(responses.calls) == 1
+    assert responses.calls[0].request.body == 'grant_type=client_credentials'
+
+
+@responses.activate
+def test_get_access_token_connection_timeout():
+    responses.add(responses.POST, 'https://dgfip/impotparticulier/1.0', body=requests.ConnectTimeout())
+    with pytest.raises(APIError, match=r'.*is unavailable'):
+        Resource.get_access_token(requests, 'https://dgfip/', 'user', 'pass', 'scope')
+
+
+@responses.activate
+def test_get_tax_data():
+    responses.add(responses.GET, 'https://dgfip/impotparticulier/1.0/123/truc/xyz/', json={'data': 1})
+
+    assert Resource.get_tax_data(
+        requests,
+        'https://dgfip/impotparticulier/1.0',
+        access_token='1234',
+        correlation_id='abcd',
+        endpoint_template='{var1}/truc/{var2}/',
+        var1='123',
+        var2='xyz',
+    ) == {'data': 1}
+
+    assert len(responses.calls) == 1
+    request = responses.calls[0].request
+    assert request.headers['authorization'] == 'Bearer 1234'
+    assert request.headers['x-correlation-id'] == 'abcd'
+    assert request.headers['Accept'] == 'application/prs.dgfip.part.situations.ir.assiettes.v1+json'
+
+
+@responses.activate
+def test_get_tax_data_connection_timeout():
+    responses.add(
+        responses.GET, 'https://dgfip/impotparticulier/1.0/123/truc/xyz/', body=requests.ConnectTimeout()
+    )
+
+    with pytest.raises(APIError, match=r'.*is unavailable'):
+        assert Resource.get_tax_data(
+            requests,
+            'https://dgfip/impotparticulier/1.0',
+            access_token='1234',
+            correlation_id='abcd',
+            endpoint_template='{var1}/truc/{var2}/',
+            var1='123',
+            var2='xyz',
+        ) == {'data': 1}
+
+
+@responses.activate
+@pytest.mark.parametrize(
+    'responses_kwargs,response',
+    [
+        ({'json': {'erreur': {'message': 'foobar'}}, 'status': 403}, {'message': 'foobar'}),
+        ({'json': {'erreur': {'message': 'foobar'}}, 'status': 204}, {'message': 'foobar'}),
+        ({'headers': {'codeapp': 'foobar'}, 'status': 403}, {'codeapp': 'foobar'}),
+        ({'headers': {'codeapp': 'foobar'}, 'status': 204}, {'codeapp': 'foobar'}),
+    ],
+)
+def test_get_tax_data_errors(responses_kwargs, response):
+    responses.add(
+        responses.GET,
+        'https://dgfip/impotparticulier/1.0/123/truc/xyz/',
+        **responses_kwargs,
+    )
+
+    with pytest.raises(APIError) as ei:
+        Resource.get_tax_data(
+            requests,
+            'https://dgfip/impotparticulier/1.0',
+            access_token='1234',
+            correlation_id='abcd',
+            endpoint_template='{var1}/truc/{var2}/',
+            var1='123',
+            var2='xyz',
+        )
+    assert ei.value.data == response
+
+
+@pytest.fixture
+def resource(db):
+    return make_resource(
+        Resource,
+        slug='test',
+        title='API Impot Particulier',
+        description='API Impot Particulier',
+        oauth_username='user',
+        oauth_password='pass',
+        oauth_scopes='scope',
+    )
+
+
+class TestGetMethods:
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        with responses._default_mock:
+            with mock.patch('passerelle.apps.api_impot_particulier.models.uuid') as uuid_mock:
+                uuid_mock.uuid4.return_value.hex = 'abcd'
+                responses.add(
+                    responses.POST, 'https://gw.dgfip.finances.gouv.fr/token', json={'access_token': '1234'}
+                )
+                yield None
+
+    def test_spi_situations_ir_assiettes_annrev(self, resource):
+        responses.add(
+            responses.GET,
+            'https://gw.dgfip.finances.gouv.fr/impotparticulier/1.0/spi/1234/situations/ir/assiettes/annrev/2023',
+            json={'data': 2},
+        )
+        assert resource.get_spi_situations_ir_assiettes_annrev(
+            numero_fiscal='1234', annee_de_revenu='2023'
+        ) == {'data': 2}
+        assert len(responses.calls) <= 2
+        if len(responses.calls) == 2:
+            request = responses.calls[0].request
+            assert request.body == 'grant_type=client_credentials&scope=scope'
+            assert request.headers['authorization'] == 'Basic dXNlcjpwYXNz'
+        request = responses.calls[-1].request
+        assert request.headers['authorization'] == 'Bearer 1234'
+        assert request.headers['x-correlation-id'] == 'abcd'
+        assert request.headers['Accept'] == 'application/prs.dgfip.part.situations.ir.assiettes.v1+json'
+
+    def test_spi_situations_connection_error(self, resource):
+        responses.add(
+            responses.GET,
+            re.compile('https://gw.dgfip.finances.gouv.fr/.*'),
+            body=requests.ConnectTimeout(),
+        )
+        with pytest.raises(APIError, match=r'.*is unavailable'):
+            resource.get_spi_situations_ir_assiettes_annrev(numero_fiscal='1234', annee_de_revenu='2023')
+
+
+class TestEndpoints:
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        with responses._default_mock:
+            with mock.patch('passerelle.apps.api_impot_particulier.models.uuid') as uuid_mock:
+                uuid_mock.uuid4.return_value.hex = 'abcd'
+                responses.add(
+                    responses.POST, 'https://gw.dgfip.finances.gouv.fr/token', json={'access_token': '1234'}
+                )
+                yield None
+
+    @mock.patch(
+        'passerelle.apps.api_impot_particulier.models.Resource.get_spi_situations_ir_assiettes_annrev'
+    )
+    def test_spi_situations_ir_assiettes_annrev(self, mock_get, app, resource):
+        mock_get.return_value = 'ok'
+        response = app.get(
+            '/api-impot-particulier/test/spi-situations-ir-assiettes-annrev',
+            params={'numero_fiscal': '1234', 'annee_de_revenu': last_year},
+        )
+        assert mock_get.call_args[1]['numero_fiscal'] == '1234'
+        assert response.json == {'data': 'ok', 'err': 0}
+
+    @mock.patch(
+        'passerelle.apps.api_impot_particulier.models.Resource.get_spi_situations_ir_assiettes_annrev'
+    )
+    def test_spi_situations_ir_assiettes_annrev_strip(self, mock_get, app, resource):
+        mock_get.return_value = 'ok'
+        response = app.get(
+            '/api-impot-particulier/test/spi-situations-ir-assiettes-annrev',
+            params={'numero_fiscal': ' 1 2 3 4 ', 'annee_de_revenu': last_year},
+        )
+        assert mock_get.call_args[1]['numero_fiscal'] == '1234'
+        assert response.json == {'data': 'ok', 'err': 0}
+
+    def test_spi_situations_ir_assiettes_annrev_invalid(self, app, resource):
+        response = app.get(
+            '/api-impot-particulier/test/spi-situations-ir-assiettes-annrev',
+            params={'numero_fiscal': '1234', 'annee_de_revenu': '2000'},
+        )
+        assert response.json['err'] == 1