misc: change django-upgrade target version to 3.2 (#75442)

.pre-commit-config.yaml

@@ -7,10 +7,10 @@ repos:
     - id: pyupgrade
       args: ['--keep-percent-format', '--py37-plus']
 -   repo: https://github.com/adamchainz/django-upgrade
-    rev: 1.10.0
+    rev: 1.13.0
     hooks:
     - id: django-upgrade
-      args: ['--target-version', '2.2']
+      args: ['--target-version', '3.2']
 -   repo: https://github.com/psf/black
     rev: 22.3.0
     hooks:

README

@@ -76,7 +76,7 @@ djhtml is used to automatically indent html files, using those parameters:
 
 django-upgrade is used to automatically upgrade Django syntax, using those parameters:
 
-    django-upgrade --target-version 2.2
+    django-upgrade --target-version 3.2
 
 There is .pre-commit-config.yaml to use pre-commit to automatically run these tools
 before commits. (execute `pre-commit install` to install the git hook.)

passerelle/apps/atos_genesys/models.py

@@ -19,8 +19,8 @@ import xml.etree.ElementTree as ET
 from urllib import parse as urlparse
 
 import requests
-from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models import JSONField
 from django.utils.translation import gettext_lazy as _
 
 from passerelle.base.models import BaseResource, HTTPResource

passerelle/apps/base_adresse/models.py

@@ -5,10 +5,9 @@ import json
 from io import StringIO
 from urllib import parse as urlparse
 
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import FieldError
 from django.db import connection, models
-from django.db.models import Q
+from django.db.models import JSONField, Q
 from django.utils import timezone
 from django.utils.http import urlencode
 from django.utils.translation import gettext_lazy as _

passerelle/apps/bbb/models.py

@@ -18,10 +18,10 @@ import hashlib
 import time
 import uuid
 
-from django.contrib.postgres.fields import JSONField
 from django.core.cache import cache
 from django.core.exceptions import PermissionDenied
 from django.db import models, transaction
+from django.db.models import JSONField
 from django.http import HttpResponseRedirect, JsonResponse
 from django.shortcuts import get_object_or_404
 from django.urls import reverse

passerelle/apps/bdp/admin.py

@@ -3,8 +3,6 @@ from django.contrib import admin
 from passerelle.apps.bdp.models import Bdp
 
 
+@admin.register(Bdp)
 class BdpAdmin(admin.ModelAdmin):
     prepopulated_fields = {'slug': ('title',)}
-
-
-admin.site.register(Bdp, BdpAdmin)

passerelle/apps/cartads_cs/models.py

@@ -30,10 +30,10 @@ import pdfrw.findobjs
 import zeep.helpers as zeep_helpers
 from Cryptodome.Cipher import AES
 from django.conf import settings
-from django.contrib.postgres.fields import JSONField
 from django.core.files.storage import default_storage
 from django.core.signing import Signer
 from django.db import models
+from django.db.models import JSONField
 from django.http import HttpResponse
 from django.urls import reverse
 from django.utils.encoding import force_str

passerelle/apps/csvdatasource/__init__.py

@@ -1,30 +0,0 @@
-# passerelle - uniform access to multiple data sources and services
-# Copyright (C) 2016 Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import django.apps
-
-
-class AppConfig(django.apps.AppConfig):
-    name = 'passerelle.apps.csvdatasource'
-    label = 'csvdatasource'
-
-    def get_connector_model(self):
-        from . import models
-
-        return models.CsvDataSource
-
-
-default_app_config = 'passerelle.apps.csvdatasource.AppConfig'

passerelle/apps/csvdatasource/apps.py

@@ -0,0 +1,27 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2016 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import django.apps
+
+
+class AppConfig(django.apps.AppConfig):
+    name = 'passerelle.apps.csvdatasource'
+    label = 'csvdatasource'
+
+    def get_connector_model(self):
+        from . import models
+
+        return models.CsvDataSource

passerelle/apps/csvdatasource/models.py

@@ -25,11 +25,11 @@ from collections import OrderedDict
 
 import pytz
 from django.conf import settings
-from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.indexes import GinIndex
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models, transaction
+from django.db.models import JSONField
 from django.urls import reverse
 from django.utils.encoding import force_str, smart_str
 from django.utils.timezone import make_aware, now

passerelle/apps/franceconnect_data/models.py

@@ -18,9 +18,9 @@ import datetime
 import json
 import uuid
 
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import PermissionDenied
 from django.db import models
+from django.db.models import JSONField
 from django.http import HttpResponseBadRequest, HttpResponseRedirect
 from django.template import Context, Template
 from django.template.response import TemplateResponse

passerelle/apps/gesbac/models.py

@@ -17,8 +17,8 @@ import csv
 import io
 from collections import OrderedDict
 
-from django.contrib.postgres.fields import JSONField
 from django.db import IntegrityError, models, transaction
+from django.db.models import JSONField
 from django.http import Http404
 from django.utils.translation import gettext_lazy as _
 

passerelle/apps/jsondatastore/models.py

@@ -17,8 +17,8 @@
 import json
 import uuid
 
-from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models import JSONField
 from django.template import Context, Template
 from django.utils.translation import gettext_lazy as _
 

passerelle/apps/mdel_ddpacs/abstract.py

@@ -22,8 +22,8 @@ import xml.etree.ElementTree as ET
 import zipfile
 from collections import namedtuple
 
-from django.contrib.postgres.fields import JSONField
 from django.db import IntegrityError, models, transaction
+from django.db.models import JSONField
 from django.http import HttpResponse
 from django.urls import reverse
 from django.utils import functional

passerelle/apps/mobyt/admin.py

@@ -3,9 +3,7 @@ from django.contrib import admin
 from .models import MobytSMSGateway
 
 
+@admin.register(MobytSMSGateway)
 class MobytSMSGatewayAdmin(admin.ModelAdmin):
     prepopulated_fields = {'slug': ('title',)}
     list_display = ['title', 'slug', 'description', 'username', 'password', 'quality', 'default_country_code']
-
-
-admin.site.register(MobytSMSGateway, MobytSMSGatewayAdmin)

passerelle/apps/opengis/models.py

@@ -19,10 +19,9 @@ import math
 import xml.etree.ElementTree as ET
 
 import pyproj
-from django.contrib.postgres.fields import JSONField
 from django.core.cache import cache
 from django.db import models, transaction
-from django.db.models import Q
+from django.db.models import JSONField, Q
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.template import Context, Template, TemplateSyntaxError

passerelle/apps/ovh/admin.py

@@ -3,6 +3,7 @@ from django.contrib import admin
 from .models import OVHSMSGateway
 
 
+@admin.register(OVHSMSGateway)
 class OVHSMSGatewayAdmin(admin.ModelAdmin):
     prepopulated_fields = {'slug': ('title',)}
     list_display = [
@@ -16,6 +17,3 @@ class OVHSMSGatewayAdmin(admin.ModelAdmin):
         'credit_left',
         'default_country_code',
     ]
-
-
-admin.site.register(OVHSMSGateway, OVHSMSGatewayAdmin)

passerelle/apps/oxyd/admin.py

@@ -3,9 +3,7 @@ from django.contrib import admin
 from .models import OxydSMSGateway
 
 
+@admin.register(OxydSMSGateway)
 class OxydSMSGatewayAdmin(admin.ModelAdmin):
     prepopulated_fields = {'slug': ('title',)}
     list_display = ['title', 'slug', 'description', 'username', 'password', 'default_country_code']
-
-
-admin.site.register(OxydSMSGateway, OxydSMSGatewayAdmin)

passerelle/apps/pdf/models.py

@@ -21,9 +21,9 @@ import tempfile
 from collections import OrderedDict
 
 from django.conf import settings
-from django.contrib.postgres.fields.jsonb import JSONField
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.db.models import JSONField
 from django.http.response import HttpResponse
 from django.utils.translation import gettext_lazy as _
 

passerelle/apps/phonecalls/models.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models import JSONField
 from django.http import HttpResponseRedirect
 from django.shortcuts import render
 from django.utils.http import urlencode

passerelle/apps/photon/models.py

@@ -18,8 +18,8 @@ import hashlib
 import json
 from urllib import parse as urlparse
 
-from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models import JSONField
 from django.utils.encoding import force_bytes
 from django.utils.http import urlencode
 from django.utils.translation import gettext_lazy as _

passerelle/apps/sector/admin.py

@@ -19,12 +19,14 @@ from django.contrib import admin
 from passerelle.apps.sector.models import Sector, Sectorization
 
 
+@admin.register(Sector)
 class SectorAdmin(admin.ModelAdmin):
     prepopulated_fields = {'slug': ('title',)}
     list_display = ('title', 'slug', 'resource')
     list_filter = ('resource',)
 
 
+@admin.register(Sectorization)
 class SectorizationAdmin(admin.ModelAdmin):
     list_display = (
         'id',
@@ -36,7 +38,3 @@ class SectorizationAdmin(admin.ModelAdmin):
         'resource',
     )
     list_filter = ('sector__resource',)
-
-
-admin.site.register(Sector, SectorAdmin)
-admin.site.register(Sectorization, SectorizationAdmin)

passerelle/base/__init__.py

@@ -1,74 +0,0 @@
-# passerelle - uniform access to multiple data sources and services
-# Copyright (C) 2016 Entr'ouvert
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import django.apps
-from django.apps import apps
-from django.utils.module_loading import import_string
-
-
-class ConnectorAppMixin:
-    def get_connector_model(self):
-        return self._connector_model
-
-    def get_urls(self):
-        try:
-            return import_string('%s.urls.urlpatterns' % self.name)
-        except ImportError:
-            return None
-
-    def get_management_urls(self):
-        try:
-            return import_string('%s.urls.management_urlpatterns' % self.name)
-        except ImportError:
-            return None
-
-
-class ConnectorAppConfig(ConnectorAppMixin, django.apps.AppConfig):
-    pass
-
-
-class AppConfig(django.apps.AppConfig):
-    name = 'passerelle.base'
-
-    def ready(self):
-        # once all applications are ready, go through them and mark them as
-        # connectors if they have a get_connector_model() method or a model
-        # that inherits from BaseResource.
-        from .models import BaseResource
-
-        for app in apps.get_app_configs():
-            connector_model = None
-            if hasattr(app, 'get_connector_model'):
-                connector_model = app.get_connector_model()
-            else:
-                for model in app.get_models():
-                    if issubclass(model, BaseResource):
-                        connector_model = model
-                        app._connector_model = model
-                        break
-            if not connector_model:
-                continue
-            if app.__class__ is django.apps.AppConfig:
-                # switch class if it's an application without a custom
-                # appconfig.
-                app.__class__ = ConnectorAppConfig
-            else:
-                # add mixin to base classes if it's an application with a
-                # custom appconfig.
-                app.__class__.__bases__ = (ConnectorAppMixin,) + app.__class__.__bases__
-
-
-default_app_config = 'passerelle.base.AppConfig'

passerelle/base/apps.py

@@ -0,0 +1,72 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2016 Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import django.apps
+from django.apps import apps
+from django.utils.module_loading import import_string
+
+
+class ConnectorAppMixin:
+    def get_connector_model(self):
+        return self._connector_model
+
+    def get_urls(self):
+        try:
+            return import_string('%s.urls.urlpatterns' % self.name)
+        except ImportError:
+            return None
+
+    def get_management_urls(self):
+        try:
+            return import_string('%s.urls.management_urlpatterns' % self.name)
+        except ImportError:
+            return None
+
+
+class ConnectorAppConfig(ConnectorAppMixin, django.apps.AppConfig):
+    pass
+
+
+class AppConfig(django.apps.AppConfig):
+    name = 'passerelle.base'
+    default = True
+
+    def ready(self):
+        # once all applications are ready, go through them and mark them as
+        # connectors if they have a get_connector_model() method or a model
+        # that inherits from BaseResource.
+        from .models import BaseResource
+
+        for app in apps.get_app_configs():
+            connector_model = None
+            if hasattr(app, 'get_connector_model'):
+                connector_model = app.get_connector_model()
+            else:
+                for model in app.get_models():
+                    if issubclass(model, BaseResource):
+                        connector_model = model
+                        app._connector_model = model
+                        break
+            if not connector_model:
+                continue
+            if app.__class__ is django.apps.AppConfig:
+                # switch class if it's an application without a custom
+                # appconfig.
+                app.__class__ = ConnectorAppConfig
+            else:
+                # add mixin to base classes if it's an application with a
+                # custom appconfig.
+                app.__class__.__bases__ = (ConnectorAppMixin,) + app.__class__.__bases__

passerelle/base/management/commands/ensure_jsonb.py

@@ -15,9 +15,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.apps import apps
-from django.contrib.postgres.fields import JSONField
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connection
+from django.db.models import JSONField
 
 
 class Command(BaseCommand):

passerelle/base/models.py

@@ -17,11 +17,11 @@ from django.apps import apps
 from django.conf import settings
 from django.contrib.contenttypes import fields
 from django.contrib.contenttypes.models import ContentType
-from django.contrib.postgres.fields import ArrayField, JSONField
+from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import PermissionDenied, ValidationError
 from django.core.files.base import ContentFile
 from django.db import connection, models, transaction
-from django.db.models import Q
+from django.db.models import JSONField, Q
 from django.forms.models import modelform_factory
 from django.forms.widgets import ClearableFileInput
 from django.test import override_settings
@@ -1031,7 +1031,7 @@ class ProxyLogger:
                 attr['extra']['transaction_id'] = self.transaction_id
 
             if getattr(request, 'META', None):
-                if 'HTTP_X_FORWARDED_FOR' in request.META:
+                if 'x-forwarded-for' in request.headers:
                     sourceip = request.headers.get('X-Forwarded-For', '').split(",")[0].strip()
                 else:
                     sourceip = request.META.get('REMOTE_ADDR')

passerelle/contrib/fake_family/models.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
+from django.db.models import JSONField
 from django.utils.translation import gettext_lazy as _
 
 from passerelle.base.models import BaseResource

passerelle/contrib/mdph13/admin.py

@@ -3,10 +3,8 @@ from django.contrib import admin
 from .models import Link
 
 
+@admin.register(Link)
 class LinkAdmin(admin.ModelAdmin):
     data_hierarchy = 'created'
     search_fields = ['display_name', 'file_number', 'dob']
     list_display = ['id', 'created', 'resource', 'name_id', 'file_number']
-
-
-admin.site.register(Link, LinkAdmin)

passerelle/contrib/planitech/models.py

@@ -22,9 +22,9 @@ import uuid
 from datetime import date, datetime, time, timedelta
 from urllib import parse as urlparse
 
-from django.contrib.postgres.fields import JSONField
 from django.core.cache import cache
 from django.db import models, transaction
+from django.db.models import JSONField
 from django.utils import dateformat, dateparse
 from django.utils.encoding import force_bytes
 from django.utils.translation import gettext_lazy as _

passerelle/contrib/teamnet_axel/models.py

@@ -20,8 +20,8 @@ import logging
 import xml.etree.ElementTree as ET
 from datetime import datetime
 
-from django.contrib.postgres.fields import JSONField
 from django.db import models
+from django.db.models import JSONField
 from django.http import HttpResponse, HttpResponseNotFound
 from django.utils.encoding import smart_str
 from django.utils.translation import gettext_lazy as _

passerelle/contrib/toulouse_foederis/models.py

@@ -17,10 +17,9 @@
 import base64
 
 import requests
-from django.contrib.postgres.fields import JSONField
 from django.core.files.base import ContentFile
 from django.db import models, transaction
-from django.db.models import Q
+from django.db.models import JSONField, Q
 from django.http import Http404, HttpResponse
 from django.urls import reverse
 from django.utils.timezone import localtime, now

passerelle/contrib/toulouse_maelis/models.py

@@ -21,9 +21,9 @@ from urllib.parse import urljoin
 
 import zeep
 from dateutil import rrule
-from django.contrib.postgres.fields import JSONField
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
+from django.db.models import JSONField
 from django.utils import dateformat
 from django.utils.dateparse import parse_date
 from django.utils.text import slugify

passerelle/contrib/toulouse_smart/models.py

@@ -22,10 +22,10 @@ from uuid import uuid4
 
 import lxml.etree as ET
 from django.conf import settings
-from django.contrib.postgres.fields import JSONField
 from django.core.exceptions import ValidationError
 from django.core.files.base import ContentFile
 from django.db import models
+from django.db.models import JSONField
 from django.db.transaction import atomic
 from django.urls import reverse
 from django.utils.text import slugify

passerelle/plugins.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.apps import apps
-from django.urls import include, re_path
+from django.urls import include, path, re_path
 
 from .urls_utils import app_enabled, manager_required, required, trust_required
 
@@ -36,11 +36,11 @@ def register_apps_urls(urlpatterns):
                 if hasattr(obj, 'get_before_urls'):
                     urls = obj.get_before_urls()
                     if urls:
-                        before_urls.append(re_path('^', include(urls)))
+                        before_urls.append(path('', include(urls)))
                 if hasattr(obj, 'get_after_urls'):
                     urls = obj.get_after_urls()
                     if urls:
-                        after_urls.append(re_path('^', include(urls)))
+                        after_urls.append(path('', include(urls)))
             elif hasattr(obj, 'get_urls'):
                 url_prefix = '^%s/' % connector_slug
                 urls = obj.get_urls()

passerelle/settings.py

@@ -33,7 +33,7 @@ ALLOWED_HOSTS = []
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'NAME': 'passerelle.sqlite3',
     }
 }

passerelle/urls.py

@@ -37,7 +37,7 @@ urlpatterns = [
     re_path(r'^admin/', admin.site.urls),
     re_path(r'^manage/access/', decorated_includes(manager_required, include(access_urlpatterns))),
     re_path(r'^manage/', decorated_includes(manager_required, include(import_export_urlpatterns))),
-    re_path('^api/', include(api_urls)),
+    path('api/', include(api_urls)),
 ]
 
 # add patterns from apps
@@ -51,8 +51,8 @@ urlpatterns += [
 
 if 'mellon' in settings.INSTALLED_APPS:
     urlpatterns += [
-        re_path(
+        path(
-            r'^accounts/mellon/',
+            'accounts/mellon/',
             include('mellon.urls'),
             kwargs={
                 'template_base': 'passerelle/base.html',
@@ -128,7 +128,7 @@ if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS:
     import debug_toolbar  # pylint: disable=import-error
 
     urlpatterns = [
-        re_path(r'^__debug__/', include(debug_toolbar.urls)),
+        path('__debug__/', include(debug_toolbar.urls)),
     ] + urlpatterns
 
 urlpatterns += staticfiles_urlpatterns()

passerelle/utils/__init__.py

@@ -91,7 +91,7 @@ def get_request_users(request):
     elif 'apikey' in request.GET:
         users.extend(ApiUser.objects.filter(keytype='API', key=request.GET['apikey']))
 
-    elif 'HTTP_AUTHORIZATION' in request.META:
+    elif 'authorization' in request.headers:
         http_authorization = request.headers['Authorization'].split(' ', 1)
         scheme = http_authorization[0].lower()
         if scheme == 'basic' and len(http_authorization) > 1:

tests/settings.py

@@ -63,7 +63,7 @@ CACHES = {
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'TEST': {
             'NAME': ('passerelle-test-%s' % os.environ.get("RAND_TEST", "")),
         },