misc: remove usage of django.utils.six (#63684)

This commit is contained in:
Frédéric Péters 2022-04-15 17:12:55 +02:00
parent 656d02d970
commit 9877b65265
31 changed files with 59 additions and 83 deletions

View File

@ -3,6 +3,7 @@ import datetime
import json
import logging
import threading
import urllib.parse
from itertools import chain, islice
import requests
@ -14,7 +15,6 @@ from django.contrib.auth import get_user_model
from django.db import connection, transaction
from django.urls import reverse
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import urljoin
from django_rbac.utils import get_ou_model, get_role_model, get_role_parenting_model
from hobo.agent.common import notify_agents
@ -417,7 +417,7 @@ class Provisionning(threading.local):
tenant = getattr(connection, 'tenant', None)
assert tenant
base_url = tenant.get_base_url()
return urljoin(base_url, reverse('a2-idp-saml-metadata'))
return urllib.parse.urljoin(base_url, reverse('a2-idp-saml-metadata'))
def pre_save(self, sender, instance, raw, using, update_fields, **kwargs):
if not self.stack:

View File

@ -22,7 +22,6 @@ from authentic2.validators import EmailValidator
from django import forms
from django.core import validators
from django.core.exceptions import ValidationError
from django.utils import six
from django.utils.translation import ugettext_lazy as _
@ -42,7 +41,7 @@ class CommaSeparatedInput(forms.TextInput):
def format_value(self, value):
if not value:
return ''
if not isinstance(value, six.string_types):
if not isinstance(value, str):
return u', '.join(value)
return value

View File

@ -5,13 +5,13 @@ import os
import subprocess
import sys
import tempfile
import urllib.parse
import requests
from django.conf import settings
from django.core.management import call_command, get_commands
from django.core.management.base import BaseCommand, CommandError
from django.utils.encoding import force_text
from django.utils.six.moves.urllib import parse as urlparse
from tenant_schemas.utils import tenant_context
from hobo.multitenant.middleware import TenantMiddleware, TenantNotFound
@ -73,7 +73,7 @@ class Command(BaseCommand):
if self.me.get('secondary') and self.early_secondary_exit:
# early exit, we don't redeploy secondary services
return
domain = urlparse.urlparse(self.me.get('base_url')).netloc.split(':')[0]
domain = urllib.parse.urlparse(self.me.get('base_url')).netloc.split(':')[0]
legacy_domain = None
try:
@ -81,7 +81,7 @@ class Command(BaseCommand):
except TenantNotFound:
# might be a domain change request
for legacy_urls in self.me.get('legacy_urls', []):
old_domain = urlparse.urlparse(legacy_urls['base_url']).netloc.split(':')[0]
old_domain = urllib.parse.urlparse(legacy_urls['base_url']).netloc.split(':')[0]
try:
tenant = TenantMiddleware.get_tenant_by_hostname(old_domain)
legacy_domain = old_domain

View File

@ -23,9 +23,9 @@ import multiprocessing.pool
import os
import subprocess
import sys
import urllib.parse
from django.utils.encoding import force_bytes
from django.utils.six.moves.urllib import parse as urlparse
from . import settings
@ -59,7 +59,7 @@ class BaseService(object):
patterns = settings.AGENT_HOST_PATTERNS.get(cls.service_id)
if patterns is None:
return True
parsed_url = urlparse.urlsplit(url)
parsed_url = urllib.parse.urlsplit(url)
netloc = parsed_url.netloc
match = False
for pattern in patterns:
@ -94,7 +94,7 @@ class BaseService(object):
tenants = list(cls.get_actual_tenants())
if tenants:
for audience in data.get('audience', []):
parsed_url = urlparse.urlsplit(audience)
parsed_url = urllib.parse.urlsplit(audience)
netloc = parsed_url.netloc.split(':')[0]
if netloc in tenants:
break

View File

@ -4,6 +4,7 @@ import json
import logging
import os
import threading
import urllib.parse
import requests
from django.conf import settings
@ -11,7 +12,6 @@ from django.core.cache import cache
from django.template import Template
from django.utils.encoding import smart_bytes
from django.utils.http import urlencode
from django.utils.six.moves.urllib import parse as urlparse
from hobo.scrutiny.wsgi.middleware import VersionMiddleware
@ -68,7 +68,7 @@ class RemoteTemplate(object):
@property
def cache_key(self):
return hashlib.md5(
urlparse.urlunparse(urlparse.urlparse(self.source)[:3] + ('', '', '')).encode('ascii')
urllib.parse.urlunparse(urllib.parse.urlparse(self.source)[:3] + ('', '', '')).encode('ascii')
).hexdigest()
def get_template(self):
@ -116,7 +116,7 @@ class RemoteTemplate(object):
def update_all_pages_cache(self):
# always cache root
root_url = urlparse.urlunparse(urlparse.urlparse(self.source)[:2] + ('/', '', '', ''))
root_url = urllib.parse.urlunparse(urllib.parse.urlparse(self.source)[:2] + ('/', '', '', ''))
if not root_url in self.combo_skeleton_pages.values():
self.combo_skeleton_pages['__root'] = root_url

View File

@ -20,6 +20,7 @@ import json
import os
import string
import subprocess
import urllib.parse
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
@ -28,8 +29,6 @@ from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
from django.db.models import Max
from django.utils import six
from django.utils.six.moves.urllib import parse as urlparse
from django.utils.text import slugify
from hobo.agent.common.management.commands.hobo_deploy import Command as HoboDeployCommand
@ -56,7 +55,7 @@ from hobo.theme.utils import set_theme
def get_domain(url):
return urlparse.urlparse(url).netloc.split(':')[0]
return urllib.parse.urlparse(url).netloc.split(':')[0]
class Command(BaseCommand):
@ -100,7 +99,7 @@ class Command(BaseCommand):
for step in recipe.get('steps', []):
action, action_args = list(step.items())[0]
for arg in action_args:
if not isinstance(action_args[arg], six.string_types):
if not isinstance(action_args[arg], str):
continue
action_args[arg] = string.Template(action_args[arg]).substitute(variables)
if not self.permissive:

View File

@ -20,6 +20,7 @@ import random
import re
import socket
import time
import urllib.parse
import requests
from django.conf import settings
@ -30,10 +31,8 @@ from django.core.cache import cache
from django.core.exceptions import ValidationError
from django.core.validators import URLValidator
from django.db import models
from django.utils import six
from django.utils.crypto import get_random_string
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import urlparse
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _
@ -154,13 +153,7 @@ class ServiceBase(models.Model):
return (self.last_operational_check_timestamp - self.last_update_timestamp) < two_minutes
def as_dict(self):
as_dict = dict(
[
(x, y)
for (x, y) in self.__dict__.items()
if isinstance(y, six.integer_types + six.string_types)
]
)
as_dict = dict([(x, y) for (x, y) in self.__dict__.items() if isinstance(y, (int, str))])
as_dict['base_url'] = self.get_base_url_path()
if self.legacy_urls:
as_dict['legacy_urls'] = self.legacy_urls
@ -233,7 +226,7 @@ class ServiceBase(models.Model):
def is_resolvable(self):
try:
netloc = urlparse(self.base_url).netloc
netloc = urllib.parse.urlparse(self.base_url).netloc
if netloc and socket.gethostbyname(netloc):
return True
except socket.gaierror:

View File

@ -16,13 +16,13 @@
import sys
import time
import urllib.parse
from django.conf import settings
from django.core.management.base import CommandError
from django.db import connection, transaction
from django.urls import reverse
from django.utils.encoding import force_text
from django.utils.six.moves.urllib.parse import urlparse
from hobo.middleware.utils import StoreRequestMiddleware
from hobo.multitenant.settings_loaders import KnownServices
@ -147,7 +147,7 @@ def create_base_url(hostname, service):
Distinguish mutualised domains (matching a "-" in the first part of the netloc)
from the normal scenario with a dedicated parent domain.
"""
ph = urlparse(hostname)
ph = urllib.parse.urlparse(hostname)
parts = ph.netloc.split('.')
if '-' in parts[0]:
netloc = '%s-%s.%s' % (service, parts[0].split('-')[1], '.'.join(parts[1:]))

View File

@ -1,5 +1,6 @@
import urllib.parse
from django.core.exceptions import ValidationError
from django.utils.six.moves.urllib import parse as urlparse
from django.utils.translation import gettext_lazy as _
from hobo.environment.models import ServiceBase
@ -11,7 +12,7 @@ def validate_service_url(url):
raise ValidationError(
_('Error: %(netloc)s is not resolvable in URL %(url)s'),
code='not-resolvable',
params={'netloc': urlparse.urlsplit(url).netloc, 'url': url},
params={'netloc': urllib.parse.urlsplit(url).netloc, 'url': url},
)
if not service.has_valid_certificate():
raise ValidationError(

View File

@ -23,7 +23,6 @@ import sys as _sys
import traceback as _traceback
from syslog import LOG_ALERT, LOG_CRIT, LOG_DEBUG, LOG_ERR, LOG_INFO, LOG_WARNING
from django.utils import six
from systemd._journal import sendv
_IDENT_CHARACTER = set('ABCDEFGHIJKLMNOPQRTSUVWXYZ_0123456789')
@ -36,7 +35,7 @@ def _valid_field_name(s):
def _make_line(field, value):
if isinstance(value, bytes):
return field.encode('utf-8') + b'=' + value
elif isinstance(value, six.string_types):
elif isinstance(value, str):
return field + '=' + value
else:
return field + '=' + str(value)

View File

@ -17,6 +17,7 @@
import hashlib
import re
import string
import urllib.parse
from random import choice, randint
import requests
@ -24,7 +25,6 @@ from django.conf import settings
from django.core import exceptions
from django.db import connection
from django.utils.encoding import force_bytes
from django.utils.six.moves.urllib import parse as urlparse
from lxml import etree
from hobo.environment.models import Combo, Fargo, Variable, Wcs
@ -94,7 +94,7 @@ def get_tenant_name_and_public_urls():
tenant_name = None
services = [x for x in Combo.objects.all() if 'portal-user' in x.template_name and not x.secondary]
if services != [] and services[0] != '':
tenant_name = urlparse.urlparse(services[0].base_url).netloc
tenant_name = urllib.parse.urlparse(services[0].base_url).netloc
services += [x for x in Wcs.objects.all() if not x.secondary]
services += [x for x in Fargo.objects.all() if not x.secondary]
site_urls = [x.base_url for x in services if x.base_url != '']

View File

@ -10,11 +10,6 @@ from django.conf import settings
from django.core.management import call_command, get_commands, load_command_class
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
try:
from django.utils.six.moves import input
except ImportError:
input = raw_input
from tenant_schemas.utils import get_public_schema_name
from hobo.multitenant.middleware import TenantMiddleware

View File

@ -6,7 +6,6 @@ from django.contrib.staticfiles.handlers import StaticFilesHandler
from django.contrib.staticfiles.management.commands.runserver import Command as StaticRunserverCommand
from django.contrib.staticfiles.views import serve
from django.core.management.commands.runserver import Command as RunserverCommand
from django.utils.six.moves.urllib.request import url2pathname
from django.views import static
from hobo.multitenant.middleware import TenantMiddleware, TenantNotFound

View File

@ -14,7 +14,6 @@ from django.core.exceptions import ImproperlyConfigured
from django.core.management import call_command, get_commands, load_command_class
from django.core.management.base import BaseCommand, CommandError, SystemCheckError, handle_default_options
from django.db import connection, connections
from django.utils import six
from django.utils.encoding import force_text
from hobo.multitenant.management.commands import InteractiveTenantOption
@ -23,7 +22,7 @@ from hobo.multitenant.middleware import TenantMiddleware
def exception_to_text(e):
try:
return six.text_type(e)
return str(e)
except Exception:
pass

View File

@ -1,11 +1,11 @@
import json
import os
import urllib.parse
from shutil import rmtree
from django.conf import settings
from django.db import connection
from django.utils import timezone
from django.utils.six.moves.urllib.parse import urljoin
from tenant_schemas.models import TenantMixin
from tenant_schemas.postgresql_backend.base import _check_schema_name
from tenant_schemas.utils import django_is_in_test_mode, get_public_schema_name, schema_exists
@ -48,7 +48,7 @@ class Tenant(TenantMixin):
return 'https://%s' % self.domain_url
def build_absolute_uri(self, location):
return urljoin(self.get_base_url(), location)
return urllib.parse.urljoin(self.get_base_url(), location)
def create_schema(self, check_if_exists=False, sync_schema=True, verbosity=1, legacy_schema_name=None):
if not legacy_schema_name:

View File

@ -6,7 +6,6 @@ import urllib.parse
from django.conf import settings
from django.utils.encoding import force_bytes
from django.utils.http import urlencode
from django.utils.six.moves.urllib import parse as urlparse
from hobo.theme.utils import get_theme
@ -76,13 +75,13 @@ class KnownServices(FileBaseSettingsLoader):
services = hobo_json.get('services')
this = [s for s in services if s.get('this')][0]
base_url = this['base_url']
orig = urlparse.urlparse(base_url).netloc.split(':')[0]
orig = urllib.parse.urlparse(base_url).netloc.split(':')[0]
secret = this['secret_key']
for service in services:
service_id = service.get('service-id')
url = service.get('base_url')
verif_orig = urlparse.urlparse(url).netloc.split(':')[0]
verif_orig = urllib.parse.urlparse(url).netloc.split(':')[0]
service_data = {
'url': url,
'backoffice-menu-url': service.get('backoffice-menu-url'),
@ -238,7 +237,7 @@ class CORSSettings(FileBaseSettingsLoader):
base_url = service.get('base_url')
if not base_url:
continue
base_url = urlparse.urlparse(base_url)
base_url = urllib.parse.urlparse(base_url)
origin = '%s://%s' % (base_url.scheme, base_url.netloc)
if not origin in whitelist:
whitelist.append(origin)

View File

@ -16,7 +16,6 @@
from django.core.validators import RegexValidator
from django.db import models
from django.utils import six
from django.utils.translation import ugettext_lazy as _
validate_attribute_name = RegexValidator(
@ -68,7 +67,7 @@ class AttributeDefinition(models.Model):
ordering = ['order']
def as_dict(self):
as_dict = dict([(x, y) for (x, y) in self.__dict__.items() if type(y) in (str, six.text_type, bool)])
as_dict = dict([(x, y) for (x, y) in self.__dict__.items() if type(y) in (str, bool)])
return as_dict
def get_real_kind_display(self):

View File

@ -17,13 +17,13 @@
import json
import logging
import sys
import urllib.parse
from django.conf import settings
from django.db import connection
from django.http import HttpResponseBadRequest, HttpResponseForbidden, JsonResponse
from django.utils.deprecation import MiddlewareMixin
from django.utils.encoding import force_bytes, force_text
from django.utils.six.moves.urllib.parse import urlparse
from hobo.provisionning.utils import NotificationProcessing
from hobo.rest_authentication import PublikAuthentication, PublikAuthenticationFailed
@ -102,7 +102,7 @@ class ProvisionningMiddleware(MiddlewareMixin, NotificationProcessing):
from hobo.multitenant.settings_loaders import KnownServices
authentic = Authentic.objects.all().first()
orig = urlparse(authentic.base_url).netloc.split(':')[0]
orig = urllib.parse.urlparse(authentic.base_url).netloc.split(':')[0]
# create stub settings.KNOWN_SERVICES with just enough to get
# authentication passing.
idp_service = list(settings.KNOWN_SERVICES['authentic'].values())[0]

View File

@ -1,8 +1,8 @@
import json
import urllib.parse
import pkg_resources
from django.utils.deprecation import MiddlewareMixin
from django.utils.six.moves.urllib.parse import quote
try:
import apt.cache as apt_cache
@ -49,8 +49,8 @@ class VersionMiddleware(MiddlewareMixin):
def __call__(self, environ, start_response):
path = ''
path += quote(environ.get('SCRIPT_NAME', ''))
path += quote(environ.get('PATH_INFO', ''))
path += urllib.parse.quote(environ.get('SCRIPT_NAME', ''))
path += urllib.parse.quote(environ.get('PATH_INFO', ''))
method = environ.get('REQUEST_METHOD', 'GET')
if method == 'GET' and (path == '/__version__' or path == '/__version__/'):
packages_version = self.get_packages_version()

View File

@ -4,11 +4,10 @@ import hashlib
import hmac
import random
import secrets
import urllib.parse
from django.utils import six
from django.utils.encoding import smart_bytes
from django.utils.http import quote, urlencode
from django.utils.six.moves.urllib import parse as urlparse
'''Simple signature scheme for query strings'''
@ -18,9 +17,9 @@ class SignatureError(Exception):
def sign_url(url, key, algo='sha256', timestamp=None, nonce=None):
parsed = urlparse.urlparse(url)
parsed = urllib.parse.urlparse(url)
new_query = sign_query(parsed.query, key, algo, timestamp, nonce)
return urlparse.urlunparse(parsed[:4] + (new_query,) + parsed[5:])
return urllib.parse.urlunparse(parsed[:4] + (new_query,) + parsed[5:])
def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
@ -42,21 +41,21 @@ def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
def sign_string(s, key, algo='sha256'):
digestmod = getattr(hashlib, algo)
if isinstance(key, six.text_type):
if isinstance(key, str):
key = key.encode('utf-8')
hash = hmac.HMAC(smart_bytes(key), digestmod=digestmod, msg=smart_bytes(s))
return hash.digest()
def check_url(url, key, known_nonce=None, timedelta=30, raise_on_error=False):
parsed = urlparse.urlparse(url, 'https')
parsed = urllib.parse.urlparse(url, 'https')
return check_query(
parsed.query, key, known_nonce=known_nonce, timedelta=timedelta, raise_on_error=raise_on_error
)
def check_query(query, key, known_nonce=None, timedelta=30, raise_on_error=False):
parsed = urlparse.parse_qs(query)
parsed = urllib.parse.parse_qs(query)
parsed = {key: value[0] if len(value) == 1 else value for key, value in parsed.items()}
signature = parsed.get('signature')
if not signature or not isinstance(signature, str):

View File

@ -18,7 +18,6 @@ import json
import os
from django.conf import settings
from django.utils import six
def get_themes():
@ -62,7 +61,7 @@ def set_theme(theme_id):
for variable in theme.get('variables', {}).keys():
theme_variable, created = Variable.objects.get_or_create(name=variable, service_pk__isnull=True)
theme_variable.auto = True
if isinstance(theme['variables'][variable], six.string_types):
if isinstance(theme['variables'][variable], str):
theme_variable.value = theme['variables'][variable]
else:
theme_variable.value = json.dumps(theme['variables'][variable])

View File

@ -1,11 +1,11 @@
import json
import os
from io import StringIO
import pytest
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.management.base import CommandError
from django.utils.six import StringIO
from mock import Mock, call, mock_open, patch
from hobo.environment.management.commands.cook import Command

View File

@ -3,9 +3,9 @@
import json
import os
import sys
from io import StringIO
import pytest
from django.utils.six import StringIO
from mock import Mock, call, patch
from requests import Response, exceptions

View File

@ -1,6 +1,5 @@
import datetime
from django.utils.six.moves.urllib import parse as urllib
import urllib.parse
from hobo import signature
@ -38,7 +37,7 @@ def test_signature():
# Test timedelta parameter
now = datetime.datetime.utcnow()
assert '&timestamp=%s' % urllib.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in signature.sign_url(
assert '&timestamp=%s' % urllib.parse.quote(now.strftime('%Y-%m-%dT%H:%M:%SZ')) in signature.sign_url(
URL, KEY, timestamp=now
)

View File

@ -1,12 +1,9 @@
from django.utils import six
def byteify(input):
if isinstance(input, dict):
return {byteify(key): byteify(value) for key, value in input.items()}
elif isinstance(input, list):
return [byteify(element) for element in input]
elif isinstance(input, six.text_type):
elif isinstance(input, str):
return input.encode('utf-8')
else:
return input

View File

@ -1,6 +1,6 @@
import builtins
import os
from django.utils.six.moves import builtins
from mock import mock_open, patch
# Debian defaults

View File

@ -15,11 +15,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import re
import urllib.parse
import pytest
from django.contrib.auth import get_user_model
from django.test import RequestFactory
from django.utils.six.moves.urllib import parse as urllib
from rest_framework import permissions
from rest_framework.response import Response
from rest_framework.views import APIView
@ -39,7 +39,7 @@ def test_publik_authentication(tenant, settings):
User = get_user_model()
user = User.objects.create(username='foo', password='foo')
ORIG = 'other.example.net'
AUTH_QUERY = '&NameID=%s&orig=%s' % (user.uuid, urllib.quote(ORIG))
AUTH_QUERY = '&NameID=%s&orig=%s' % (user.uuid, urllib.parse.quote(ORIG))
URL = '/api/?coucou=zob'
factory = RequestFactory()
@ -54,7 +54,7 @@ def test_publik_authentication(tenant, settings):
assert result[1] is None
# Test anonymous user
AUTH_QUERY = '&orig=%s' % urllib.quote(ORIG)
AUTH_QUERY = '&orig=%s' % urllib.parse.quote(ORIG)
request = factory.get(signature.sign_url(URL + AUTH_QUERY, key))
publik_authentication = rest_authentication.PublikAuthentication()
@ -73,7 +73,7 @@ def test_publik_authentication(tenant, settings):
# Test user named after service orig
service_user = User.objects.create(username=ORIG)
AUTH_QUERY = '&orig=%s' % urllib.quote(ORIG)
AUTH_QUERY = '&orig=%s' % urllib.parse.quote(ORIG)
request = factory.get(signature.sign_url(URL + AUTH_QUERY, key))
publik_authentication = rest_authentication.PublikAuthentication()

View File

@ -1,6 +1,6 @@
import builtins
import os.path
from django.utils.six.moves import builtins
from mock import mock_open, patch
from hobo.settings import *

View File

@ -1,6 +1,6 @@
import builtins
import os
from django.utils.six.moves import builtins
from mock import mock_open, patch
LANGUAGE_CODE = 'en-us'

View File

@ -1,6 +1,6 @@
import builtins
import os
from django.utils.six.moves import builtins
from mock import mock_open, patch
# Debian defaults

View File

@ -1,10 +1,10 @@
import json
import os
import sys
from io import StringIO
from django.core.management import call_command
from django.db import connection
from django.utils.six import StringIO
from passerelle.utils import export_site
from hobo.multitenant.middleware import TenantMiddleware