tests: add a tox.ini and other tox/pytest-related files (#61045)

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
 # trivial: apply black
 7a234d5fe7ae6bee3ba1d0f688967e8e6cf209e3
+# trivial: apply isort & pyupgrade
+1abbbadd9469a3f2ff7eafb0ec6956c2b1c6763c

.pre-commit-config.yaml

@@ -6,3 +6,13 @@ repos:
     hooks:
     - id: black
       args: ['--target-version', 'py37', '--skip-string-normalization', '--line-length', '110']
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.7.0
+    hooks:
+    - id: isort
+      args: ['--profile', 'black', '--line-length', '110']
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.20.0
+    hooks:
+    - id: pyupgrade
+      args: ['--keep-percent-format', '--py37-plus']

Jenkinsfile

@@ -2,14 +2,36 @@
 
 pipeline {
     agent any
+    options { disableConcurrentBuilds() }
+    environment {
+        TMPDIR = "/tmp/$BUILD_TAG"
+    }
     stages {
+        stage('Unit Tests') {
+            steps {
+                sh "mkdir ${env.TMPDIR}"
+                sh """
+python3 -m venv ${env.TMPDIR}/venv/
+${env.TMPDIR}/venv/bin/pip install tox
+PGPORT=`python3 -c 'import struct; import socket; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` pg_virtualenv -o fsync=off ${env.TMPDIR}/venv/bin/tox -r"""
+            }
+            post {
+                always {
+                    script {
+                        utils = new Utils()
+                        utils.publish_coverage('coverage.xml')
+                        utils.publish_coverage_native('index.html')
+                        utils.publish_pylint('pylint.out')
+                    }
+                    mergeJunitResults()
+                }
+            }
+        }
         stage('Packaging') {
             steps {
                 script {
                     if (env.JOB_NAME == 'authentic2-auth-fedict' && env.GIT_BRANCH == 'origin/main') {
                         sh 'sudo -H -u eobuilder /usr/local/bin/eobuilder -d buster,bullseye authentic2-auth-fedict'
-                    } else if (env.GIT_BRANCH.startsWith('hotfix/')) {
-                        sh "sudo -H -u eobuilder /usr/local/bin/eobuilder -d buster,bullseye --branch ${env.GIT_BRANCH} --hotfix authentic2-auth-fedict"
                     }
                 }
             }
@@ -22,7 +44,8 @@ pipeline {
                 utils.mail_notify(currentBuild, env, 'ci+jenkins-authentic2-auth-fedict@entrouvert.org')
             }
         }
-        success {
+        cleanup {
+            sh "find ${env.TMPDIR} -type f -delete; rm -rf ${env.TMPDIR}"
             cleanWs()
         }
     }

README

@@ -28,8 +28,17 @@ black is used to format the code, using thoses parameters:
 
     black --target-version py37 --skip-string-normalization --line-length 110
 
-There is .pre-commit-config.yaml to use pre-commit to automatically run black
-before commits. (execute `pre-commit install` to install the git hook.)
+isort is used to format the imports, using those parameter:
+
+    isort --profile black --line-length 110
+
+pyupgrade is used to automatically upgrade syntax, using those parameters:
+
+    pyupgrade --keep-percent-format --py37-plus
+
+There is .pre-commit-config.yaml to use pre-commit to automatically run black,
+isort and pyupgrade before commits. (execute `pre-commit install` to install
+the git hook.)
 
 
 License

check-migrations.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+
+# https://stackoverflow.com/questions/49778988/makemigrations-in-dev-machine-without-database-instance
+CHECK_MIGRATIONS_SETTINGS=`mktemp`
+trap "rm -f ${CHECK_MIGRATIONS_SETTINGS}" EXIT
+cat <<EOF >${CHECK_MIGRATIONS_SETTINGS}
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.dummy',
+    }
+}
+EOF
+TEMPFILE=`mktemp`
+trap "rm -f ${TEMPFILE} ${CHECK_MIGRATIONS_SETTINGS}" EXIT
+
+DJANGO_SETTINGS_MODULE=authentic2.settings AUTHENTIC2_SETTINGS_FILE=${CHECK_MIGRATIONS_SETTINGS}  django-admin makemigrations --dry-run --noinput authentic2_auth_fedict >${TEMPFILE} 2>&1 || true
+
+if ! grep 'No changes detected' -q ${TEMPFILE}; then
+   echo '!!! Missing migration detected !!!'
+   cat ${TEMPFILE}
+   exit 1
+else
+   exit 0
+fi

getlasso3.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# Get venv site-packages path
+DSTDIR=`python3 -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'`
+
+# Get not venv site-packages path
+# Remove first path (assuming that is the venv path)
+NONPATH=`echo $PATH | sed 's/^[^:]*://'`
+SRCDIR=`PATH=$NONPATH python3 -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'`
+
+# Clean up
+rm -f $DSTDIR/lasso.*
+rm -f $DSTDIR/_lasso.*
+
+# Link
+ln -sv /usr/lib/python3/dist-packages/lasso.py $DSTDIR/
+for SOFILE in /usr/lib/python3/dist-packages/_lasso.cpython-*.so
+do
+  ln -sv $SOFILE $DSTDIR/
+done
+
+exit 0

pylint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+if [ -f /var/lib/jenkins/pylint.django.rc ]; then
+	PYLINT_RC=/var/lib/jenkins/pylint.django.rc
+elif [ -f pylint.django.rc ]; then
+	PYLINT_RC=pylint.django.rc
+else
+	echo No pylint RC found
+	exit 0
+fi
+pylint -f parseable --rcfile ${PYLINT_RC} "$@" > pylint.out || /bin/true

setup.py

@@ -1,13 +1,13 @@
 #!/usr/bin/python
-import sys
 import os
 import subprocess
-
-from setuptools import setup, find_packages
-from setuptools.command.install_lib import install_lib as _install_lib
+import sys
+from distutils.cmd import Command
 from distutils.command.build import build as _build
 from distutils.command.sdist import sdist
-from distutils.cmd import Command
+
+from setuptools import find_packages, setup
+from setuptools.command.install_lib import install_lib as _install_lib
 
 
 class compile_translations(Command):
@@ -60,7 +60,7 @@ class install_lib(_install_lib):
 
 def get_version():
     if os.path.exists('VERSION'):
-        with open('VERSION', 'r') as v:
+        with open('VERSION') as v:
             return v.read()
     if os.path.exists('.git'):
         p = subprocess.Popen(

src/authentic2_auth_fedict/__init__.py

@@ -17,8 +17,8 @@
 import json
 
 import django.apps
-from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth.signals import user_logged_in
+from django.utils.translation import ugettext_lazy as _
 
 
 class AppConfig(django.apps.AppConfig):
@@ -39,7 +39,7 @@ class AppConfig(django.apps.AppConfig):
 default_app_config = 'authentic2_auth_fedict.AppConfig'
 
 
-class Plugin(object):
+class Plugin:
     def get_before_urls(self):
         from . import urls
 

src/authentic2_auth_fedict/adapters.py

@@ -20,19 +20,16 @@ import logging
 import os
 import time
 
+import lasso
+import mellon.utils as mellon_utils
 import requests
-
+from authentic2.a2_rbac.utils import get_default_ou
+from authentic2.models import Attribute
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.files.storage import default_storage
 from django.utils.encoding import force_bytes, force_text
-
-import lasso
-
 from mellon.adapters import DefaultAdapter, app_settings
-import mellon.utils as mellon_utils
-from authentic2.models import Attribute
-from authentic2.a2_rbac.utils import get_default_ou
 
 try:
     import authentic2.utils.misc as a2_utils_misc
@@ -95,7 +92,7 @@ class AuthenticAdapter(DefaultAdapter):
         saml_attributes['name_id_content_orig'] = saml_attributes['name_id_content']
         saml_attributes['name_id_content'] = saml_attributes['urn:be:fedict:iam:attr:fedid'][0]
         saml_attributes['name_id_format'] = lasso.SAML2_NAME_IDENTIFIER_FORMAT_UNSPECIFIED
-        user = super(AuthenticAdapter, self).lookup_user(idp, saml_attributes)
+        user = super().lookup_user(idp, saml_attributes)
         if not user.ou_id:
             user.ou = get_default_ou()
             user.save()
@@ -164,7 +161,7 @@ class AuthenticAdapter(DefaultAdapter):
                 pass
 
     def provision_attribute(self, user, idp, saml_attributes):
-        super(AuthenticAdapter, self).provision_attribute(user, idp, saml_attributes)
+        super().provision_attribute(user, idp, saml_attributes)
         if not user.email:
             # make sure the account is not usable for now
             user.is_active = False

src/authentic2_auth_fedict/app_settings.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-class AppSettings(object):
+class AppSettings:
     '''Thanks django-allauth'''
 
     __SENTINEL = object()

src/authentic2_auth_fedict/authenticators.py

@@ -14,13 +14,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.template.loader import render_to_string
-from django.shortcuts import render
-from django.utils.translation import ugettext_lazy as _
-
-from mellon.utils import get_idp, get_idps
-
 from authentic2.authenticators import BaseAuthenticator
+from django.shortcuts import render
+from django.template.loader import render_to_string
+from django.utils.translation import ugettext_lazy as _
+from mellon.utils import get_idp, get_idps
 
 try:
     from authentic2.utils import redirect_to_login

src/authentic2_auth_fedict/fields.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # authentic2_auth_fedict - Fedict authentication for Authentic
 # Copyright (C) 2016  Entr'ouvert
 #
@@ -16,20 +15,19 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
-import requests
 import time
 from urllib.parse import urljoin
 
+import requests
 from django import forms
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
-
 from gadjo.templatetags.gadjo import xstatic
 
 
 class NrnField(forms.CharField):
     def validate(self, value):
-        super(NrnField, self).validate(value)
+        super().validate(value)
         if not value:
             return
         try:
@@ -62,7 +60,7 @@ class DateField(forms.CharField):
     widget = DateWidget
 
     def validate(self, value):
-        super(DateField, self).validate(value)
+        super().validate(value)
         if not value:
             return
         for format_string in ('%d/%m/%Y', '%Y-%m-%d'):
@@ -133,7 +131,7 @@ class CountryField(forms.CharField):
 
 class NumHouseField(forms.CharField):
     def validate(self, value):
-        super(NumHouseField, self).validate(value)
+        super().validate(value)
         if not value:
             return
         try:
@@ -145,11 +143,11 @@ class NumHouseField(forms.CharField):
 
 class NumPhoneField(forms.CharField):
     def validate(self, value):
-        super(NumPhoneField, self).validate(value)
+        super().validate(value)
         if not value:
             return
         try:
-            if not re.match("^(0|\\+|00)(\d{8,})", value):
+            if not re.match("^(0|\\+|00)(\\d{8,})", value):
                 raise ValueError()
         except ValueError:
             raise forms.ValidationError(getattr(settings, 'A2_NUMPHONE_ERROR_MESSAGE', _('Invalid format')))

src/authentic2_auth_fedict/urls.py

@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.conf.urls import url, include
+from django.conf.urls import include, url
 
 from . import views
 

src/authentic2_auth_fedict/views.py

@@ -14,15 +14,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import urllib.parse
 import random
+import urllib.parse
 
 from django.conf import settings
 from django.core import signing
-from django.urls import reverse
 from django.db import transaction
 from django.http import HttpResponseRedirect
 from django.shortcuts import resolve_url
+from django.urls import reverse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import View
 
@@ -65,7 +65,7 @@ class LoginView(mellon.views.LoginView):
             return HttpResponseRedirect(a2_utils_misc.build_activation_url(request, **data))
         user.is_active = True
         user.save()
-        return super(LoginView, self).authenticate(request, login, attributes)
+        return super().authenticate(request, login, attributes)
 
 
 login = transaction.non_atomic_requests(csrf_exempt(LoginView.as_view()))

tests/conftest.py

@@ -0,0 +1,80 @@
+import django_webtest
+import pytest
+
+try:
+    import pathlib
+except ImportError:
+    import pathlib2 as pathlib
+
+from django.contrib.auth import get_user_model
+from django_rbac.utils import get_ou_model
+
+User = get_user_model()
+TEST_DIR = pathlib.Path(__file__).parent
+
+
+@pytest.fixture
+def app(request, db, settings, tmpdir):
+    wtm = django_webtest.WebTestMixin()
+    wtm._patch_settings()
+    request.addfinalizer(wtm._unpatch_settings)
+    settings.MEDIA_DIR = str(tmpdir.mkdir('media'))
+    return django_webtest.DjangoTestApp(extra_environ={'HTTP_HOST': 'localhost'})
+
+
+class AllHook:
+    def __init__(self):
+        self.calls = {}
+        from authentic2 import hooks
+
+        hooks.get_hooks.cache.clear()
+
+    def __call__(self, hook_name, *args, **kwargs):
+        calls = self.calls.setdefault(hook_name, [])
+        calls.append({'args': args, 'kwargs': kwargs})
+
+    def __getattr__(self, name):
+        return self.calls.get(name, [])
+
+    def clear(self):
+        self.calls = {}
+
+
+@pytest.fixture
+def user(db):
+    user = User.objects.create(
+        username='john.doe',
+        email='john.doe@example.net',
+        first_name='John',
+        last_name='Doe',
+        email_verified=True,
+    )
+    user.set_password('john.doe')
+    return user
+
+
+@pytest.fixture
+def hooks(settings):
+    if hasattr(settings, 'A2_HOOKS'):
+        hooks = settings.A2_HOOKS
+    else:
+        hooks = settings.A2_HOOKS = {}
+    hook = hooks['__all__'] = AllHook()
+    yield hook
+    hook.clear()
+    del settings.A2_HOOKS['__all__']
+
+
+@pytest.fixture
+def admin(db):
+    user = User(username='admin', email='admin@example.net', is_superuser=True, is_staff=True)
+    user.set_password('admin')
+    user.save()
+    return user
+
+
+@pytest.fixture(autouse=True)
+def clean_caches():
+    from authentic2.apps.journal.models import event_type_cache
+
+    event_type_cache.cache.clear()

tests/settings.py

@@ -0,0 +1,22 @@
+import os
+
+ALLOWED_HOSTS = ['localhost']
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'authentic2-auth-fedict',
+    }
+}
+
+if 'postgres' in DATABASES['default']['ENGINE']:
+    for key in ('PGPORT', 'PGHOST', 'PGUSER', 'PGPASSWORD'):
+        if key in os.environ:
+            DATABASES['default'][key[2:]] = os.environ[key]
+
+LANGUAGE_CODE = 'en'
+A2_FC_CLIENT_ID = ''
+A2_FC_CLIENT_SECRET = ''
+
+# test hook handlers
+A2_HOOKS_PROPAGATE_EXCEPTIONS = True

tests/test_all.py

@@ -0,0 +1,11 @@
+import pytest
+from django.contrib.auth import get_user_model
+from utils import login
+
+User = get_user_model()
+
+pytestmark = pytest.mark.django_db
+
+
+def test_dummy(app):
+    assert 1

tests/utils.py

@@ -0,0 +1,24 @@
+from authentic2.utils.misc import make_url
+from django.urls import reverse
+
+
+def login(app, user, path=None, password=None, remember_me=None):
+    if path:
+        real_path = make_url(path)
+        login_page = app.get(real_path, status=302).maybe_follow()
+    else:
+        login_page = app.get(reverse('auth_login'))
+    assert login_page.request.path == reverse('auth_login')
+    form = login_page.form
+    form.set('username', user.username if hasattr(user, 'username') else user)
+    # password is supposed to be the same as username
+    form.set('password', password or user.username)
+    if remember_me is not None:
+        form.set('remember_me', bool(remember_me))
+    response = form.submit(name='login-password-submit').follow()
+    if path:
+        assert response.request.path == real_path
+    else:
+        assert response.request.path == reverse('auth_homepage')
+    assert '_auth_user_id' in app.session
+    return response

tox.ini

@@ -0,0 +1,85 @@
+# Tox (http://tox.testrun.org/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/authentic2-auth-fedict/{env:BRANCH_NAME:}
+envlist =
+  py3-dj22
+
+[tox:jenkins]
+envlist =
+  pylint
+  code-style
+  py3-dj22
+
+
+[testenv]
+setenv =
+  AUTHENTIC2_SETTINGS_FILE=tests/settings.py
+  DJANGO_SETTINGS_MODULE=authentic2.settings
+
+  JUNIT={tty::-o junit_suite_name={envname} --junit-xml=junit-{envname}.xml}
+  COVERAGE={tty::--junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=src/ --cov-config .coveragerc}
+passenv=
+  BRANCH_NAME
+  # support for pg_virtualenv
+  PGPORT
+  PGHOST
+  PGUSER
+  PGPASSWORD
+usedevelop = true
+deps =
+  !local: https://git.entrouvert.org/authentic.git/snapshot/authentic-main.tar.gz
+  local: ../authentic2
+  # dependency constraints for authentic
+  py3: file-magic
+  djangorestframework>=3.12,<3.13
+  dj22: django<2.3
+  django-tables<2.0
+  psycopg2-binary<2.9
+  oldldap: python-ldap<3
+  ldaptools
+
+  # pytest requirements
+  pytest
+  pytest-cov
+  pytest-django
+  pytest-freezegun
+  pytest-random
+  django-webtest
+  pyquery
+  astroid!=2.5.7
+
+commands =
+  py3: ./getlasso3.sh
+  ./check-migrations.sh
+  py.test {env:COVERAGE:} {env:JUNIT:} {tty:--sw:} {posargs:tests/}
+
+[testenv:pylint]
+usedevelop = true
+basepython = python3
+deps =
+    https://git.entrouvert.org/authentic.git/snapshot/authentic-main.tar.gz
+    Django<2.3
+    pylint<1.8
+    pylint-django<0.8.1
+commands =
+    /bin/bash -c "./pylint.sh src/*/"
+
+[testenv:code-style]
+skip_install = true
+deps =
+  pre-commit
+commands =
+  pre-commit run --all-files --show-diff-on-failure
+
+[pytest]
+junit_family=xunit2
+filterwarnings =
+  once
+  ignore:::django\..*
+  ignore:::authentic2\..*
+  ignore:::rest_framework\..*
+  ignore:::gettext\..*