python3: update tox to check against both python versions

Jenkinsfile

@@ -16,7 +16,7 @@ pipeline {
                         utils.publish_coverage_native('index.html')
                         utils.publish_pylint('pylint.out')
                     }
-                    junit '*_results.xml'
+                    mergeJunitResults()
                 }
             }
         }

debian/debian_config.py

@@ -1,4 +1,4 @@
-# This file is sourced by "execfile" from welco.settings
+# This file is sourced by "exec(open(..." from welco.settings
 
 import os
 
@@ -10,12 +10,12 @@ INSTALLED_APPS += ('mellon',)
 #
 # hobotization (multitenant)
 #
-execfile('/usr/lib/hobo/debian_config_common.py')
+exec(open('/usr/lib/hobo/debian_config_common.py').read())
 
 #
 # local settings
 #
-execfile(os.path.join(ETC_DIR, 'settings.py'))
+exec(open(os.path.join(ETC_DIR, 'settings.py')).read())
 
 # run additional settings snippets
-execfile('/usr/lib/hobo/debian_config_settings_d.py')
+exec(open('/usr/lib/hobo/debian_config_settings_d.py').read())

setup.py

@@ -112,7 +112,7 @@ setup(
         'gadjo',
         'django-ckeditor<=4.5.3',
         'django-haystack<2.8',
-        'django-reversion>=2.0',
+        'django-reversion>=2.0,<3',
         'django-taggit',
         'djangorestframework>=3.3, <3.7',
         'requests',

tests/test_source_maarch.py

@@ -19,6 +19,7 @@ import json
 import pytest
 
 from django.contrib.auth.models import User
+from django.utils.encoding import force_text
 
 from httmock import urlmatch, HTTMock
 
@@ -50,32 +51,34 @@ class BaseMock(object):
 
 class MaarchMock(BaseMock):
     def list_endpoint(self, url, request):
-        self.requests.append(('list_endpoint', url, request, json.loads(request.body)))
+        self.requests.append(('list_endpoint', url, request, json.loads(force_text(request.body))))
         return {
             'content': json.dumps(self.next_response()),
             'headers': {
                 'content-type': 'application/json',
             },
+            'status_code': 200,
         }
     list_endpoint.path = '^/rest/res/list$'
 
     def update_external_infos(self, url, request):
-        self.requests.append(('update_external_infos', url, request, json.loads(request.body)))
+        self.requests.append(('update_external_infos', url, request, json.loads(force_text(request.body))))
         return json.dumps({})
     update_external_infos.path = '^/rest/res/externalInfos$'
 
     def update_status(self, url, request):
-        self.requests.append(('update_status', url, request, json.loads(request.body)))
+        self.requests.append(('update_status', url, request, json.loads(force_text(request.body))))
         return {
             'content': json.dumps(self.next_response()),
             'headers': {
                 'content-type': 'application/json',
             },
+            'status_code': 200,
         }
     update_status.path = '^/rest/res/resource/status$'
 
     def post_courrier(self, url, request):
-        self.requests.append(('post_courrier', url, request, json.loads(request.body)))
+        self.requests.append(('post_courrier', url, request, json.loads(force_text(request.body))))
     post_courrier.path = '^/rest/res$'
 
 
@@ -152,7 +155,7 @@ def test_utils(maarch):
     assert welco_maarch_obj.grc_refused_status == 'GRCREFUSED'
 
 
-PDF_MOCK = '%PDF-1.4 ...'
+PDF_MOCK = b'%PDF-1.4 ...'
 
 
 def test_feed(settings, app, maarch, wcs, user):
@@ -173,7 +176,7 @@ def test_feed(settings, app, maarch, wcs, user):
             'resources': [
                 {
                     'res_id': 1,
-                    'fileBase64Content': base64.b64encode(PDF_MOCK),
+                    'fileBase64Content': force_text(base64.b64encode(PDF_MOCK)),
                 }
             ],
         })
@@ -250,7 +253,7 @@ def test_feed(settings, app, maarch, wcs, user):
     user.set_password('test')
     user.save()
     # verify authentication error
-    response = app.post_json('/api/mail/response/', params={}, status=403)
+    response = app.post_json('/api/mail/response/', params={}, status=(401, 403))
     app.authorization = ('Basic', ('test', 'test'))
     # verify serializer error
     response = app.post_json('/api/mail/response/', params={}, status=400)

tests/test_source_phone.py

@@ -21,6 +21,8 @@ import pytest
 
 from django.core.urlresolvers import reverse
 from django.test import override_settings
+from django.utils import six
+from django.utils.encoding import force_text
 from django.utils.timezone import now, timedelta
 
 from welco.sources.phone import models
@@ -42,7 +44,7 @@ def test_call_start_stop(client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneCall.objects.count() == 1
     assert models.PhoneCall.objects.filter(
         caller='0033699999999',
@@ -53,7 +55,7 @@ def test_call_start_stop(client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneCall.objects.count() == 2
     assert models.PhoneCall.objects.filter(
         caller='0033699999999',
@@ -69,7 +71,7 @@ def test_call_start_stop(client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneCall.objects.count() == 2
     assert models.PhoneCall.objects.filter(
         caller='0033699999999',
@@ -80,7 +82,7 @@ def test_call_start_stop(client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneCall.objects.count() == 2
     assert models.PhoneCall.objects.filter(
         caller='0033699999999',
@@ -140,7 +142,7 @@ def test_current_calls(user, client):
                                content_type='application/json')
         assert response.status_code == 200
         assert response['content-type'] == 'application/json'
-        assert json.loads(response.content) == {'err': 0}
+        assert response.json() == {'err': 0}
 
     # register user to some lines
     # then remove from some
@@ -152,7 +154,7 @@ def test_current_calls(user, client):
     response = client.get(reverse('phone-current-calls'))
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    payload = json.loads(response.content)
+    payload = response.json()
     assert isinstance(payload, dict)
     assert set(payload.keys()) == set(['err', 'data'])
     assert payload['err'] == 0
@@ -166,13 +168,13 @@ def test_current_calls(user, client):
     assert len(data['all-lines']) == 10
     for call in data['calls']:
         assert set(call.keys()) <= set(['caller', 'callee', 'start', 'data'])
-        assert isinstance(call['caller'], unicode)
-        assert isinstance(call['callee'], unicode)
-        assert isinstance(call['start'], unicode)
+        assert isinstance(call['caller'], six.string_types)
+        assert isinstance(call['callee'], six.string_types)
+        assert isinstance(call['start'], six.string_types)
         if 'data' in call:
             assert isinstance(call['data'], dict)
-    assert len([call for call in data['lines'] if isinstance(call, unicode)]) == 5
-    assert len([call for call in data['all-lines'] if isinstance(call, unicode)]) == 10
+    assert len([call for call in data['lines'] if isinstance(call, six.string_types)]) == 5
+    assert len([call for call in data['all-lines'] if isinstance(call, six.string_types)]) == 10
 
     # unregister user to all remaining lines
     for number in range(0, 5):
@@ -180,7 +182,7 @@ def test_current_calls(user, client):
     response = client.get(reverse('phone-current-calls'))
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    payload = json.loads(response.content)
+    payload = response.json()
     assert isinstance(payload, dict)
     assert set(payload.keys()) == set(['err', 'data'])
     assert payload['err'] == 0
@@ -201,7 +203,7 @@ def test_take_release_line(user, client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneLine.objects.count() == 1
     assert models.PhoneLine.objects.filter(
         users=user, callee='102').count() == 1
@@ -209,7 +211,7 @@ def test_take_release_line(user, client):
                            content_type='application/json')
     assert response.status_code == 200
     assert response['content-type'] == 'application/json'
-    assert json.loads(response.content) == {'err': 0}
+    assert response.json() == {'err': 0}
     assert models.PhoneLine.objects.count() == 1
     assert models.PhoneLine.objects.filter(
         users=user, callee='102').count() == 0
@@ -219,18 +221,18 @@ def test_phone_zone(user, client):
     client.login(username='toto', password='toto')
     response = client.get(reverse('phone-zone'))
     assert response.status_code == 200
-    assert 'You do not have a phoneline configured' in response.content
+    assert 'You do not have a phoneline configured' in force_text(response.content)
 
     models.PhoneLine.take(callee='102', user=user)
 
     response = client.get(reverse('phone-zone'))
     assert response.status_code == 200
-    assert 'You do not have a phoneline configured' not in response.content
-    assert '<li>102' in response.content
-    assert 'data-callee="102"' in response.content
+    assert 'You do not have a phoneline configured' not in force_text(response.content)
+    assert '<li>102' in force_text(response.content)
+    assert 'data-callee="102"' in force_text(response.content)
     currents = re.search('<div id="source-mainarea" '
                          'data-current-calls="/api/phone/current-calls/">'
-                         '(.*?)</div>', response.content, flags=re.DOTALL)
+                         '(.*?)</div>', force_text(response.content), flags=re.DOTALL)
     assert currents.group(1).strip() == ''
 
     # create a call
@@ -240,7 +242,7 @@ def test_phone_zone(user, client):
     assert response.status_code == 200
     response = client.get(reverse('phone-zone'))
     assert response.status_code == 200
-    assert '<h1>Current Call: <strong>003369999999</strong></h1>' in response.content
+    assert '<h1>Current Call: <strong>003369999999</strong></h1>' in force_text(response.content)
 
     # simulate a mellon user
     session = client.session
@@ -248,19 +250,19 @@ def test_phone_zone(user, client):
     session.save()
     response = client.get(reverse('phone-zone'))
     assert response.status_code == 200
-    assert 'agent007' not in response.content
-    assert 'data-callee="agent007"' not in response.content
-    assert '<li>102' in response.content
-    assert 'data-callee="102"' in response.content
+    assert 'agent007' not in force_text(response.content)
+    assert 'data-callee="agent007"' not in force_text(response.content)
+    assert '<li>102' in force_text(response.content)
+    assert 'data-callee="102"' in force_text(response.content)
 
     with override_settings(PHONE_AUTOTAKE_MELLON_USERNAME=True):
         response = client.get(reverse('phone-zone'))
         assert response.status_code == 200
-        assert '<h1>Current Call: <strong>003369999999</strong></h1>' in response.content
-        assert 'agent007' in response.content
-        assert 'data-callee="agent007"' in response.content
-        assert '<li>102' in response.content
-        assert 'data-callee="102"' in response.content
+        assert '<h1>Current Call: <strong>003369999999</strong></h1>' in force_text(response.content)
+        assert 'agent007' in force_text(response.content)
+        assert 'data-callee="agent007"' in force_text(response.content)
+        assert '<li>102' in force_text(response.content)
+        assert 'data-callee="102"' in force_text(response.content)
 
 
 def test_call_expiration(user, client):
@@ -277,7 +279,7 @@ def test_call_expiration(user, client):
     client.login(username='toto', password='toto')
     response = client.get(reverse('phone-current-calls'))
     assert response.status_code == 200
-    payload = json.loads(response.content)
+    payload = response.json()
     assert payload['err'] == 0
     assert len(payload['data']['calls']) == 1
 
@@ -288,7 +290,7 @@ def test_call_expiration(user, client):
     # get list of calls without expiration
     response = client.get(reverse('phone-current-calls'))
     assert response.status_code == 200
-    payload = json.loads(response.content)
+    payload = response.json()
     assert payload['err'] == 0
     assert len(payload['data']['calls']) == 1 # still here
 
@@ -296,7 +298,7 @@ def test_call_expiration(user, client):
     with override_settings(PHONE_MAX_CALL_DURATION=2):
         response = client.get(reverse('phone-current-calls'))
         assert response.status_code == 200
-        payload = json.loads(response.content)
+        payload = response.json()
         assert payload['err'] == 0
         assert len(payload['data']['calls']) == 0 # call is expired
 

tox.ini

@@ -1,6 +1,6 @@
 [tox]
-envlist = py27-django111
-toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/welco/
+envlist = py27-django111-coverage-pylint,py3-django111
+toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/welco/{env:BRANCH_NAME:}
 
 [testenv]
 usedevelop =
@@ -9,12 +9,13 @@ setenv =
   DJANGO_SETTINGS_MODULE=welco.settings
   WELCO_SETTINGS_FILE=tests/settings.py
   fast: FAST=--nomigrations
+  coverage: COVERAGE=--junitxml=junit-{envname}.xml --cov-report xml --cov-report html --cov=welco/
 deps =
   django111: django>=1.11,<1.12
   pytest-cov
   pytest-django
-  pytest<4.1
-  attrs<19.2
+  pytest!=5.3.3
+  attrs
   WebTest
   mock
   httmock
@@ -26,5 +27,5 @@ deps =
   lxml
   git+https://git.entrouvert.org/debian/django-ckeditor.git
 commands =
-  django111: ./pylint.sh welco/
-  django111: py.test {posargs: --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=welco/ tests/}
+  pylint: ./pylint.sh welco/
+  py.test {env:COVERAGE:} {posargs:tests/}

welco/kb/search_indexes.py

@@ -14,9 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from HTMLParser import HTMLParser
-
 from django.utils.html import strip_tags
+from django.utils.six.moves.html_parser import HTMLParser
 
 from haystack import indexes
 

welco/settings.py

@@ -220,4 +220,4 @@ REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'] = ['rest_framework.authenticati
 local_settings_file = os.environ.get('WELCO_SETTINGS_FILE',
         os.path.join(os.path.dirname(__file__), 'local_settings.py'))
 if os.path.exists(local_settings_file):
-    execfile(local_settings_file)
+    exec(open(local_settings_file).read())

welco/sources/mail/maarch.py

@@ -14,10 +14,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import urlparse
 import base64
 
 from dateutil.parser import parse as parse_datetime
+from django.utils import six
+from django.utils.six.moves.urllib import parse as urlparse
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -87,8 +88,8 @@ class MaarchCourrier(object):
             excluded_keys = ['content', 'format', 'status', 'maarch_courrier', 'pk']
             data = {key: self.__dict__[key] for key in self.__dict__ if key not in excluded_keys}
             if data:
-                for key, value in data.iteritems():
-                    if isinstance(value, basestring):
+                for key, value in data.items():
+                    if isinstance(value, six.string_types):
                         d.append({'column': key, 'value': value, 'type': 'string'})
                     elif isinstance(value, int):
                         d.append({'column': key, 'value': str(value), 'type': 'int'})

welco/sources/phone/views.py

@@ -24,6 +24,8 @@ from django.template import RequestContext
 from django.views.decorators.csrf import csrf_exempt
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponseBadRequest, HttpResponse
+from django.utils import six
+from django.utils.encoding import force_text
 from django.utils.timezone import now
 from django.views.generic import TemplateView
 
@@ -83,20 +85,20 @@ def call_event(request):
     '''
     logger = logging.getLogger(__name__)
     try:
-        payload = json.loads(request.body)
+        payload = json.loads(force_text(request.body))
         assert isinstance(payload, dict), 'payload is not a JSON object'
         assert set(payload.keys()) <= set(['event', 'caller', 'callee', 'data']), \
             'payload keys must be "event", "caller", "callee" and optionnaly "data"'
         assert set(['event', 'caller', 'callee']) <= set(payload.keys()), \
             'payload keys must be "event", "caller", "callee" and optionnaly "data"'
         assert payload['event'] in ('start', 'stop'), 'event must be "start" or "stop"'
-        assert isinstance(payload['caller'], unicode), 'caller must be a string'
-        assert isinstance(payload['callee'], unicode), 'callee must be a string'
+        assert isinstance(payload['caller'], six.string_types), 'caller must be a string'
+        assert isinstance(payload['callee'], six.string_types), 'callee must be a string'
         if 'data' in payload:
             assert isinstance(payload['data'], dict), 'data must be a JSON object'
-    except (TypeError, ValueError, AssertionError), e:
+    except (TypeError, ValueError, AssertionError) as e:
         return HttpResponseBadRequest(json.dumps({'err': 1, 'msg':
-                                                  unicode(e)}),
+                                                  force_text(e)}),
                                       content_type='application/json')
     # janitoring: stop active calls to the callee
     if settings.PHONE_ONE_CALL_PER_CALLEE:
@@ -196,12 +198,12 @@ def take_line(request):
     '''
     logger = logging.getLogger(__name__)
     try:
-        payload = json.loads(request.body)
+        payload = json.loads(force_text(request.body))
         assert isinstance(payload, dict), 'payload is not a JSON object'
-        assert payload.keys() == ['callee'], 'payload must have only one key: callee'
-    except (TypeError, ValueError, AssertionError), e:
+        assert list(payload.keys()) == ['callee'], 'payload must have only one key: callee'
+    except (TypeError, ValueError, AssertionError) as e:
         return HttpResponseBadRequest(json.dumps({'err': 1, 'msg':
-                                                  unicode(e)}),
+                                                  force_text(e)}),
                                       content_type='application/json')
     PhoneLine.take(payload['callee'], request.user)
     logger.info(u'user %s took line %s', request.user, payload['callee'])
@@ -217,12 +219,12 @@ def release_line(request):
     '''
     logger = logging.getLogger(__name__)
     try:
-        payload = json.loads(request.body)
+        payload = json.loads(force_text(request.body))
         assert isinstance(payload, dict), 'payload is not a JSON object'
-        assert payload.keys() == ['callee'], 'payload must have only one key: callee'
-    except (TypeError, ValueError, AssertionError), e:
+        assert list(payload.keys()) == ['callee'], 'payload must have only one key: callee'
+    except (TypeError, ValueError, AssertionError) as e:
         return HttpResponseBadRequest(json.dumps({'err': 1, 'msg':
-                                                  unicode(e)}),
+                                                  force_text(e)}),
                                       content_type='application/json')
     PhoneLine.release(payload['callee'], request.user)
     logger.info(u'user %s released line %s', request.user, payload['callee'])

welco/utils.py

@@ -22,13 +22,14 @@ import json
 import random
 import re
 import requests
-import urllib
-import urlparse
 
 from django.conf import settings
 from django.core.cache import cache
 from django.http import HttpResponse, HttpResponseBadRequest
-from django.utils.http import urlencode
+from django.utils.encoding import smart_bytes
+from django.utils.http import urlencode, quote
+from django.utils.six.moves.urllib import parse as urlparse
+
 
 def sign_url(url, key, algo='sha256', timestamp=None, nonce=None):
     parsed = urlparse.urlparse(url)
@@ -44,17 +45,17 @@ def sign_query(query, key, algo='sha256', timestamp=None, nonce=None):
     new_query = query
     if new_query:
         new_query += '&'
-    new_query += urllib.urlencode((
+    new_query += urlencode((
         ('algo', algo),
         ('timestamp', timestamp),
         ('nonce', nonce)))
     signature = base64.b64encode(sign_string(new_query, key, algo=algo))
-    new_query += '&signature=' + urllib.quote(signature)
+    new_query += '&signature=' + quote(signature)
     return new_query
 
 def sign_string(s, key, algo='sha256', timedelta=30):
     digestmod = getattr(hashlib, algo)
-    hash = hmac.HMAC(str(key), digestmod=digestmod, msg=s)
+    hash = hmac.HMAC(smart_bytes(key), digestmod=digestmod, msg=smart_bytes(s))
     return hash.digest()
 
 def get_wcs_services():
@@ -78,7 +79,7 @@ def get_wcs_json(wcs_url, path, wcs_site, params={}):
 
 def get_wcs_options(url, condition=None, params={}):
     categories = {}
-    for wcs_key, wcs_site in get_wcs_services().iteritems():
+    for wcs_key, wcs_site in get_wcs_services().items():
         site_title = wcs_site.get('title')
         response_json = get_wcs_json(wcs_site.get('url'), url, wcs_site, params)
         if type(response_json) is dict:
@@ -99,7 +100,7 @@ def get_wcs_options(url, condition=None, params={}):
             categories[category_key].append((reference, title))
     options = []
     for category in sorted(categories.keys()):
-        options.append((category, sorted(categories[category], lambda x, y: cmp(x[1], y[1]))))
+        options.append((category, sorted(categories[category], key=lambda x: x[1])))
     return options
 
 def get_wcs_formdef_details(formdef_reference):
@@ -134,7 +135,7 @@ def push_wcs_formdata(request, formdef_reference, context=None):
     if request.session.get('mellon_session'):
         mellon = request.session['mellon_session']
         nameid = mellon['name_id_content']
-        url += '&NameID=' + urllib.quote(nameid)
+        url += '&NameID=' + quote(nameid)
 
     url = sign_url(url, wcs_site.get('secret'))
 
@@ -153,17 +154,12 @@ def get_wcs_data(endpoint, params=None):
         wcs_site_url += '/'
     url = wcs_site_url + endpoint
 
-    if params:
-        params = params.copy()
-        for param, value in params.items():
-            if isinstance(value, unicode):
-                params[param] = value.encode('utf-8')
-    else:
+    if not params:
         params = {}
 
     params['orig'] = wcs_site.get('orig')
     if params:
-        url += '?' + urllib.urlencode(params.items())
+        url += '?' + urlencode(params.items())
     url = sign_url(url, wcs_site.get('secret'))
 
     response = requests.get(url)

welco/views.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
-import urllib
 
 from django.conf import settings
 from django.contrib.auth import logout as auth_logout
@@ -29,6 +28,7 @@ from django.shortcuts import resolve_url
 from django import template
 from django.template import RequestContext
 from django.utils.encoding import force_text
+from django.utils.http import quote
 from django.utils.translation import ugettext_lazy as _
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import TemplateView
@@ -38,9 +38,9 @@ try:
 except ImportError:
     get_idps = lambda: []
 
-from sources.mail.views import Home as MailHome
-from sources.phone.views import Home as PhoneHome
-from sources.counter.views import Home as CounterHome
+from .sources.mail.views import Home as MailHome
+from .sources.phone.views import Home as PhoneHome
+from .sources.counter.views import Home as CounterHome
 from .qualif.models import Association
 from .kb.views import HomeZone as KbHomeZone, check_user_perms as check_kb_user_perms
 from .contacts.views import HomeZone as ContactsHomeZone
@@ -52,7 +52,7 @@ def login(request, *args, **kwargs):
         if not 'next' in request.GET:
             return HttpResponseRedirect(resolve_url('mellon_login'))
         return HttpResponseRedirect(resolve_url('mellon_login') + '?next='
-                                    + urllib.quote(request.GET.get('next')))
+                                    + quote(request.GET.get('next')))
     return auth_views.login(request, *args, **kwargs)
 
 def logout(request, next_page=None):
@@ -180,7 +180,7 @@ def create_formdata(request, *args, **kwargs):
     qualif = Association.objects.get(id=kwargs.get('pk'))
     try:
         qualif.push(request)
-    except Exception, e:
+    except Exception as e:
         json.dump({'err': 1, 'msg': str(e)}, response)
         return response
     json.dump({'result': 'ok', 'url': qualif.formdata_url}, response)