use unicode_literals (#34008)

mellon/adapters.py

@@ -13,8 +13,11 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 from xml.etree import ElementTree as ET
 import hashlib
+
 import logging
 import os
 import threading
@@ -230,15 +233,15 @@ class DefaultAdapter(object):
         try:
             doc = ET.fromstring(metadata)
         except (TypeError, ET.ParseError):
-            logger.error(u'METADATA of %d-th idp is invalid', i)
+            logger.error('METADATA of %d-th idp is invalid', i)
             return None
         if doc.tag != '{%s}EntityDescriptor' % lasso.SAML2_METADATA_HREF:
-            logger.error(u'METADATA of %d-th idp has no EntityDescriptor root tag', i)
+            logger.error('METADATA of %d-th idp has no EntityDescriptor root tag', i)
             return None
 
         if 'entityID' not in doc.attrib:
             logger.error(
-                u'METADATA of %d-th idp has no entityID attribute on its root tag', i)
+                'METADATA of %d-th idp has no entityID attribute on its root tag', i)
             return None
         return doc.attrib['entityID']
 
@@ -264,12 +267,12 @@ class DefaultAdapter(object):
             username = force_text(username_template).format(
                 realm=realm, attributes=saml_attributes, idp=idp)[:30]
         except ValueError:
-            logger.error(u'invalid username template %r', username_template)
+            logger.error('invalid username template %r', username_template)
         except (AttributeError, KeyError, IndexError) as e:
             logger.error(
-                u'invalid reference in username template %r: %s', username_template, e)
+                'invalid reference in username template %r: %s', username_template, e)
         except Exception:
-            logger.exception(u'unknown error when formatting username')
+            logger.exception('unknown error when formatting username')
         else:
             return username
 
@@ -380,15 +383,15 @@ class DefaultAdapter(object):
                     logger.debug('looking for users by attribute %r and user field %r with value %r: not found',
                                  saml_attribute, user_field, value)
                     continue
-                logger.info(u'looking for user by attribute %r and user field %r with value %r: found %s',
+                logger.info('looking for user by attribute %r and user field %r with value %r: found %s',
                             saml_attribute, user_field, value, display_truncated_list(users_found))
                 users.update(users_found)
         if len(users) == 1:
             user = list(users)[0]
-            logger.info(u'looking for user by attributes %r: found user %s', lookup_by_attributes, user)
+            logger.info('looking for user by attributes %r: found user %s', lookup_by_attributes, user)
             return user
         elif len(users) > 1:
-            logger.warning(u'looking for user by attributes %r: too many users found(%d), failing',
+            logger.warning('looking for user by attributes %r: too many users found(%d), failing',
                            lookup_by_attributes, len(users))
         return None
 
@@ -413,10 +416,10 @@ class DefaultAdapter(object):
             try:
                 value = force_text(tpl).format(realm=realm, attributes=saml_attributes, idp=idp)
             except ValueError:
-                logger.warning(u'invalid attribute mapping template %r', tpl)
+                logger.warning('invalid attribute mapping template %r', tpl)
             except (AttributeError, KeyError, IndexError, ValueError) as e:
                 logger.warning(
-                    u'invalid reference in attribute mapping template %r: %s', tpl, e)
+                    'invalid reference in attribute mapping template %r: %s', tpl, e)
             else:
                 model_field = user._meta.get_field(field)
                 if hasattr(model_field, 'max_length'):
@@ -425,7 +428,7 @@ class DefaultAdapter(object):
                     old_value = getattr(user, field)
                     setattr(user, field, value)
                     attribute_set = True
-                    logger.info(u'set field %s of user %s to value %r (old value %r)', field, user, value, old_value)
+                    logger.info('set field %s of user %s to value %r (old value %r)', field, user, value, old_value)
         if attribute_set:
             user.save()
 
@@ -478,10 +481,10 @@ class DefaultAdapter(object):
                 groups.append(group)
             for group in Group.objects.filter(pk__in=[g.pk for g in groups]).exclude(user=user):
                 logger.info(
-                    u'adding group %s (%s) to user %s (%s)', group, group.pk, user, user.pk)
+                    'adding group %s (%s) to user %s (%s)', group, group.pk, user, user.pk)
                 User.groups.through.objects.get_or_create(group=group, user=user)
             qs = User.groups.through.objects.exclude(
                 group__pk__in=[g.pk for g in groups]).filter(user=user)
             for rel in qs:
-                logger.info(u'removing group %s (%s) from user %s (%s)', rel.group, rel.group.pk, rel.user, rel.user.pk)
+                logger.info('removing group %s (%s) from user %s (%s)', rel.group, rel.group.pk, rel.user, rel.user.pk)
             qs.delete()

mellon/backends.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 from django.contrib.auth.backends import ModelBackend
 
 from . import utils

mellon/middleware.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 from django.utils.http import urlencode
 from django.http import HttpResponseRedirect
 from django.core.urlresolvers import reverse

mellon/models.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings

mellon/urls.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from django.conf.urls import url
 import django
 

mellon/utils.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import logging
 import datetime
 import importlib
@@ -99,7 +101,7 @@ def create_server(request):
                 try:
                     server.addProviderFromBuffer(lasso.PROVIDER_ROLE_IDP, idp['METADATA'])
                 except lasso.Error as e:
-                    logger.error(u'bad metadata in idp %s, %s', idp['ENTITY_ID'], e)
+                    logger.error('bad metadata in idp %s, %s', idp['ENTITY_ID'], e)
         cache[root] = server
         settings._MELLON_SERVER_CACHE = cache
     return settings._MELLON_SERVER_CACHE.get(root)

mellon/views.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import logging
 import requests
 import lasso
@@ -114,9 +116,9 @@ class ProfileMixin(object):
 
     def show_message_status_is_not_success(self, profile, prefix):
         status_codes, idp_message = utils.get_status_codes_and_message(profile)
-        args = [u'%s: status is not success codes: %r', prefix, status_codes]
+        args = ['%s: status is not success codes: %r', prefix, status_codes]
         if idp_message:
-            args[0] += u' message: %s'
+            args[0] += ' message: %s'
             args.append(idp_message)
         self.log.warning(*args)
 
@@ -196,9 +198,9 @@ class LoginView(ProfileMixin, LogMixin, View):
             for at in ats.attribute:
                 values = attributes.setdefault(at.name, [])
                 for value in at.attributeValue:
-                    content = [any.exportToXml() for any in value.any]
+                    contents = [lasso_decode(any.exportToXml()) for any in value.any]
-                    content = ''.join(content)
+                    content = ''.join(contents)
-                    values.append(lasso_decode(content))
+                    values.append(content)
         attributes['issuer'] = login.remoteProviderId
         if login.nameIdentifier:
             name_id = login.nameIdentifier
@@ -295,8 +297,8 @@ class LoginView(ProfileMixin, LogMixin, View):
         try:
             login.initRequest(message, method)
         except lasso.ProfileInvalidArtifactError:
-            self.log.warning(u'artifact is malformed %r', artifact)
+            self.log.warning('artifact is malformed %r', artifact)
-            return HttpResponseBadRequest(u'artifact is malformed %r' % artifact)
+            return HttpResponseBadRequest('artifact is malformed %r' % artifact)
         except lasso.ServerProviderNotFoundError:
             self.log.warning('no entity id found for artifact %s', artifact)
             return HttpResponseBadRequest(
@@ -425,14 +427,15 @@ class LoginView(ProfileMixin, LogMixin, View):
                 # lasso>2.5.1 introduced a better API
                 if hasattr(authn_request.extensions, 'any'):
                     authn_request.extensions.any = (
-                        '<eo:next_url xmlns:eo="https://www.entrouvert.com/">%s</eo:next_url>' % eo_next_url,)
+                        str('<eo:next_url xmlns:eo="https://www.entrouvert.com/">%s</eo:next_url>' % eo_next_url),
+                    )
                 else:
                     authn_request.extensions.setOriginalXmlnode(
-                        '''<samlp:Extensions
+                        str('''<samlp:Extensions
                                  xmlns:samlp="urn:oasis:names:tc:SAML:2.0:protocol"
                                  xmlns:eo="https://www.entrouvert.com/">
                                <eo:next_url>%s</eo:next_url>
-                            </samlp:Extensions>''' % eo_next_url
+                            </samlp:Extensions>''' % eo_next_url)
                     )
             self.set_next_url(next_url)
             self.add_login_hints(idp, authn_request, request=request, next_url=next_url)
@@ -502,7 +505,7 @@ class LogoutView(ProfileMixin, LogMixin, View):
             self.log.warning('error validating logout request: %r' % e)
         issuer = request.session.get('mellon_session', {}).get('issuer')
         if issuer == logout.remoteProviderId:
-            self.log.info(u'user logged out by IdP SLO request')
+            self.log.info('user logged out by IdP SLO request')
             auth.logout(request)
         try:
             logout.buildResponseMsg()
@@ -539,7 +542,7 @@ class LogoutView(ProfileMixin, LogMixin, View):
                     # set next_url after local logout, as the session is wiped by auth.logout
                     if logout:
                         self.set_next_url(next_url)
-                    self.log.info(u'user logged out, SLO request sent to IdP')
+                    self.log.info('user logged out, SLO request sent to IdP')
         else:
             self.log.warning('logout refused referer %r is not of the same origin', referer)
         return HttpResponseRedirect(next_url)

tests/test_default_adapter.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import datetime
 import re
 import lasso
@@ -120,9 +122,9 @@ def test_lookup_user_transaction(transactional_db, concurrency, idp, saml_attrib
 def test_provision_user_attributes(settings, django_user_model, idp, saml_attributes, caplog):
     settings.MELLON_IDENTITY_PROVIDERS = [idp]
     settings.MELLON_ATTRIBUTE_MAPPING = {
-        'email': u'{attributes[email][0]}',
+        'email': '{attributes[email][0]}',
-        'first_name': u'{attributes[first_name][0]}',
+        'first_name': '{attributes[first_name][0]}',
-        'last_name': u'{attributes[last_name][0]}',
+        'last_name': '{attributes[last_name][0]}',
     }
     user = SAMLBackend().authenticate(saml_attributes=saml_attributes)
     assert user.username == 'x' * 30
@@ -205,7 +207,7 @@ def test_provision_long_attribute(settings, django_user_model, idp, saml_attribu
     assert len(caplog.records) == 4
     assert 'created new user' in caplog.text
     assert 'set field first_name' in caplog.text
-    assert 'to value %r ' % (u'y' * 30) in caplog.text
+    assert 'to value %r ' % ('y' * 30) in caplog.text
     assert 'set field last_name' in caplog.text
     assert 'set field email' in caplog.text
 

tests/test_sso_slo.py

@@ -13,6 +13,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
+import datetime
 import re
 import base64
 import zlib
@@ -25,6 +28,7 @@ from pytest import fixture
 from django.core.urlresolvers import reverse
 from django.utils import six
 from django.utils.six.moves.urllib import parse as urlparse
+from django.utils.encoding import force_str
 
 from mellon.utils import create_metadata
 
@@ -100,10 +104,40 @@ class MockIdp(object):
             pass
         else:
             login.buildAssertion(lasso.SAML_AUTHENTICATION_METHOD_PASSWORD,
-                                 "FIXME",
+                                 datetime.datetime.now().isoformat(),
-                                 "FIXME",
+                                 None,
-                                 "FIXME",
+                                 datetime.datetime.now().isoformat(),
-                                 "FIXME")
+                                 datetime.datetime.now().isoformat())
+
+            def add_attribute(name, *values, **kwargs):
+                fmt = kwargs.get('fmt', lasso.SAML2_ATTRIBUTE_NAME_FORMAT_BASIC)
+                statements = login.response.assertion[0].attributeStatement or [lasso.Saml2AttributeStatement()]
+                statement = statements[0]
+                login.response.assertion[0].attributeStatement = statements
+                attributes = list(statement.attribute)
+                for attribute in attributes:
+                    if attribute.name == name and attribute.nameFormat == fmt:
+                        break
+                else:
+                    attribute = lasso.Saml2Attribute()
+                    attributes.append(attribute)
+                    statement.attribute = attributes
+                attribute_values = list(attribute.attributeValue)
+                atv = lasso.Saml2AttributeValue()
+                attribute_values.append(atv)
+                attribute.attributeValue = attribute_values
+                value_any = []
+                for value in values:
+                    if isinstance(value, lasso.Node):
+                        value_any.append(value)
+                    else:
+                        mtn = lasso.MiscTextNode.newWithString(force_str(value))
+                        mtn.textChild = True
+                        value_any.append(mtn)
+                atv.any = value_any
+            add_attribute('email', 'john', '.doe@gmail.com')
+            add_attribute('wtf', 'john', lasso.MiscTextNode.newWithXmlNode('<a>coucou</a>'))
+
         if not auth_result and msg:
             login.response.status.statusMessage = msg
         if login.protocolProfile == lasso.LOGIN_PROTOCOL_PROFILE_BRWS_ART:
@@ -174,7 +208,7 @@ def test_sso_request_denied(db, app, idp, caplog, sp_settings):
     url, body, relay_state = idp.process_authn_request_redirect(
         response['Location'],
         auth_result=False,
-        msg=u'User is not allowed to login')
+        msg='User is not allowed to login')
     assert not relay_state
     assert url.endswith(reverse('mellon_login'))
     response = app.post(reverse('mellon_login'), params={'SAMLResponse': body, 'RelayState': relay_state})
@@ -195,7 +229,7 @@ def test_sso_request_denied_artifact(db, app, caplog, sp_settings, idp_metadata,
     url, body, relay_state = idp.process_authn_request_redirect(
         response['Location'],
         auth_result=False,
-        msg=u'User is not allowed to login')
+        msg='User is not allowed to login')
     assert not relay_state
     assert body is None
     assert reverse('mellon_login') in url

tests/test_utils.py

@@ -13,12 +13,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import datetime
 
 import mock
 import lasso
 
 from mellon.utils import create_metadata, iso8601_to_datetime, flatten_datetime
+from mellon.views import check_next_url
 from xml_utils import assert_xml_constraints
 
 
@@ -84,9 +87,19 @@ def test_flatten_datetime():
     d = {
         'x': datetime.datetime(2010, 10, 10, 10, 10, 34),
         'y': 1,
-        'z': 'uu',
+        'z': 'u',
     }
     assert set(flatten_datetime(d).keys()) == set(['x', 'y', 'z'])
     assert flatten_datetime(d)['x'] == '2010-10-10T10:10:34'
     assert flatten_datetime(d)['y'] == 1
-    assert flatten_datetime(d)['z'] == 'uu'
+    assert flatten_datetime(d)['z'] == 'u'
+
+
+def test_check_next_url(rf):
+    assert not check_next_url(rf.get('/'), u'')
+    assert not check_next_url(rf.get('/'), None)
+    assert not check_next_url(rf.get('/'), u'\x00')
+    assert not check_next_url(rf.get('/'), u'\u010e')
+    assert not check_next_url(rf.get('/'), u'https://example.invalid/')
+    # default hostname is testserver
+    assert check_next_url(rf.get('/'), u'http://testserver/ok/')

tests/test_views.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import unicode_literals
+
 import pytest
 import mock
 import lasso
@@ -208,8 +210,8 @@ def test_sp_initiated_login_chosen(private_settings, client):
 def test_sp_initiated_login_requested_authn_context(private_settings, client):
     private_settings.MELLON_IDENTITY_PROVIDERS = [{
         'METADATA': open('tests/metadata.xml').read(),
-        'AUTHN_CLASSREF': [u'urn:be:fedict:iam:fas:citizen:eid',
+        'AUTHN_CLASSREF': ['urn:be:fedict:iam:fas:citizen:eid',
-                           u'urn:be:fedict:iam:fas:citizen:token'],
+                           'urn:be:fedict:iam:fas:citizen:token'],
     }]
     response = client.get('/login/')
     assert response.status_code == 302