remove all remaining uses of urllib2.urlopen (fixes #11388)

We replace it by requests API to improve SSL support (especially SNI).
2016-06-16 11:50:24 +02:00 · 2016-06-16 11:50:24 +02:00 · d70c7ecb2a
parent cd6bd15c39
commit d70c7ecb2a
4 changed files with 134 additions and 57 deletions
--- a/src/authentic2/saml/common.py
+++ b/src/authentic2/saml/common.py
@ -1,12 +1,13 @@
 import urlparse
 import os.path
 import urllib
-import urllib2
 import httplib
 import logging
 import re
 import datetime

+import requests
+
 from authentic2.compat_lasso import lasso
 from django.template import RequestContext
 from django.conf import settings
@ -448,42 +449,17 @@ class SOAPException(Exception):
    pass


-def soap_call(url, msg, client_cert=None):
-    if not client_cert:
-        request = urllib2.Request(url, data=msg,
-                                  headers={'Content-Type': 'text/xml'})
-        return urllib2.urlopen(request).read()
-
-    if url.startswith('http://'):
-        host, query = urllib.splithost(url[5:])
-        conn = httplib.HTTPConnection(host)
-    else:
-        host, query = urllib.splithost(url[6:])
-        conn = httplib.HTTPSConnection(host, key_file=client_cert,
-                                       cert_file=client_cert)
-    logger.debug('host %r', host)
-    logger.debug('query %r', query)
-    logger.debug('msg %r', msg)
+def soap_call(url, msg):
+    logger = logging.getLogger(__name__)
    try:
-        conn.request('POST', query, msg, {'Content-Type': 'text/xml'})
-        response = conn.getresponse()
-    except Exception, err:
-        logging.error('SOAP error (on %s): %s' % (url, err))
-        raise SOAPException(url, err)
-    logger.debug('response %r', response)
-    try:
-        data = response.read()
-    except Exception, err:
-        logging.error('SOAP error (on %s): %s' % (url, err))
-        raise SOAPException(url, err)
-    logger.debug('data %r', data)
-    conn.close()
-    if response.status not in (200, 204):  # 204 ok for federation termination
-        logging.warning('SOAP error (%s) (on %s)' % (response.status, url))
-        raise SOAPException(url, 'http status code error', response.status)
-    if not data:
-        raise SOAPException(url, 'no content returned')
-    return data
+        logger.debug('SOAP call to %r with data %r', url, msg[:10000])
+        response = requests.post(url, data=msg, headers={'Content-Type': 'text/xml'})
+        response.raise_for_status()
+    except requests.RequestException, e:
+        logging.error('SOAP call to %r error %s with data %r', url, e, msg[:10000])
+        raise SOAPException(url, e)
+    logger.debug('SOAP call response %r', response.content[:10000])
+    return response.content


 def send_soap_request(request, profile):
--- a/src/authentic2/saml/forms.py
+++ b/src/authentic2/saml/forms.py
@ -1,5 +1,7 @@
-import urllib2
 import xml.etree.ElementTree as ET
+
+import requests
+
 from authentic2.compat_lasso import lasso

 from django import forms
@ -12,12 +14,14 @@ from authentic2.a2_rbac.utils import get_default_ou

 from django_rbac.utils import get_ou_model

+
 class AddLibertyProviderFromUrlForm(forms.Form):
    name = forms.CharField(max_length=140, label=_('Name'))
    slug = forms.SlugField(max_length=140, label=_('Shortcut'),
-            help_text=_("Internal nickname for the service provider"))
+                           help_text=_("Internal nickname for the service provider"))
    url = forms.URLField(label=_("Metadata's URL"))
-    ou = forms.ModelChoiceField(queryset=get_ou_model().objects, label=_('Organizational unit'))
+    ou = forms.ModelChoiceField(queryset=get_ou_model().objects, initial=get_default_ou,
+                                label=_('Organizational unit'))

    def clean(self):
        cleaned_data = super(AddLibertyProviderFromUrlForm, self).clean()
@ -29,24 +33,25 @@ class AddLibertyProviderFromUrlForm(forms.Form):
        self.childs = []
        if name and slug and url:
            try:
-                content = urllib2.urlopen(url).read().decode('utf-8')
-                root = ET.fromstring(content)
-                if root.tag != '{%s}EntityDescriptor' % lasso.SAML2_METADATA_HREF:
-                    raise ValidationError(_('Invalid SAML metadata: %s') % _('missing EntityDescriptor tag'))
-                is_sp = not root.find('{%s}SPSSODescriptor' % lasso.SAML2_METADATA_HREF) is None
-                if not is_sp:
-                    raise ValidationError(_('Invalid SAML metadata: %s') % _('missing SPSSODescriptor tags'))
-                liberty_provider = LibertyProvider(name=name,
-                    slug=slug, metadata=content, metadata_url=url, ou=ou)
-                liberty_provider.full_clean(exclude=
-                        ('entity_id', 'protocol_conformance'))
-                self.childs.append(LibertyServiceProvider(
-                    liberty_provider=liberty_provider,
-                    enabled=True))
-            except ValidationError, e:
-                raise
-            except Exception, e:
-                raise ValidationError('unsupported error: %s' % e)
+                response = requests.get(url)
+                response.raise_for_status()
+                content = response.content
+            except requests.RequestException, e:
+                raise ValidationError(_('Retrieval of %s failed: %s') % (url, e))
+            root = ET.fromstring(content)
+            if root.tag != '{%s}EntityDescriptor' % lasso.SAML2_METADATA_HREF:
+                raise ValidationError(_('Invalid SAML metadata: %s')
+                                      % _('missing EntityDescriptor tag'))
+            is_sp = not root.find('{%s}SPSSODescriptor' % lasso.SAML2_METADATA_HREF) is None
+            if not is_sp:
+                raise ValidationError(_('Invalid SAML metadata: %s')
+                                      % _('missing SPSSODescriptor tags'))
+            liberty_provider = LibertyProvider(name=name, slug=slug, metadata=content,
+                                               metadata_url=url, ou=ou)
+            liberty_provider.full_clean(exclude=('entity_id', 'protocol_conformance'))
+            self.childs.append(LibertyServiceProvider(
+                liberty_provider=liberty_provider,
+                enabled=True))
            self.instance = liberty_provider
        return cleaned_data

--- a/tests/metadata.xml
+++ b/tests/metadata.xml
@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<EntityDescriptor xmlns="urn:oasis:names:tc:SAML:2.0:metadata"
+    xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion"
+    xmlns:ds="http://www.w3.org/2000/09/xmldsig#"
+    entityID="http://sp6/metadata">
+<SPSSODescriptor
+    AuthnRequestsSigned="true"
+    protocolSupportEnumeration="urn:oasis:names:tc:SAML:2.0:protocol">
+
+<KeyDescriptor use="signing">
+    <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+      <ds:X509Data>
+        <ds:X509Certificate>-----BEGIN CERTIFICATE-----
+MIIDnjCCAoagAwIBAgIBATANBgkqhkiG9w0BAQUFADBUMQswCQYDVQQGEwJGUjEP
+MA0GA1UECBMGRnJhbmNlMQ4wDAYDVQQHEwVQYXJpczETMBEGA1UEChMKRW50cm91
+dmVydDEPMA0GA1UEAxMGRGFtaWVuMB4XDTA2MTAyNzA5MDc1NFoXDTExMTAyNjA5
+MDc1NFowVDELMAkGA1UEBhMCRlIxDzANBgNVBAgTBkZyYW5jZTEOMAwGA1UEBxMF
+UGFyaXMxEzARBgNVBAoTCkVudHJvdXZlcnQxDzANBgNVBAMTBkRhbWllbjCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM06Hx6VgHYR9wUf/tZVVTRkVWNq
+h9x+PvHA2qH4OYMuqGs4Af6lU2YsZvnrmRdcFWv0+UkdAgXhReCWAZgtB1pd/W9m
+6qDRldCCyysow6xPPKRz/pOTwRXm/fM0QGPeXzwzj34BXOIOuFu+n764vKn18d+u
+uVAEzk1576pxTp4pQPzJfdNLrLeQ8vyCshoFU+MYJtp1UA+h2JoO0Y8oGvywbUxH
+ioHN5PvnzObfAM4XaDQohmfxM9Uc7Wp4xKAc1nUq5hwBrHpjFMRSz6UCfMoJSGIi
+3xJMkNCjL0XEw5NKVc5jRKkzSkN5j8KTM/k1jPPsDHPRYzbWWhnNtd6JlkCAwEA
+AaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBHZW5lcmF0
+ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFP2WWMDShux3iF74+SoO1xf6qhqaMB8G
+A1UdIwQYMBaAFGjl6TRXbQDHzSlZu+e8VeBaZMB5MA0GCSqGSIb3DQEBBQUAA4IB
+AQAZ/imK7UMognXbs5RfSB8cMW6iNAI+JZqe9XWjvtmLfIIPbHM96o953SiFvrvQ
+BZjGmmPMK3UH29cjzDx1R/RQaYTyMrHyTePLh3BMd5mpJ/9eeJCSxPzE2ECqWRUa
+pkjukecFXqmRItwgTxSIUE9QkpzvuQRb268PwmgroE0mwtiREADnvTFkLkdiEMew
+fiYxZfJJLPBqwlkw/7f1SyzXoPXnz5QbNwDmrHelga6rKSprYKb3pueqaIe8j/AP
+NC1/bzp8cGOcJ88BD5+Ny6qgPVCrMLE5twQumJ12V3SvjGNtzFBvg2c/9S5OmVqR
+LlTxKnCrWAXftSm1rNtewTsF
+-----END CERTIFICATE-----</ds:X509Certificate>
+      </ds:X509Data>
+    </ds:KeyInfo>
+  </KeyDescriptor>
+
+<KeyDescriptor use="encryption">
+    <ds:KeyInfo xmlns:ds="http://www.w3.org/2000/09/xmldsig#">
+      <ds:X509Data>
+        <ds:X509Certificate>-----BEGIN CERTIFICATE-----
+MIIDnjCCAoagAwIBAgIBATANBgkqhkiG9w0BAQUFADBUMQswCQYDVQQGEwJGUjEP
+MA0GA1UECBMGRnJhbmNlMQ4wDAYDVQQHEwVQYXJpczETMBEGA1UEChMKRW50cm91
+dmVydDEPMA0GA1UEAxMGRGFtaWVuMB4XDTA2MTAyNzA5MDc1NFoXDTExMTAyNjA5
+MDc1NFowVDELMAkGA1UEBhMCRlIxDzANBgNVBAgTBkZyYW5jZTEOMAwGA1UEBxMF
+UGFyaXMxEzARBgNVBAoTCkVudHJvdXZlcnQxDzANBgNVBAMTBkRhbWllbjCCASIw
+DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM06Hx6VgHYR9wUf/tZVVTRkVWNq
+h9x+PvHA2qH4OYMuqGs4Af6lU2YsZvnrmRdcFWv0+UkdAgXhReCWAZgtB1pd/W9m
+6qDRldCCyysow6xPPKRz/pOTwRXm/fM0QGPeXzwzj34BXOIOuFu+n764vKn18d+u
+uVAEzk1576pxTp4pQPzJfdNLrLeQ8vyCshoFU+MYJtp1UA+h2JoO0Y8oGvywbUxH
+ioHN5PvnzObfAM4XaDQohmfxM9Uc7Wp4xKAc1nUq5hwBrHpjFMRSz6UCfMoJSGIi
+3xJMkNCjL0XEw5NKVc5jRKkzSkN5j8KTM/k1jPPsDHPRYzbWWhnNtd6JlkCAwEA
+AaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0EHxYdT3BlblNTTCBHZW5lcmF0
+ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFP2WWMDShux3iF74+SoO1xf6qhqaMB8G
+A1UdIwQYMBaAFGjl6TRXbQDHzSlZu+e8VeBaZMB5MA0GCSqGSIb3DQEBBQUAA4IB
+AQAZ/imK7UMognXbs5RfSB8cMW6iNAI+JZqe9XWjvtmLfIIPbHM96o953SiFvrvQ
+BZjGmmPMK3UH29cjzDx1R/RQaYTyMrHyTePLh3BMd5mpJ/9eeJCSxPzE2ECqWRUa
+pkjukecFXqmRItwgTxSIUE9QkpzvuQRb268PwmgroE0mwtiREADnvTFkLkdiEMew
+fiYxZfJJLPBqwlkw/7f1SyzXoPXnz5QbNwDmrHelga6rKSprYKb3pueqaIe8j/AP
+NC1/bzp8cGOcJ88BD5+Ny6qgPVCrMLE5twQumJ12V3SvjGNtzFBvg2c/9S5OmVqR
+LlTxKnCrWAXftSm1rNtewTsF
+-----END CERTIFICATE-----</ds:X509Certificate>
+      </ds:X509Data>
+    </ds:KeyInfo>
+  </KeyDescriptor>
+
+  <SingleLogoutService
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:SOAP"
+    Location="http://sp6/singleLogoutSOAP" />
+  <SingleLogoutService
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
+    Location="http://sp6/singleLogout"
+    ResponseLocation="http://sp6/singleLogoutReturn" />
+  <ManageNameIDService
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:SOAP"
+    Location="http://sp6/manageNameIdSOAP" />
+  <ManageNameIDService
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect"
+    Location="http://sp6/manageNameId"
+    ResponseLocation="http://sp6/manageNameIdReturn" />
+  <AssertionConsumerService isDefault="true" index="0"
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Artifact"
+    Location="http://sp6/singleSignOnArtifact" />
+  <AssertionConsumerService index="1"
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST"
+    Location="http://sp6/singleSignOnPost" />
+  <AssertionConsumerService index="2"
+    Binding="urn:oasis:names:tc:SAML:2.0:bindings:PAOS"
+    Location="http://sp6/singleSignOnSOAP" />
+</SPSSODescriptor>
+<Organization>
+   <OrganizationName xml:lang="en">WCS Entrouvert</OrganizationName>
+</Organization>
+</EntityDescriptor>
--- a/tox.ini
+++ b/tox.ini
@ -39,6 +39,7 @@ deps =
  django-webtest
  WebTest
  pyquery
+  httmock
 commands =
  ./getlasso.sh
  authentic: py.test {env:FAST:} {env:COVERAGE:} {posargs:tests/}