misc: accept only http and https as URL scheme (#20523)

tests/test_datasource.py

@@ -24,6 +24,9 @@ def setup_module(module):
     global pub, req
 
     pub = create_temporary_pub()
+    pub.cfg['debug'] = {'logger': True}
+    pub.write_cfg()
+    pub.set_config()
 
     req = HTTPRequest(None, {'SERVER_NAME': 'example.net', 'SCRIPT_NAME': ''})
     pub._set_request(req)
@@ -228,6 +231,36 @@ def test_json_datasource(http_requests):
     assert data_sources.get_items(datasource) == []
     assert data_sources.get_structured_items(datasource) == []
 
+def test_json_datasource_bad_url(http_requests, caplog):
+    datasource = {'type': 'json', 'value': 'http://remote.example.net/404'}
+    assert data_sources.get_items(datasource) == []
+    assert 'Error loading JSON data source' in caplog.records[-1].message
+    assert 'status: 404' in caplog.records[-1].message
+
+    datasource = {'type': 'json', 'value': 'http://remote.example.net/xml'}
+    assert data_sources.get_items(datasource) == []
+    assert 'Error reading JSON data source output' in caplog.records[-1].message
+    assert 'No JSON object could be decoded' in caplog.records[-1].message
+
+    datasource = {'type': 'json', 'value': 'http://remote.example.net/connection-error'}
+    assert data_sources.get_items(datasource) == []
+    assert 'Error loading JSON data source' in caplog.records[-1].message
+    assert 'error' in caplog.records[-1].message
+
+def test_json_datasource_bad_url_scheme(caplog):
+    datasource = {'type': 'json', 'value': ''}
+    assert data_sources.get_items(datasource) == []
+    assert caplog.records[-1].message == 'Empty URL in JSON data source'
+
+    datasource = {'type': 'json', 'value': 'foo://bar'}
+    assert data_sources.get_items(datasource) == []
+    assert 'Error loading JSON data source' in caplog.records[-1].message
+    assert 'invalid scheme in URL' in caplog.records[-1].message
+
+    datasource = {'type': 'json', 'value': '/bla/blo'}
+    assert data_sources.get_items(datasource) == []
+    assert 'Error loading JSON data source' in caplog.records[-1].message
+    assert 'invalid scheme in URL' in caplog.records[-1].message
 
 def test_item_field_named_python_datasource():
     NamedDataSource.wipe()

wcs/qommon/misc.py

@@ -277,8 +277,10 @@ def _http_request(url, method='GET', body=None, headers={}, cert_file=None, time
 
     if url.startswith('http://'):
         hostname, query = urllib.splithost(url[5:])
-    else:
+    elif url.startswith('https://'):
         hostname, query = urllib.splithost(url[6:])
+    else:
+        raise ConnectionError('invalid scheme in URL %s' % url)
 
     auth = None
     if '@' in hostname: