misc: use Django/ezt templates in variadic URLs (#19442)

This commit is contained in:
Thomas NOËL 2017-11-27 22:05:14 +01:00
parent 18c9258e9f
commit 3906f827cd
8 changed files with 234 additions and 7 deletions

View File

@ -3812,6 +3812,21 @@ def test_backoffice_formdata_named_wscall(pub):
resp = app.get('/backoffice/submission/test/')
assert '<p class="comment-field ">XbarY</p>' in resp.body
# django-templated URL
wscall.request = {'url': '{{ example_url }}json'}
wscall.store()
resp = app.get('/backoffice/submission/test/')
assert '<p class="comment-field ">XbarY</p>' in resp.body
# webservice call in django template
formdef.fields = [
fields.CommentField(id='7', label='dja-{{ webservice.hello_world.foo}}-ngo', type='comment'),
]
formdef.store()
formdef.data_class().wipe()
resp = app.get('/backoffice/submission/test/')
assert '<p class="comment-field ">dja-bar-ngo</p>' in resp.body
def test_backoffice_session_var(pub):
open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w').write('''[options]
query_string_allowed_vars = foo,bar

View File

@ -170,6 +170,16 @@ def test_json_datasource():
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'more': 'xxx'}),
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# same with django templated url
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
pub.substitutions.feed(JsonUrlPath())
datasource = {'type': 'json', 'value': '{{ json_url }}'}
assert data_sources.get_items(datasource) == [
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'more': 'xxx'}),
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# json specified with a variadic url with an erroneous space
class JsonUrlPath(object):
def get_substitution_variables(self):
@ -180,6 +190,16 @@ def test_json_datasource():
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'more': 'xxx'}),
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# same with django templated url
class JsonUrlPath(object):
def get_substitution_variables(self):
return {'json_url': 'file://%s' % json_file_path}
pub.substitutions.feed(JsonUrlPath())
datasource = {'type': 'json', 'value': ' {{ json_url }}'}
assert data_sources.get_items(datasource) == [
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'more': 'xxx'}),
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})]
# a json file with integer as 'id'
json_file = open(json_file_path, 'w')
json.dump({'data': [{'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]}, json_file)

View File

@ -1,9 +1,13 @@
import pytest
from wcs.qommon.misc import get_variadic_url
from wcs.qommon.ezt import EZTException
def test_url_unchanged():
assert get_variadic_url('http://www.example.net/foobar', {}) == 'http://www.example.net/foobar'
def test_url_scheme():
assert get_variadic_url('http[https]://www.example.net/foobar',
{'https': 's'}) == 'https://www.example.net/foobar'
@ -16,15 +20,36 @@ def test_url_scheme():
assert get_variadic_url('http[https]://www.example.net/foo/bar',
{'https': ''}) == 'http://www.example.net/foo/bar'
assert get_variadic_url('http{{ https }}://www.example.net/foobar',
{'https': 's'}) == 'https://www.example.net/foobar'
assert get_variadic_url('http{{ https }}://www.example.net/foobar',
{'https': ''}) == 'http://www.example.net/foobar'
assert get_variadic_url('http{{ https }}://www.example.net/foobar/',
{'https': ''}) == 'http://www.example.net/foobar/'
assert get_variadic_url('http{{ https }}://www.example.net/foo/bar/',
{'https': ''}) == 'http://www.example.net/foo/bar/'
assert get_variadic_url('http{{ https }}://www.example.net/foo/bar',
{'https': ''}) == 'http://www.example.net/foo/bar'
def test_url_netloc():
assert get_variadic_url('http://[hostname]/foobar',
{'hostname': 'www.example.net'}) == 'http://www.example.net/foobar'
assert get_variadic_url('http://[hostname]/foobar',
{'hostname': 'www.example.com'}) == 'http://www.example.com/foobar'
assert get_variadic_url('http://{{ hostname }}/foobar',
{'hostname': 'www.example.net'}) == 'http://www.example.net/foobar'
assert get_variadic_url('http://{{ hostname }}/foobar',
{'hostname': 'www.example.com'}) == 'http://www.example.com/foobar'
def test_url_netloc_port():
assert get_variadic_url('http://www.example.net:[port]/foobar',
{'port': '80'}) == 'http://www.example.net:80/foobar'
assert get_variadic_url('http://www.example.net:{{ port }}/foobar',
{'port': '80'}) == 'http://www.example.net:80/foobar'
def test_url_path():
assert get_variadic_url('http://www.example.net/[path]',
@ -32,6 +57,14 @@ def test_url_path():
assert get_variadic_url('http://www.example.net/[path]',
{'path': 'foo bar'}) == 'http://www.example.net/foo%20bar'
assert get_variadic_url('http://www.example.net/{{ path }}',
{'path': 'foobar'}) == 'http://www.example.net/foobar'
assert get_variadic_url('http://www.example.net/{{ path }}',
{'path': 'foo bar'}) == 'http://www.example.net/foo bar'
assert get_variadic_url('http://www.example.net/{{ path|urlencode }}',
{'path': 'foo bar'}) == 'http://www.example.net/foo%20bar'
def test_url_query_variable():
assert get_variadic_url('http://www.example.net/foobar?hello=[world]',
{'world': 'world'}) == 'http://www.example.net/foobar?hello=world'
@ -42,6 +75,22 @@ def test_url_query_variable():
assert get_variadic_url('http://www.example.net/foobar?hello=[world]',
{'world': 'a=b'}) == 'http://www.example.net/foobar?hello=a%3Db'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world }}',
{'world': 'world'}) == 'http://www.example.net/foobar?hello=world'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world }}',
{'world': 'a b'}) == 'http://www.example.net/foobar?hello=a b'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world|urlencode }}',
{'world': 'a b'}) == 'http://www.example.net/foobar?hello=a%20b'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world }}',
{'world': 'a&b'}) == 'http://www.example.net/foobar?hello=a&amp;b'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world|urlencode }}',
{'world': 'a&b'}) == 'http://www.example.net/foobar?hello=a%26b'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world }}',
{'world': 'a=b'}) == 'http://www.example.net/foobar?hello=a=b'
assert get_variadic_url('http://www.example.net/foobar?hello={{ world|urlencode }}',
{'world': 'a=b'}) == 'http://www.example.net/foobar?hello=a%3Db'
def test_url_query_key():
assert get_variadic_url('http://www.example.net/foobar?[hello]=world',
{'hello': 'hello'}) == 'http://www.example.net/foobar?hello=world'
@ -52,24 +101,56 @@ def test_url_query_key():
assert get_variadic_url('http://www.example.net/foobar?[hello]=world',
{'hello': 'a=b'}) == 'http://www.example.net/foobar?a%3Db=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello }}=world',
{'hello': 'hello'}) == 'http://www.example.net/foobar?hello=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello }}=world',
{'hello': 'a b'}) == 'http://www.example.net/foobar?a b=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello|urlencode }}=world',
{'hello': 'a b'}) == 'http://www.example.net/foobar?a%20b=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello }}=world',
{'hello': 'a&b'}) == 'http://www.example.net/foobar?a&amp;b=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello|urlencode }}=world',
{'hello': 'a&b'}) == 'http://www.example.net/foobar?a%26b=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello }}=world',
{'hello': 'a=b'}) == 'http://www.example.net/foobar?a=b=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello|urlencode }}=world',
{'hello': 'a=b'}) == 'http://www.example.net/foobar?a%3Db=world'
def test_url_query_whole():
assert get_variadic_url('http://www.example.net/foobar?[hello]',
{'hello': 'hello=world'}) == 'http://www.example.net/foobar?hello=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello }}',
{'hello': 'hello=world'}) == 'http://www.example.net/foobar?hello=world'
assert get_variadic_url('http://www.example.net/foobar?{{ hello|urlencode }}',
{'hello': 'hello=world'}) == 'http://www.example.net/foobar?hello%3Dworld'
def test_url_netloc_port_and_path():
assert get_variadic_url('http://www.example.net:[port]/foobar/[path]',
{'port': '80', 'path': 'baz'}) == 'http://www.example.net:80/foobar/baz'
assert get_variadic_url('http://www.example.net:[port]/foobar/[path]',
{'port': '80', 'path': 'b z'}) == 'http://www.example.net:80/foobar/b%20z'
assert get_variadic_url('http://www.example.net:{{ port }}/foobar/{{ path }}',
{'port': '80', 'path': 'baz'}) == 'http://www.example.net:80/foobar/baz'
assert get_variadic_url('http://www.example.net:{{ port }}/foobar/{{ path }}',
{'port': '80', 'path': 'b z'}) == 'http://www.example.net:80/foobar/b z'
assert get_variadic_url('http://www.example.net:{{ port|urlencode }}/foobar/{{ path|urlencode }}',
{'port': '80', 'path': 'b z'}) == 'http://www.example.net:80/foobar/b%20z'
def test_url_whole():
assert get_variadic_url('[url]',
{'url': 'http://www.example.net/foobar'}) == 'http://www.example.net/foobar'
assert get_variadic_url('{{ url }}',
{'url': 'http://www.example.net/foobar'}) == 'http://www.example.net/foobar'
def test_url_server():
for url in ('http://www.example.net', 'http://www.example.net/'):
assert get_variadic_url('[url]/foobar',
{'url': url}) == 'http://www.example.net/foobar'
assert get_variadic_url('[url]/foobar',
{'url': url}) == 'http://www.example.net/foobar'
assert get_variadic_url('[url]/foobar/',
@ -81,6 +162,24 @@ def test_url_server():
assert get_variadic_url('[url]foo/bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foo/bar'
assert get_variadic_url('{{ url }}/foobar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar'
assert get_variadic_url('{{ url }}/foobar', # Django is more conservative here:
{'url': 'http://www.example.net/'}) == 'http://www.example.net//foobar'
# to be "smart", use Django templates language:
for url in ('http://www.example.net', 'http://www.example.net/'):
assert get_variadic_url('{{ url }}{% if url|last != "/" %}/{% endif %}foobar',
{'url': url}) == 'http://www.example.net/foobar'
assert get_variadic_url('{{ url }}{% if url|last != "/" %}/{% endif %}foobar/',
{'url': url}) == 'http://www.example.net/foobar/'
assert get_variadic_url('{{ url }}foo/bar/',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foo/bar/'
assert get_variadic_url('{{ url }}foobar/',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/'
assert get_variadic_url('{{ url }}foo/bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foo/bar'
def test_url_server_qs():
assert get_variadic_url('[url]?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/?foo=bar'
@ -91,6 +190,17 @@ def test_url_server_qs():
assert get_variadic_url('[url]/?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/?foo=bar'
# Django is more conservative
assert get_variadic_url('{{ url }}?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net?foo=bar'
assert get_variadic_url('{{ url }}?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/?foo=bar'
assert get_variadic_url('{{ url }}/?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/?foo=bar'
assert get_variadic_url('{{ url }}/?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net//?foo=bar'
def test_url_server_more():
assert get_variadic_url('[url]/foobar/json?toto',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/json?toto'
@ -99,6 +209,15 @@ def test_url_server_more():
assert get_variadic_url('[url]foobar/json?toto',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/json?toto'
# Django is more conservative
assert get_variadic_url('{{ url }}/foobar/json?toto',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/json?toto'
assert get_variadic_url('{{ url }}/foobar/json?toto',
{'url': 'http://www.example.net/'}) == 'http://www.example.net//foobar/json?toto'
assert get_variadic_url('{{ url }}foobar/json?toto',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/json?toto'
def test_url_server_even_more():
assert get_variadic_url('[url]/foobar/json?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/json?foo=bar'
@ -107,6 +226,15 @@ def test_url_server_even_more():
assert get_variadic_url('[url]foobar/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/json?foo=bar'
# Django is more conservative
assert get_variadic_url('{{ url }}/foobar/json?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/json?foo=bar'
assert get_variadic_url('{{ url }}/foobar/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net//foobar/json?foo=bar'
assert get_variadic_url('{{ url }}foobar/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/json?foo=bar'
def test_url_server_even_more_more():
assert get_variadic_url('[url]/foobar/baz/json?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/baz/json?foo=bar'
@ -115,21 +243,63 @@ def test_url_server_even_more_more():
assert get_variadic_url('[url]foobar/baz/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/baz/json?foo=bar'
# Django is more conservative
assert get_variadic_url('{{ url }}/foobar/baz/json?foo=bar',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/baz/json?foo=bar'
assert get_variadic_url('{{ url }}/foobar/baz/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net//foobar/baz/json?foo=bar'
assert get_variadic_url('{{ url }}foobar/baz/json?foo=bar',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/baz/json?foo=bar'
def test_url_whole_with_qs():
assert get_variadic_url('[url]',
{'url': 'http://www.example.net/?foo=bar'}) == 'http://www.example.net/?foo=bar'
assert get_variadic_url('{{ url }}',
{'url': 'http://www.example.net/?foo=bar'}) == 'http://www.example.net/?foo=bar'
def test_url_whole_with_qs_2():
for url in ('[url]?bar=foo', '[url]&bar=foo', '[url]/?bar=foo'):
assert get_variadic_url(url, {'url': 'http://www.example.net/?foo=bar'}) in \
('http://www.example.net/?bar=foo&foo=bar', 'http://www.example.net/?foo=bar&bar=foo')
assert get_variadic_url('{{ url }}&bar=foo',
{'url': 'http://www.example.net/?foo=bar'}) == 'http://www.example.net/?foo=bar&bar=foo'
# to be "smart", use Django templates language:
assert get_variadic_url('{{ url }}{% if "?" in url %}&{% else %}?{% endif %}bar=foo',
{'url': 'http://www.example.net/?foo=bar'}) == 'http://www.example.net/?foo=bar&bar=foo'
assert get_variadic_url('{{ url }}{% if "?" in url %}&{% else %}?{% endif %}bar=foo',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/?bar=foo'
def test_path_missing_var():
assert get_variadic_url('http://www.example.net/foobar/[path]',
{}) == 'http://www.example.net/foobar/[path]'
# Django is permissive here
assert get_variadic_url('http://www.example.net/foobar/{{ path }}',
{}) == 'http://www.example.net/foobar/'
def test_url_base_and_missing_var():
assert get_variadic_url('[url]/foobar/[path]',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/[path]'
assert get_variadic_url('[url]foobar/[path]',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/[path]'
assert get_variadic_url('{{ url }}/foobar/{{ path }}',
{'url': 'http://www.example.net'}) == 'http://www.example.net/foobar/'
assert get_variadic_url('{{ url }}foobar/{{ path }}',
{'url': 'http://www.example.net/'}) == 'http://www.example.net/foobar/'
def test_url_bad_syntax():
with pytest.raises(EZTException):
get_variadic_url('[if-any form_avr]https://example.net/[foo]/', {'foo': 'bar'})
# Django TemplateSyntaxError
assert get_variadic_url('{% if %}https://www/{{ foo }}',
{'bar': 'nofoo'}) == '{% if %}https://www/{{ foo }}'
# Django VariableDoesNotExist
assert get_variadic_url('{{ foo|default:notexist }}', {}) == '{{ foo|default:notexist }}'

View File

@ -2932,6 +2932,15 @@ def test_redirect_to_url(pub):
pub.substitutions.feed(formdata)
assert item.perform(formdata) == 'https://www.example.net/?foo=bar'
item.url = 'https://www.example.net/?django={{ form_var_foo }}'
assert item.render_as_line() == 'Redirect to URL "https://www.example.net/?django={{ form_var_foo }}"'
pub.substitutions.feed(formdata)
assert item.perform(formdata) == 'https://www.example.net/?django=bar'
item.url = '[if-any nada]https://www.example.net/[end]'
pub.substitutions.feed(formdata)
assert item.perform(formdata) == None
item.url = '{% if nada %}https://www.example.net/{% endif %}'
pub.substitutions.feed(formdata)
assert item.perform(formdata) == None

View File

@ -29,6 +29,7 @@ import qommon.misc
from qommon import get_logger
from qommon.storage import StorableObject
from qommon.template import Template
from qommon.xml_storage import XmlStorableObject
from wcs.api_utils import sign_url, get_secret_and_orig, MissingSecret
@ -147,7 +148,7 @@ def get_structured_items(data_source):
get_logger().warn('Empty URL in JSON data source')
return []
url = url.strip()
if '[' in url:
if Template.is_template_string(url):
vars = get_publisher().substitutions.get_context_variables()
url = get_variadic_url(url, vars)
try:

View File

@ -1796,7 +1796,7 @@ var wcs_select2_%(id)s = $("#form_%(id)s").select2({
'loadmore': _('Loading more results...'),
'searching': _('Searching...')})
if '[' in self.url:
if Template.is_template_string(self.url):
vars = get_publisher().substitutions.get_context_variables()
# skip variables that were not set (None)
vars = dict((x, y) for x, y in vars.items() if y is not None)
@ -1901,7 +1901,7 @@ class AutocompleteStringWidget(WcsExtraStringWidget):
def render_content(self):
get_response().add_javascript(['jquery.js', 'jquery-ui.js'])
if self.url and '[' in self.url:
if Template.is_template_string(self.url):
vars = get_publisher().substitutions.get_context_variables()
# skip variables that were not set (None)
vars = dict((x, y) for x, y in vars.items() if y is not None)

View File

@ -37,6 +37,7 @@ except ImportError:
from django.conf import settings
from django.utils import datetime_safe
from django.template import Template, Context, TemplateSyntaxError, VariableDoesNotExist
from quixote import get_publisher, get_response, get_request
from quixote.html import htmltext
@ -44,6 +45,7 @@ from quixote.html import htmltext
from qommon import _
from qommon import get_cfg, get_logger, ezt
from qommon.errors import ConnectionError
from qommon.template import Template
from urllib import urlencode, quote
from urllib2 import urlparse
@ -326,8 +328,17 @@ def http_delete_request(url, **kwargs):
return _http_request(url, 'DELETE', **kwargs)
def get_variadic_url(url, variables, encode_query=True):
# substitution in an URL : try to be safe
if not Template.is_template_string(url):
return url
# django template
if '{{' in url or '{%' in url:
try:
return Template(url).render(Context(variables))
except (TemplateSyntaxError, VariableDoesNotExist):
return url
# ezt template, try to be safe
def ezt_substitute(template, variables):
tmpl = ezt.Template()
tmpl.parse(template)

View File

@ -29,6 +29,7 @@ from qommon.xml_storage import XmlStorableObject
from qommon.form import (CompositeWidget, StringWidget, WidgetDict,
ComputedExpressionWidget, RadiobuttonsWidget, CheckboxWidget)
import qommon.misc
from qommon.template import Template
from wcs.api_utils import sign_url, get_secret_and_orig, MissingSecret
from wcs.workflows import WorkflowStatusItem
@ -40,7 +41,7 @@ def call_webservice(url, qs_data=None, request_signature_key=None,
**kwargs):
url = url.strip()
if '[' in url:
if Template.is_template_string(url):
variables = get_publisher().substitutions.get_context_variables()
url = get_variadic_url(url, variables)