misc: load json input as strings, not unicode (#8406)

tests/test_misc.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+
+import json
 import pytest
 
 from quixote import cleanup
@@ -6,7 +8,7 @@ from quixote import cleanup
 import wcs.api # workaround against circular dependencies :/
 from wcs.qommon.form import FileSizeWidget
 from wcs.qommon.humantime import humanduration2seconds, seconds2humanduration
-from wcs.qommon.misc import simplify
+from wcs.qommon.misc import simplify, json_loads
 from wcs.admin.settings import FileTypesDirectory
 
 def setup_module(module):
@@ -80,3 +82,16 @@ def test_simplify_remove():
 def test_simplify_mix():
     assert simplify(u'  this is: (a) "cliché" ') == 'this-is-a-cliche'
     assert simplify(u'  À "cliché"; again? ') == 'a-cliche-again'
+
+def test_json_str_decoder():
+    json_str = json.dumps({
+        'lst': [{'a': 'b'}, 1, 2],
+        'bla': u'éléphant'
+        })
+    assert type(json.loads(json_str).keys()[0]) is unicode
+    assert type(json.loads(json_str)['lst'][0]['a']) is unicode
+
+    assert type(json_loads(json_str).keys()[0]) is str
+    assert type(json_loads(json_str)['lst'][0]['a']) is str
+    assert type(json_loads(json_str)['bla']) is str
+    assert json_loads(json_str)['bla'] == u'éléphant'.encode('utf-8')

wcs/qommon/http_request.py

@@ -105,10 +105,11 @@ class HTTPRequest(quixote.http_request.HTTPRequest):
         quixote.http_request.HTTPRequest.process_inputs(self)
         ctype = self.environ.get("CONTENT_TYPE")
         if ctype == 'application/json':
+            from .misc import json_loads
             length = int(self.environ.get('CONTENT_LENGTH') or '0')
             payload = self.stdin.read(length)
             try:
-                self.json = json.loads(payload)
+                self.json = json_loads(payload)
             except ValueError, e:
                 raise RequestError('invalid json payload (%s)' % str(e))
         # Make sure request.form doesn't contain unicode strings, converting

wcs/qommon/misc.py

@@ -390,6 +390,24 @@ class JSONEncoder(json.JSONEncoder):
         # Let the base class default method raise the TypeError
         return json.JSONEncoder.default(self, obj)
 
+def json_encode_helper(d, charset):
+    '''Encode a JSON structure into local charset'''
+    if isinstance(d, unicode):
+        return d.encode(charset)
+    elif isinstance(d, list):
+        return [json_encode_helper(e, charset) for e in d]
+    elif isinstance(d, dict):
+        new_d = {}
+        for k, v in d.iteritems():
+            new_d[json_encode_helper(k, charset)] = json_encode_helper(v, charset)
+        return new_d
+    else:
+        return d
+
+def json_loads(value, charset=None):
+    charset = (get_publisher() and get_publisher().site_charset) or 'utf-8'
+    return json_encode_helper(json.loads(value), charset)
+
 def can_decorate_as_pdf():
     return os.path.exists('/usr/bin/phantomjs')
 

wcs/wf/wscall.py

@@ -22,7 +22,8 @@ import xml.etree.ElementTree as ET
 from quixote.html import TemplateIO, htmltext
 from qommon.errors import ConnectionError
 from qommon.form import *
-from qommon.misc import http_get_page, http_post_request, get_variadic_url, JSONEncoder
+from qommon.misc import (http_get_page, http_post_request, get_variadic_url,
+        JSONEncoder, json_loads)
 from wcs.workflows import (WorkflowStatusItem, register_item_class,
         template_on_formdata, AbortActionException)
 from wcs.api import sign_url
@@ -173,7 +174,7 @@ class WebserviceCallStatusItem(WorkflowStatusItem):
             workflow_data = {'%s_status' % self.varname: status}
             if status == 200:
                 try:
-                    d = json.loads(data)
+                    d = json_loads(data)
                 except (ValueError, TypeError) as e:
                     self.action_on_error(self.action_on_bad_data, formdata,
                             response, data=data, exc_info=sys.exc_info())