use force_str for formdef/workflow imports (#36515)

wcs/fields.py

@@ -34,7 +34,7 @@ from django.utils.formats import date_format as django_date_format
 from django.utils.html import urlize
 from django.utils.six.moves.html_parser import HTMLParser
 
-from .qommon import _
+from .qommon import _, force_str
 from .qommon import evalutils
 from .qommon.form import *
 from .qommon.misc import localstrftime, strftime, date_format, ellipsize, can_thumbnail
@@ -193,9 +193,8 @@ class Field(object):
 
     @property
     def unhtmled_label(self):
-        charset = get_publisher().site_charset
-        return HTMLParser().unescape(force_text(
-            re.sub('<.*?>', ' ', self.label), charset)).strip().encode(charset)
+        return force_str(HTMLParser().unescape(force_text(
+            re.sub('<.*?>', ' ', self.label))).strip())
 
     def get_admin_attributes(self):
         return ['label', 'type', 'condition']
@@ -272,11 +271,11 @@ class Field(object):
                 continue
             if list(el):
                 if type(getattr(self, attribute)) is list:
-                    v = [x.text.encode(charset) for x in el]
+                    v = [force_str(x.text) for x in el]
                 elif type(getattr(self, attribute)) is dict:
                     v = {}
                     for e in el:
-                        v[e.tag] = e.text.encode(charset)
+                        v[e.tag] = force_str(e.text)
                 else:
                     print('currently:', self.__dict__)
                     print('  attribute:', attribute)
@@ -289,12 +288,12 @@ class Field(object):
                 elif el.text in ('False', 'True'): # bools
                     setattr(self, attribute, el.text == 'True')
                 elif type(getattr(self, attribute)) is int:
-                    setattr(self, attribute, int(el.text.encode(charset)))
+                    setattr(self, attribute, int(el.text))
                 else:
-                    setattr(self, attribute, el.text.encode(charset))
+                    setattr(self, attribute, force_str(el.text))
         if include_id:
             try:
-                self.id = elem.find('id').text.encode(charset)
+                self.id = force_str(elem.find('id').text)
             except:
                 pass
 
@@ -304,11 +303,11 @@ class Field(object):
             return
         if node.findall('type'):
             self.condition = {
-                'type': node.find('type').text.encode(charset),
-                'value': node.find('value').text.encode(charset),
+                'type': force_str(node.find('type').text),
+                'value': force_str(node.find('value').text),
             }
         elif node.text:
-            self.condition = {'type': 'python', 'value': node.text.strip().encode(charset)}
+            self.condition = {'type': 'python', 'value': force_str(node.text).strip()}
 
     def get_rst_view_value(self, value, indent=''):
         return indent + self.get_view_value(value)
@@ -753,7 +752,7 @@ class StringField(WidgetField):
         if value.startswith('http://') or value.startswith('https://'):
             charset = get_publisher().site_charset
             value = force_text(value, charset)
-            return htmltext(urlize(value, nofollow=True, autoescape=True).encode(charset))
+            return htmltext(force_str(urlize(value, nofollow=True, autoescape=True)))
         return str(value)
 
     def get_rst_view_value(self, value, indent=''):
@@ -1761,19 +1760,19 @@ class PageField(Field):
         for post_condition_node in node.findall('post_condition'):
             if post_condition_node.findall('condition/type'):
                 condition = {
-                    'type': post_condition_node.find('condition/type').text.encode(charset),
-                    'value': post_condition_node.find('condition/value').text.encode(charset),
+                    'type': force_str(post_condition_node.find('condition/type').text),
+                    'value': force_str(post_condition_node.find('condition/value').text),
                 }
             elif post_condition_node.find('condition').text:
                 condition = {
                     'type': 'python',
-                    'value': post_condition_node.find('condition').text.encode(charset),
+                    'value': force_str(post_condition_node.find('condition').text),
                 }
             else:
                 continue
             self.post_conditions.append({
                 'condition': condition,
-                'error_message': post_condition_node.find('error_message').text.encode(charset),
+                'error_message': force_str(post_condition_node.find('error_message').text),
             })
 
     def post_conditions_export_to_xml(self, node, charset, include_id=False):

wcs/formdef.py

@@ -30,7 +30,7 @@ from django.utils.encoding import force_text
 from quixote import get_request, get_publisher
 from quixote.http_request import Upload
 
-from .qommon import _
+from .qommon import _, force_str
 from .qommon.storage import StorableObject, fix_key
 from .qommon.cron import CronJob
 from .qommon.form import *
@@ -782,8 +782,8 @@ class FormDef(StorableObject):
                 return dict([(unicode2str(k), unicode2str(v)) for k, v in v.items()])
             elif isinstance(v, list):
                 return [unicode2str(x) for x in v]
-            elif isinstance(v, unicode):
-                return v.encode(charset)
+            elif isinstance(v, six.string_types):
+                return force_str(v)
             else:
                 return v
 
@@ -1036,6 +1036,7 @@ class FormDef(StorableObject):
     def import_from_xml_tree(cls, tree, include_id=False, charset=None, fix_on_error=False):
         if charset is None:
             charset = get_publisher().site_charset
+        assert charset == 'utf-8'
         formdef = cls()
         if tree.find('name') is None or not tree.find('name').text:
             raise FormdefImportError(N_('Missing name'))
@@ -1054,7 +1055,7 @@ class FormDef(StorableObject):
             value = tree.find(text_attribute)
             if value is None:
                 continue
-            setattr(formdef, text_attribute, value.text.encode(charset))
+            setattr(formdef, text_attribute, force_str(value.text))
 
         for boolean_attribute in cls.BOOLEAN_ATTRIBUTES:
             value = tree.find(boolean_attribute)
@@ -1088,7 +1089,7 @@ class FormDef(StorableObject):
             if option.attrib.get('type') == 'date':
                 option_value = time.strptime(option.text, '%Y-%m-%d')
             elif option.text:
-                option_value = option.text.encode(charset)
+                option_value = force_str(option.text)
             elif option.findall('filename'):
                 filename = option.find('filename').text
                 upload = Upload(filename, content_type=option.find('content_type').text)
@@ -1109,7 +1110,7 @@ class FormDef(StorableObject):
                 if Category.has_key(category_id):
                     formdef.category_id = category_id
             else:
-                category = category_node.text.encode(charset)
+                category = force_str(category_node.text)
                 for c in Category.select():
                     if c.name == category:
                         formdef.category_id = c.id
@@ -1123,7 +1124,7 @@ class FormDef(StorableObject):
                 if Workflow.has_key(workflow_id):
                     formdef.workflow_id = workflow_id
             else:
-                workflow = workflow_node.text.encode(charset)
+                workflow = force_str(workflow_node.text)
                 for w in Workflow.select():
                     if w.name == workflow:
                         formdef.workflow_id = w.id
@@ -1131,7 +1132,7 @@ class FormDef(StorableObject):
 
         def get_role_by_node(role_node):
             role_id = None
-            value = role_node.text.encode(charset)
+            value = force_str(role_node.text)
             if value.startswith('_') or value == 'logged-users':
                 role_id = value
             elif include_id:
@@ -1176,7 +1177,7 @@ class FormDef(StorableObject):
             formdef.geolocations = {}
             for child in geolocations_node:
                 geoloc_key = child.attrib['key']
-                geoloc_value = child.text.encode(charset)
+                geoloc_value = force_str(child.text)
                 formdef.geolocations[geoloc_key] = geoloc_value
 
         if tree.find('required_authentication_contexts') is not None:

wcs/workflows.py

@@ -31,7 +31,7 @@ from django.utils.six import StringIO
 
 from quixote import get_request, get_response, redirect
 
-from .qommon import _
+from .qommon import _, force_str
 from .qommon.misc import C_, get_as_datetime, file_digest, get_foreground_colour
 from .qommon.storage import StorableObject, atomic_write, NotEqual, Contains, Null
 from .qommon.form import *
@@ -597,12 +597,12 @@ class Workflow(StorableObject):
         if include_id and tree.attrib.get('id'):
             workflow.id = tree.attrib.get('id')
 
-        workflow.name = tree.find('name').text.encode(charset)
+        workflow.name = force_str(tree.find('name').text)
 
         if tree.find('roles') is not None:
             workflow.roles = {}
             for role_node in tree.findall('roles/role'):
-                workflow.roles[role_node.attrib['id']] = role_node.text.encode(charset)
+                workflow.roles[role_node.attrib['id']] = force_str(role_node.text)
 
         if tree.find('last_modification') is not None:
             node = tree.find('last_modification')
@@ -874,11 +874,11 @@ class XmlSerialisable(object):
                 continue
             if list(el):
                 if type(getattr(self, attribute)) is list:
-                    v = [x.text.encode(charset) if x.text is not None else '' for x in el]
+                    v = [force_str(x.text) if x.text is not None else '' for x in el]
                 elif type(getattr(self, attribute)) is dict:
                     v = {}
                     for e in el:
-                        v[e.tag] = e.text.encode(charset)
+                        v[e.tag] = force_str(e.text)
                 else:
                     # ???
                     raise AssertionError
@@ -889,9 +889,9 @@ class XmlSerialisable(object):
                 elif el.text in ('False', 'True'): # bools
                     setattr(self, attribute, el.text == 'True')
                 elif type(getattr(self, attribute)) is int:
-                    setattr(self, attribute, int(el.text.encode(charset)))
+                    setattr(self, attribute, int(el.text))
                 else:
-                    setattr(self, attribute, el.text.encode(charset))
+                    setattr(self, attribute, force_str(el.text))
 
     def _roles_export_to_xml(self, attribute, item, charset, include_id=False):
         if not hasattr(self, attribute) or not getattr(self, attribute):
@@ -942,7 +942,7 @@ class XmlSerialisable(object):
         if elem is None:
             return None
 
-        value = elem.text.encode(charset)
+        value = force_str(elem.text)
 
         # look for known static values
         if value.startswith('_') or value == 'logged-users':
@@ -950,7 +950,7 @@ class XmlSerialisable(object):
 
         # if we import using id, look at the role_id attribute
         if include_id and 'role_id' in elem.attrib:
-            role_id = elem.attrib['role_id'].encode(charset)
+            role_id = force_str(elem.attrib['role_id'])
             if Role.has_key(role_id):
                 return role_id
             if WorkflowStatusItem.get_expression(role_id)['type'] in ('python', 'template'):
@@ -1351,10 +1351,10 @@ class WorkflowGlobalAction(object):
         return status
 
     def init_with_xml(self, elem, charset, include_id=False):
-        self.id = elem.find('id').text.encode(charset)
-        self.name = elem.find('name').text.encode(charset)
+        self.id = force_str(elem.find('id').text)
+        self.name = force_str(elem.find('name').text)
         if elem.find('backoffice_info_text') is not None:
-            self.backoffice_info_text = elem.find('backoffice_info_text').text.encode(charset)
+            self.backoffice_info_text = force_str(elem.find('backoffice_info_text').text)
 
         self.items = []
         for item in elem.find('items'):
@@ -1392,10 +1392,10 @@ class WorkflowCriticalityLevel(object):
         return level
 
     def init_with_xml(self, elem, charset, include_id=False):
-        self.id = elem.find('id').text.encode(charset)
-        self.name = elem.find('name').text.encode(charset)
+        self.id = force_str(elem.find('id').text)
+        self.name = force_str(elem.find('name').text)
         if elem.find('colour') is not None:
-            self.colour = elem.find('colour').text.encode(charset)
+            self.colour = force_str(elem.find('colour').text)
 
 
 class WorkflowStatus(object):
@@ -1641,16 +1641,16 @@ class WorkflowStatus(object):
         return status
 
     def init_with_xml(self, elem, charset, include_id=False):
-        self.id = elem.find('id').text.encode(charset)
-        self.name = elem.find('name').text.encode(charset)
+        self.id = force_str(elem.find('id').text)
+        self.name = force_str(elem.find('name').text)
         if elem.find('colour') is not None:
-            self.colour = elem.find('colour').text.encode(charset)
+            self.colour = force_str(elem.find('colour').text)
         if elem.find('extra_css_class') is not None:
-            self.extra_css_class = elem.find('extra_css_class').text.encode(charset)
+            self.extra_css_class = force_str(elem.find('extra_css_class').text)
         if elem.find('forced_endpoint') is not None:
             self.forced_endpoint = (elem.find('forced_endpoint').text == 'true')
         if elem.find('backoffice_info_text') is not None:
-            self.backoffice_info_text = elem.find('backoffice_info_text').text.encode(charset)
+            self.backoffice_info_text = force_str(elem.find('backoffice_info_text').text)
 
         self.visibility = []
         for visibility_role in elem.findall('visibility/role'):
@@ -2003,12 +2003,12 @@ class WorkflowStatusItem(XmlSerialisable):
             return
         if node.findall('type'):
             self.condition = {
-                'type': node.find('type').text.encode(charset),
-                'value': node.find('value').text.encode(charset),
+                'type': force_str(node.find('type').text),
+                'value': force_str(node.find('value').text),
             }
         elif node.text:
             # backward compatibility
-            self.condition = {'type': 'python', 'value': node.text.encode(charset)}
+            self.condition = {'type': 'python', 'value': force_str(node.text)}
 
     def q_admin_lookup(self, workflow, status, component, html_top):
         return None
@@ -2023,7 +2023,7 @@ class WorkflowStatusItem(XmlSerialisable):
         if elem is None:
             self.attachments = None
         else:
-            self.attachments = [item.text.encode(charset) for item in elem.findall('attachment')]
+            self.attachments = [force_str(item.text) for item in elem.findall('attachment')]
 
     def get_attachments_options(self):
         attachments_options = [(None, '---', None)]
@@ -2293,7 +2293,7 @@ class CommentableWorkflowStatusItem(WorkflowStatusItem):
             # to None, meaning "no button".
             self.button_label = None
         else:
-            self.button_label = element.text.encode(charset)
+            self.button_label = force_str(element.text)
 
 register_item_class(CommentableWorkflowStatusItem)
 
@@ -2446,7 +2446,7 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
         # override to allow for destination set with computed values.
         if elem is None:
             return None
-        value = elem.text.encode(charset)
+        value = force_str(elem.text)
 
         if self.get_expression(value)['type'] != 'text' or '@' in value:
             return value