general: deal with py2/3 type conversion when unpickling (#38021)

tests/test_formdef.py

@@ -9,13 +9,16 @@ import time
 
 import pytest
 
+from django.utils import six
+from django.utils.encoding import force_bytes
 from django.utils.six import BytesIO
 from quixote import cleanup
 from wcs import fields
 from wcs.formdef import FormDef, get_formdefs_of_all_kinds
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.form import PicklableUpload
-from wcs.workflows import Workflow, AttachmentEvolutionPart
+from wcs.workflows import Workflow, AttachmentEvolutionPart, WorkflowBackofficeFieldsFormDef
+from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
 from wcs.fields import StringField, FileField, DateField, ItemField, PageField
 
 from utilities import create_temporary_pub, clean_temporary_pub
@@ -438,3 +441,82 @@ def test_get_formdefs_of_all_kinds(pub):
         ('basic formdef', FormDef),
         ('carddef', CardDef),
     ]
+
+def test_pickle_2to3_conversion(pub):
+    if six.PY2:
+        pytest.skip('only relevant for Python 3')
+        return
+
+    FormDef.wipe()
+    Workflow.wipe()
+
+    workflow = Workflow(name='blah')
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(
+            id='bo0', varname='foo_bovar', type='string', label='bo variable'),
+    ]
+    status = workflow.add_status('Status1')
+    display_form = FormWorkflowStatusItem()
+    display_form.id = '_display_form'
+    display_form.by = []
+    display_form.varname = 'blah'
+    display_form.formdef = WorkflowFormFieldsFormDef(item=display_form)
+    display_form.formdef.fields.append(
+            fields.StringField(
+                id='1', label='Test', varname='str', type='string', required=True))
+    status.items.append(display_form)
+    display_form.parent = status
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'basic formdef'
+    formdef.workflow_id = workflow.id
+    formdef.workflow_options = {'bo0': 'whatever'}
+    formdef.workflow_roles = {'_receiver': '1'}
+    formdef.fields = [
+        StringField(id='1', label='Test', type='string', varname='foo'),
+    ]
+    formdef.roles
+    formdef.store()
+
+    formdef_id = formdef.id
+    workflow_id = workflow.id
+    formdef_filename = os.path.join(formdef.get_objects_dir(), str(formdef.id))
+    workflow_filename = os.path.join(workflow.get_objects_dir(), str(workflow.id))
+
+    # turn pickle to bytes
+
+    def deep_str2bytes(obj, seen=None):
+        # reverse deep_bytes2str
+        if seen is None:
+            seen = {}
+        if obj is None or isinstance(obj, (int, float, bytes, time.struct_time, type(Ellipsis))):
+            return obj
+        if id(obj) in seen:
+            return obj
+        if isinstance(obj, str):
+            return force_bytes(obj)
+        seen[id(obj)] = True
+        if isinstance(obj, dict):
+            new_d = {}
+            for k, v in obj.items():
+                new_d[force_bytes(k)] = deep_str2bytes(v, seen)
+            return new_d
+        if isinstance(obj, list):
+            return [deep_str2bytes(x, seen) for x in obj]
+        if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
+            obj.__dict__ = deep_str2bytes(obj.__dict__, seen)
+            return obj
+        return obj
+
+    formdef.__dict__ = deep_str2bytes(formdef.__dict__)
+    pickle.dump(formdef, open(formdef_filename, 'wb'), protocol=2)
+
+    workflow.__dict__ = deep_str2bytes(workflow.__dict__)
+    pickle.dump(workflow, open(workflow_filename, 'wb'), protocol=2)
+
+    formdef = FormDef.get(formdef_id)
+    assert formdef.fields[0].label == 'Test'
+    assert formdef.workflow.possible_status[0].items[0].varname == 'blah'
+    assert formdef.workflow.possible_status[0].items[0].formdef.fields[0].varname == 'str'

wcs/qommon/__init__.py

@@ -45,7 +45,7 @@ if six.PY2:
     PICKLE_KWARGS = {}
 else:
     # unpickle python2 strings as bytes
-    PICKLE_KWARGS = {'encoding': 'bytes'}
+    PICKLE_KWARGS = {'encoding': 'bytes', 'fix_imports': True}
 
 def _(message):
     pub = get_publisher()

wcs/qommon/storage.py

@@ -26,12 +26,20 @@ import tempfile
 
 from django.utils import six
 from django.utils.encoding import force_bytes
+from django.utils.six.moves import builtins
 from django.utils.six.moves import _thread
 
 from .vendor import locket
 
 from quixote import get_publisher
-from . import PICKLE_KWARGS
+from . import PICKLE_KWARGS, force_str
+
+
+if six.PY3:
+    import copyreg
+    # add compatibility names in case those were stored in pickles
+    sys.modules['copy_reg'] = copyreg
+    sys.modules['__builtin__'] = builtins
 
 
 def cache_umask():
@@ -93,6 +101,38 @@ def atomic_write(path, content, async_op=False):
         doit()
 
 
+def deep_bytes2str(obj, seen=None):
+    # Convert obj loaded by unpickle(encoding='bytes') to a proper object using
+    # strings; this is required as encoding='utf-8' is not possible when there
+    # are pickled datetime objects. <https://bugs.python.org/issue22005>
+    if six.PY2:
+        return obj
+    if seen is None:
+        seen = {}
+    if obj is None or isinstance(obj, (int, float, str, time.struct_time, type(Ellipsis))):
+        return obj
+    if id(obj) in seen:
+        return obj
+    if isinstance(obj, bytes):
+        return obj.decode('utf-8')
+    seen[id(obj)] = True
+    if isinstance(obj, dict):
+        new_d = {}
+        for k, v in obj.items():
+            new_d[force_str(k)] = deep_bytes2str(v, seen)
+        return new_d
+    if isinstance(obj, list):
+        return [deep_bytes2str(x, seen) for x in obj]
+    if hasattr(obj, '__class__') and obj.__class__.__module__.startswith(('wcs.', 'qommon.', 'modules.')):
+        obj.__dict__ = deep_bytes2str(obj.__dict__, seen)
+        return obj
+    return obj
+
+
+def pickle_2to3_conversion(obj):
+    obj.__dict__ = deep_bytes2str(obj.__dict__)  # inplace
+
+
 class Criteria(object):
     def __init__(self, attribute, value):
         self.attribute = attribute
@@ -424,6 +464,8 @@ class StorableObject(object):
                 return None
             raise KeyError()
         o.__class__ = cls
+        if six.PY3 and any((isinstance(k, bytes) for k in o.__dict__)):
+            pickle_2to3_conversion(o)
         if not ignore_migration:
             o.id = str(o.id) # makes sure 'id' is a string
             if hasattr(cls, 'migrate'):

wcs/sql.py

@@ -31,7 +31,7 @@ from django.utils.six import BytesIO
 from quixote import get_publisher
 from . import qommon
 from wcs.qommon import force_str, PICKLE_KWARGS
-from .qommon.storage import _take, parse_clause as parse_storage_clause
+from .qommon.storage import _take, deep_bytes2str, parse_clause as parse_storage_clause
 from .qommon.substitution import invalidate_substitution_cache
 from .qommon import get_cfg
 from .publisher import UnpicklerClass
@@ -69,7 +69,10 @@ SQL_TYPE_MAPPING = {
 def pickle_loads(value):
     if hasattr(value, 'tobytes'):
         value = value.tobytes()
-    return UnpicklerClass(BytesIO(force_bytes(value)), **PICKLE_KWARGS).load()
+    obj = UnpicklerClass(BytesIO(force_bytes(value)), **PICKLE_KWARGS).load()
+    if six.PY3:
+        obj = deep_bytes2str(obj)
+    return obj
 
 
 class Criteria(qommon.storage.Criteria):

wcs/workflows.py

@@ -33,7 +33,7 @@ from quixote import get_request, get_response, redirect
 
 from .qommon import _, force_str
 from .qommon.misc import C_, get_as_datetime, file_digest, get_foreground_colour
-from .qommon.storage import StorableObject, atomic_write, NotEqual, Contains, Null
+from .qommon.storage import StorableObject, atomic_write, NotEqual, Contains, Null, pickle_2to3_conversion
 from .qommon.form import *
 from .qommon.humantime import seconds2humanduration
 from .qommon import emails, get_cfg, get_logger
@@ -483,6 +483,7 @@ class Workflow(StorableObject):
 
     def __setstate__(self, dict):
         self.__dict__.update(dict)
+        pickle_2to3_conversion(self)
         for s in self.possible_status + (self.global_actions or []):
             s.parent = self
             triggers = getattr(s, 'triggers', None) or []