storage: unpickle python2 strings as bytes (#36515)

(it would be so much easier to pass encoding='utf-8' and have them
converted but this will fail for objects with datetime/date/time
instances as those requires encoding='latin-1)

wcs/formdef.py

@@ -30,7 +30,7 @@ from django.utils.encoding import force_text
 from quixote import get_request, get_publisher
 from quixote.http_request import Upload
 
-from .qommon import _, force_str
+from .qommon import _, force_str, PICKLE_KWARGS
 from .qommon.storage import StorableObject, fix_key
 from .qommon.cron import CronJob
 from .qommon.form import *
@@ -1450,7 +1450,7 @@ class FormDef(StorableObject):
             return o
         if cls.lightweight:
             try:
-                o.fields = pickle.load(fd)
+                o.fields = pickle.load(fd, **PICKLE_KWARGS)
             except EOFError:
                 pass  # old format
         return o

wcs/qommon/__init__.py

@@ -41,6 +41,11 @@ if six.PY2:
 else:
     force_str = force_text
 
+if six.PY2:
+    PICKLE_KWARGS = {}
+else:
+    # unpickle python2 strings as bytes
+    PICKLE_KWARGS = {'encoding': 'bytes'}
 
 def _(message):
     pub = get_publisher()

wcs/qommon/storage.py

@@ -30,6 +30,7 @@ from django.utils.six.moves import _thread
 from .vendor import locket
 
 from quixote import get_publisher
+from . import PICKLE_KWARGS
 
 
 def cache_umask():
@@ -391,7 +392,7 @@ class StorableObject(object):
             unpickler = get_publisher().unpickler_class
         else:
             unpickler = pickle.Unpickler
-        return unpickler(fd).load()
+        return unpickler(fd, **PICKLE_KWARGS).load()
 
     @classmethod
     def get_filename(cls, filename, ignore_errors=False, ignore_migration=False, **kwargs):

wcs/sql.py

@@ -30,6 +30,7 @@ from django.utils.six import BytesIO
 
 from quixote import get_publisher
 from . import qommon
+from wcs.qommon import PICKLE_KWARGS
 from .qommon.storage import _take, parse_clause as parse_storage_clause
 from .qommon.substitution import invalidate_substitution_cache
 from .qommon import get_cfg
@@ -68,7 +69,7 @@ SQL_TYPE_MAPPING = {
 def pickle_loads(value):
     if hasattr(value, 'tobytes'):
         value = value.tobytes()
-    return UnpicklerClass(BytesIO(force_bytes(value))).load()
+    return UnpicklerClass(BytesIO(force_bytes(value)), **PICKLE_KWARGS).load()
 
 
 class Criteria(qommon.storage.Criteria):