serve new site in utf-8; add option to migate old sites to utf-8

wcs/admin/settings.ptl

@@ -143,7 +143,7 @@ class SettingsDirectory(AccessControlled, Directory):
     _q_exports = ['', 'themes', 'users',
             'template', 'misc', 'emails', 'debug_options', 'language', 
             ('import', 'p_import'), 'export', 'identification', 'sitename',
-            'texts']
+            'texts', 'utf8switch']
 
     emails = EmailsDirectory()
     identification = IdentificationDirectory()
@@ -171,6 +171,16 @@ class SettingsDirectory(AccessControlled, Directory):
 
         '<h2>%s</h2>' % _('Customisation')
 
+        if not get_cfg('misc', {}).get('charset'):
+            '<div class="infonotice">'
+            '<p>'
+            _('This site is still using ISO-8859-15 as its character set; '\
+              'it is advised to update to UTF-8.')
+            ' '
+            '<a href="utf8switch">%s</a>' % _('Switch to UTF-8 encoding')
+            '</p>'
+            '</div>'
+
         '<dl>'
         '<dt><a href="sitename">%s</a></dt> <dd>%s</dd>' % (
                 _('Site Name'), _('Configure site name'))
@@ -527,6 +537,88 @@ class SettingsDirectory(AccessControlled, Directory):
             redirect('.')
 
 
+    def utf8switch(self):
+        def toutf8(x):
+            if x is None:
+                return None
+            return unicode(x, 'iso-8859-1').encode('utf-8')
+
+        all_elems = []
+
+        from formdef import FormDef
+        for formdef in FormDef.select():
+            formdef.name = toutf8(formdef.name)
+            for field in formdef.fields:
+                for attr in ('label', 'hint', 'condition', 'items'):
+                    if hasattr(field, attr) and type(getattr(field, attr)) is str:
+                        setattr(field, attr, toutf8(getattr(field, attr)))
+                    elif hasattr(field, attr) and type(getattr(field, attr)) is list:
+                        setattr(field, attr, [toutf8(x) for x in getattr(field, attr)])
+                if hasattr(field, 'prefill') and getattr(field, 'prefill'):
+                    if field.prefill.get('type') == 'string':
+                        field.prefill['value'] = toutf8(field.prefill.get('value'))
+
+            for form in formdef.data_class().select():
+                for k, v in form.data.items():
+                    if type(v) is str:
+                        form.data[k] = toutf8(v)
+                if form.evolution:
+                    for evo in form.evolution:
+                        if evo.comment:
+                            evo.comment = toutf8(evo.comment)
+                all_elems.append(form)
+            all_elems.append(formdef)
+
+        from workflows import Workflow
+        for workflow in Workflow.select():
+            workflow.name = toutf8(workflow.name)
+            if workflow.possible_status:
+                for status in workflow.possible_status:
+                    status.name = toutf8(status.name)
+                    if not status.items:
+                        continue
+                    for item in status.items:
+                        for attr in ('label', 'subject', 'body', 'message'):
+                            if hasattr(item, attr) and type(getattr(item, attr)) is str:
+                                setattr(item, attr, toutf8(getattr(item, attr)))
+            all_elems.append(workflow)
+
+        from categories import Category
+        for category in Category.select():
+            category.name = toutf8(category.name)
+            category.description = toutf8(category.description)
+            all_elems.append(category)
+
+        from roles import Role
+        for role in Role.select():
+            role.name = toutf8(role.name)
+            role.details = toutf8(role.details)
+            all_elems.append(role)
+
+        from users import User
+        for user in User.select():
+            user.name = toutf8(user.name)
+            if hasattr(user, 'formdata') and user.formdata:
+                for k, v in user.formdata.items():
+                    if type(v) is str:
+                        user.formdata[k] = toutf8(v)
+            all_elems.append(user)
+
+        for cfg_section in get_publisher().cfg.values():
+            for k, v in cfg_section.items():
+                if type(v) is str:
+                    cfg_section[k] = toutf8(v)
+
+        for elem in all_elems:
+            elem.store()
+
+        misc_cfg = get_cfg('misc', {})
+        misc_cfg['charset'] = 'utf-8'
+        get_publisher().cfg['misc'] = misc_cfg
+        get_publisher().write_cfg()
+
+        redirect('.')
+
 
 
 def error_page [html] (error_message):

wcs/publisher.py

@@ -66,6 +66,22 @@ class WcsPublisher(StubWcsPublisher):
         return admin
     get_admin_module = classmethod(get_admin_module)
 
+    def set_config(self, request = None):
+        QommonPublisher.set_config(self, request = request)
+        if request:
+            filename = os.path.join(self.app_dir, 'config.pck')
+            if os.path.exists(filename):
+                # a configuration file exists, it may not have a charset
+                # specified so we default to iso-8859-15 as it was the only
+                # possible value
+                self.site_charset = self.cfg.get('misc', {}).get('charset', 'iso-8859-15')
+            else:
+                # new site, set the charset, and write it down for further
+                # requests
+                self.site_charset = 'utf-8'
+                self.cfg['misc'] = {'charset': 'utf-8'}
+                self.write_cfg()
+            request.response.charset = self.site_charset
 
 set_publisher_class(WcsPublisher)
 WcsPublisher.register_extra_dir(os.path.join(os.path.dirname(__file__), 'extra'))