admin: change export/import to use XML files for forms and workflows (#34705)

tests/test_admin_pages.py

@@ -3988,9 +3988,11 @@ def test_settings_export_import(pub):
     zip_content = StringIO.StringIO(resp.body)
     zipf = zipfile.ZipFile(zip_content, 'a')
     filelist = zipf.namelist()
-    assert 'formdefs/1' in filelist
-    assert 'workflows/1' in filelist
-    assert 'models/export_to_model-1.upload' in filelist
+    assert 'formdefs/1' not in filelist
+    assert 'formdefs_xml/1' in filelist
+    assert 'workflows/1' not in filelist
+    assert 'workflows_xml/1' in filelist
+    assert 'models/export_to_model-1.upload' not in filelist
     assert 'roles/1' in filelist
     assert 'categories/1' in filelist
     assert 'datasources/1' in filelist
@@ -4015,6 +4017,65 @@ def test_settings_export_import(pub):
     assert 'Imported successfully' in resp.body
     assert '1 forms' in resp.body
 
+    # check roles are found by name
+    wipe()
+    role = Role(name='qux')
+    role.store()
+
+    workflow = Workflow(name='Workflow One')
+    st1 = workflow.add_status(name='st1')
+    commentable = CommentableWorkflowStatusItem()
+    commentable.id = '_commentable'
+    commentable.by = [role.id]
+    commentable.label = 'foobar'
+    st1.items.append(commentable)
+    commentable.parent = st1
+    workflow.store()
+
+    formdef = FormDef()
+    formdef.name = 'foo'
+    formdef.workflow_id = workflow.id
+    formdef.roles = [role.id]
+    formdef.backoffice_submission_roles = [role.id]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    resp = app.get('/backoffice/settings/export')
+    resp.form['formdefs'] = True
+    resp.form['workflows'] = True
+    resp.form['roles'] = False
+    resp.form['categories'] = False
+    resp.form['datasources'] = False
+    resp.form['wscalls'] = False
+    resp = resp.form.submit('submit')
+    assert resp.location.startswith('http://example.net/backoffice/settings/export?job=')
+    job_id = urlparse.parse_qs(urlparse.urlparse(resp.location).query)['job'][0]
+    resp = resp.follow()
+    resp = resp.click('Download Export')
+    zip_content = StringIO.StringIO(resp.body)
+    zipf = zipfile.ZipFile(zip_content, 'a')
+    filelist = zipf.namelist()
+    assert 'formdefs_xml/%s' % formdef.id in filelist
+    assert 'workflows_xml/%s' % workflow.id in filelist
+    assert 'roles/%s' % role.id not in filelist
+
+    FormDef.wipe()
+    Workflow.wipe()
+    Role.wipe()
+
+    # create role beforehand, it should be matched by name
+    role = Role(name='qux')
+    role.id = '012345'
+    role.store()
+
+    resp = app.get('/backoffice/settings/import')
+    resp.form['file'] = Upload('export.wcs', zip_content.getvalue())
+    resp = resp.form.submit('submit')
+    assert FormDef.select()[0].roles == ['012345']
+    assert FormDef.select()[0].backoffice_submission_roles == ['012345']
+    assert FormDef.select()[0].workflow_roles == {'_receiver': '012345'}
+    assert Workflow.select()[0].possible_status[0].items[0].by == ['012345']
+
     # do not export roles when managed by idp
     pub.cfg['sp'] = {'idp-manage-roles': True}
     pub.write_cfg()
@@ -4027,15 +4088,7 @@ def test_settings_export_import(pub):
     zip_content = StringIO.StringIO(resp.body)
     zipf = zipfile.ZipFile(zip_content, 'a')
     filelist = zipf.namelist()
-    assert 'formdefs/1' in filelist
-    assert 'workflows/1' in filelist
-    assert 'models/export_to_model-1.upload' in filelist
-    assert 'roles/1' not in filelist
-    assert 'categories/1' in filelist
-    assert 'datasources/1' in filelist
-    assert 'wscalls/corge' in filelist
-    for filename in filelist:
-        assert not '.indexes' in filename
+    assert len([x for x in filelist if 'roles/' in x]) == 0
 
 def test_settings_themes(pub):
     create_superuser(pub)

wcs/admin/settings.py

@@ -53,6 +53,7 @@ import qommon.ident
 import qommon.template
 
 from formdef import FormDef
+from workflows import Workflow
 from fields import FieldDefPage, FieldsDirectory
 from wcs.roles import Role
 
@@ -876,6 +877,8 @@ class SettingsDirectory(QommonSettingsDirectory):
                 c = StringIO()
                 z = zipfile.ZipFile(c, 'w')
                 for d in self.dirs:
+                    if d not in ('roles', 'categories', 'datasources', 'wscalls'):
+                        continue
                     path = os.path.join(self.app_dir, d)
                     if not os.path.exists(path):
                         continue
@@ -883,6 +886,19 @@ class SettingsDirectory(QommonSettingsDirectory):
                         if f == '.indexes':
                             continue
                         z.write(os.path.join(path, f), os.path.join(d, f))
+                if 'formdefs' in self.dirs:
+                    for formdef in FormDef.select():
+                        node = formdef.export_to_xml(include_id=True)
+                        misc.indent_xml(node)
+                        z.writestr(os.path.join('formdefs_xml', str(formdef.id)),
+                                '<?xml version="1.0" encoding="iso-8859-15"?>\n' + ET.tostring(node))
+                if 'workflows' in self.dirs:
+                    for workflow in Workflow.select():
+                        node = workflow.export_to_xml(include_id=True)
+                        misc.indent_xml(node)
+                        z.writestr(os.path.join('workflows_xml', str(workflow.id)),
+                                '<?xml version="1.0" encoding="iso-8859-15"?>\n' + ET.tostring(node))
+
                 if self.settings:
                     z.write(os.path.join(self.app_dir, 'config.pck'), 'config.pck')
                     for f in os.listdir(self.app_dir):
@@ -901,8 +917,6 @@ class SettingsDirectory(QommonSettingsDirectory):
                 'datasources', 'wscalls'):
             if form.get_widget(w) and form.get_widget(w).parse():
                 dirs.append(w)
-        if 'workflows' in dirs:
-            dirs.append('models')
         if not dirs and not form.get_widget('settings').parse():
             return redirect('.')
 

wcs/formdef.py

@@ -1092,6 +1092,25 @@ class FormDef(StorableObject):
                         formdef.workflow_id = w.id
                         break
 
+        def get_role_by_node(role_node):
+            role_id = None
+            value = role_node.text.encode(charset)
+            if value.startswith('_') or value == 'logged-users':
+                role_id = value
+            elif include_id:
+                role_id = role_node.attrib.get('role_id')
+                if role_id and not Role.has_key(role_id):
+                    role_id = None
+
+            if not role_id:
+                for role in Role.select(ignore_errors=True):
+                    if role.name == value:
+                        role_id = role.id
+                        break
+
+            return role_id
+
+
         roles_elements = [
             ('roles', 'user-roles'),
             ('backoffice_submission_roles', 'backoffice-submission-roles')
@@ -1103,20 +1122,7 @@ class FormDef(StorableObject):
             roles = []
             setattr(formdef, attr_name, roles)
             for child in roles_node:
-                role_id = None
-                value = child.text.encode(charset)
-                if value.startswith('_') or value == 'logged-users':
-                    role_id = value
-                elif include_id:
-                    role_id = child.attrib.get('role_id')
-                    if role_id and not Role.has_key(role_id):
-                        role_id = None
-
-                if not role_id:
-                    for role in Role.select(ignore_errors=True):
-                        if role.name == value:
-                            role_id = role.id
-                            break
+                role_id = get_role_by_node(child)
                 if role_id:
                     roles.append(role_id)
 
@@ -1125,21 +1131,7 @@ class FormDef(StorableObject):
             formdef.workflow_roles = {}
             for child in roles_node:
                 role_key = child.attrib['role_key']
-                role_id = None
-                value = child.text.encode(charset)
-                if value.startswith('_') or value == 'logged-users':
-                    role_id = value
-                elif include_id:
-                    role_id = child.attrib.get('role_id')
-                else:
-                    for role in Role.select(ignore_errors=True):
-                        if role.name == value:
-                            role_id = role.id
-                            break
-
-                if role_id and not Role.has_key(role_id):
-                    role_id = None
-
+                role_id = get_role_by_node(child)
                 formdef.workflow_roles[role_key] = role_id
 
         if tree.find('geolocations') is not None:

wcs/publisher.py

@@ -195,6 +195,8 @@ class WcsPublisher(StubWcsPublisher):
         for f in z.namelist():
             if '.indexes' in f:
                 continue
+            if os.path.dirname(f) in ('formdefs_xml', 'workflows_xml'):
+                continue
             path = os.path.join(self.app_dir, f)
             if not os.path.exists(os.path.dirname(path)):
                 os.mkdir(os.path.dirname(path))
@@ -223,6 +225,24 @@ class WcsPublisher(StubWcsPublisher):
             if results.has_key(os.path.split(f)[0]):
                 results[os.path.split(f)[0]] += 1
 
+        # second pass, workflows
+        from wcs.workflows import Workflow
+        for f in z.namelist():
+            if os.path.dirname(f) == 'workflows_xml' and os.path.basename(f):
+                workflow = Workflow.import_from_xml(z.open(f), include_id=True)
+                workflow.store()
+                results['workflows'] += 1
+
+        # third pass, forms
+        from wcs.formdef import FormDef
+        formdefs = []
+        for f in z.namelist():
+            if os.path.dirname(f) == 'formdefs_xml' and os.path.basename(f):
+                formdef = FormDef.import_from_xml(z.open(f), include_id=True)
+                formdef.store()
+                formdefs.append(formdef)
+                results['formdefs'] += 1
+
         # rebuild indexes for imported objects
         for k, v in results.items():
             if k == 'settings':
@@ -248,7 +268,7 @@ class WcsPublisher(StubWcsPublisher):
             if k == 'formdefs':
                 # in case of formdefs, we store them anew in case SQL changes
                 # are required.
-                for formdef in FormDef.select():
+                for formdef in (formdefs or FormDef.select()):
                     formdef.store()
 
         z.close()

wcs/workflows.py

@@ -907,15 +907,12 @@ class XmlSerialisable(object):
             return value
 
         # if we import using id, only look at the role_id attribute
-        if include_id:
-            if not 'role_id' in elem.attrib:
-                return None
+        if include_id and 'role_id' in elem.attrib:
             role_id = elem.attrib['role_id'].encode(charset)
             if Role.has_key(role_id):
                 return role_id
             if WorkflowStatusItem.get_expression(role_id)['type'] in ('python', 'template'):
                 return role_id
-            return None
 
         # if not using id, look up on the name
         for role in Role.select(ignore_errors=True):