general: revert switch to lxml, go back to ElementTree (#78824)

pylint.rc

@@ -1,7 +1,6 @@
 [MASTER]
 persistent=yes
 ignore=vendor,Bouncers,ezt.py
-extension-pkg-allow-list=lxml.etree
 
 [MESSAGES CONTROL]
 disable=

tests/admin_pages/test_card.py

@@ -1,8 +1,8 @@
 import os
 import re
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from webtest import Upload
 
 from wcs import fields

tests/admin_pages/test_datasource.py

@@ -2,11 +2,11 @@ import io
 import json
 import os
 import re
+import xml.etree.ElementTree as ET
 from unittest import mock
 
 import pytest
 import responses
-from lxml import etree as ET
 from webtest import Upload
 
 from wcs import fields

tests/admin_pages/test_form.py

@@ -3,11 +3,11 @@ import os
 import re
 import time
 import uuid
+import xml.etree.ElementTree as ET
 
 import pytest
 import responses
 from django.utils.timezone import now
-from lxml import etree as ET
 from pyquery import PyQuery
 from webtest import Upload
 

tests/admin_pages/test_workflow.py

@@ -2,10 +2,10 @@ import io
 import os
 import re
 import uuid
+import xml.etree.ElementTree as ET
 
 import pytest
 import responses
-from lxml import etree as ET
 from quixote.http_request import Upload as QuixoteUpload
 from webtest import Radio, Upload
 

tests/admin_pages/test_wscall.py

@@ -1,7 +1,7 @@
 import io
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from webtest import Upload
 
 from wcs.qommon.http_request import HTTPRequest

tests/api/test_custom_view.py

@@ -1,9 +1,9 @@
 import io
 import os
+import xml.etree.ElementTree as ET
 import zipfile
 
 import pytest
-from lxml import etree as ET
 
 from wcs import fields
 from wcs.carddef import CardDef

tests/api/test_export_import.py

@@ -2,9 +2,9 @@ import io
 import json
 import os
 import tarfile
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 
 from wcs.applications import Application, ApplicationElement
 from wcs.blocks import BlockDef

tests/api/test_formdata.py

@@ -5,11 +5,11 @@ import json
 import os
 import re
 import time
+import xml.etree.ElementTree as ET
 import zipfile
 
 import pytest
 from django.utils.encoding import force_bytes
-from lxml import etree as ET
 from quixote import get_publisher
 from webtest import Upload
 

tests/backoffice_pages/test_export.py

@@ -4,10 +4,10 @@ import json
 import os
 import time
 import urllib.parse
+import xml.etree.ElementTree as ET
 import zipfile
 
 import pytest
-from lxml import etree as ET
 
 from wcs import fields
 from wcs.blocks import BlockDef

tests/form_pages/test_all.py

@@ -6,13 +6,13 @@ import os
 import re
 import time
 import urllib.parse
+import xml.etree.ElementTree as ET
 import zipfile
 from unittest import mock
 
 import pytest
 import responses
 from django.utils.encoding import force_bytes, force_str
-from lxml import etree as ET
 from webtest import Hidden, Upload
 
 from wcs import fields

tests/form_pages/test_formdata.py

@@ -3,13 +3,13 @@ import io
 import json
 import os
 import urllib.parse
+import xml.etree.ElementTree as ET
 import zipfile
 from unittest import mock
 
 import pytest
 import responses
 from django.utils.encoding import force_bytes
-from lxml import etree as ET
 from quixote.http_request import Upload as QuixoteUpload
 from webtest import Hidden, Upload
 
@@ -59,7 +59,7 @@ def assert_equal_zip(stream1, stream2):
                 t1, t2 = ET.tostring(ET.XML(z1.read(name))), ET.tostring(ET.XML(z2.read(name)))
                 try:
                     # >= python 3.8: tostring preserves attribute order; use canonicalize to sort them
-                    t1, t2 = ET.canonicalize(t1.decode("utf-8")), ET.canonicalize(t2.decode("utf-8"))
+                    t1, t2 = ET.canonicalize(t1), ET.canonicalize(t2)
                 except AttributeError:
                     pass
             else:

tests/test_block.py

@@ -1,8 +1,10 @@
+import xml.etree.ElementTree as ET
+
 import pytest
-from lxml import etree as ET
 
 from wcs.blocks import BlockDef
 from wcs.categories import BlockCategory
+from wcs.qommon.misc import indent_xml as indent
 
 from .utilities import clean_temporary_pub, create_temporary_pub
 
@@ -18,7 +20,7 @@ def teardown_module(module):
 
 def export_to_indented_xml(block, include_id=False):
     block_xml = block.export_to_xml(include_id=include_id)
-    ET.indent(block_xml)
+    indent(block_xml)
     return block_xml
 
 

tests/test_blockdef_import.py

@@ -1,11 +1,12 @@
 import io
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 
 from wcs import fields
 from wcs.blocks import BlockDef, BlockdefImportError
 from wcs.carddef import CardDef
+from wcs.qommon.misc import indent_xml as indent
 
 from .utilities import clean_temporary_pub, create_temporary_pub
 
@@ -21,7 +22,7 @@ def teardown_module(module):
 
 def export_to_indented_xml(blockdef, include_id=False):
     blockdef_xml = ET.fromstring(ET.tostring(blockdef.export_to_xml(include_id=include_id)))
-    ET.indent(blockdef_xml)
+    indent(blockdef_xml)
     return blockdef_xml
 
 

tests/test_carddef.py

@@ -1,8 +1,8 @@
 import io
 import json
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
@@ -11,6 +11,7 @@ from wcs.data_sources import NamedDataSource
 from wcs.fields import BlockField, ComputedField, ItemField, ItemsField, StringField
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.misc import indent_xml as indent
 from wcs.qommon.template import Template
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -32,7 +33,7 @@ def teardown_module(module):
 
 def export_to_indented_xml(carddef, include_id=False):
     carddef_xml = ET.fromstring(ET.tostring(carddef.export_to_xml(include_id=include_id)))
-    ET.indent(carddef_xml)
+    indent(carddef_xml)
     return carddef_xml
 
 

tests/test_comment_template.py

@@ -1,9 +1,9 @@
 import io
 import os
 import re
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from quixote import cleanup
 from webtest import Upload
 
@@ -13,6 +13,7 @@ from wcs.fields import FileField
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.ident.password_accounts import PasswordAccount
+from wcs.qommon.misc import indent_xml as indent
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.workflows import Workflow
 
@@ -473,7 +474,7 @@ def test_comment_templates_duplicate(pub, superuser, comment_template):
 
 def export_to_indented_xml(comment_template, include_id=False):
     comment_template_xml = comment_template.export_to_xml(include_id=include_id)
-    ET.indent(comment_template_xml)
+    indent(comment_template_xml)
     return comment_template_xml
 
 

tests/test_datasource.py

@@ -2,10 +2,10 @@ import codecs
 import json
 import os
 import urllib.parse
+import xml.etree.ElementTree as ET
 
 import pytest
 import responses
-from lxml import etree as ET
 
 from wcs import data_sources, fields
 from wcs.categories import DataSourceCategory
@@ -13,6 +13,7 @@ from wcs.data_sources import NamedDataSource, register_data_source_function
 from wcs.formdef import FormDef
 from wcs.qommon.form import Form, get_request
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.misc import indent_xml as indent
 from wcs.workflows import WorkflowStatusItem
 
 from .test_widgets import MockHtmlForm, mock_form_submission
@@ -1590,7 +1591,7 @@ def test_data_source_in_template(pub):
 
 def export_to_indented_xml(data_source, include_id=False):
     data_source_xml = data_source.export_to_xml(include_id=include_id)
-    ET.indent(data_source_xml)
+    indent(data_source_xml)
     return data_source_xml
 
 

tests/test_formdef_import.py

@@ -1,13 +1,14 @@
 import io
 import time
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import Category
 from wcs.formdef import FormDef, FormdefImportError, fields
+from wcs.qommon.misc import indent_xml as indent
 from wcs.workflows import Workflow
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -24,7 +25,7 @@ def teardown_module(module):
 
 def export_to_indented_xml(formdef, include_id=False):
     formdef_xml = ET.fromstring(ET.tostring(formdef.export_to_xml(include_id=include_id)))
-    ET.indent(formdef_xml)
+    indent(formdef_xml)
     return formdef_xml
 
 
@@ -93,9 +94,9 @@ def test_empty_display_locations_tag(pub):
     f1 = formdef_xml.findall('fields/field')[0]
     f2 = formdef_xml.findall('fields/field')[1]
     f3 = formdef_xml.findall('fields/field')[2]
-    assert '<display_locations/>' in str(ET.tostring(f1))
+    assert '<display_locations />' in str(ET.tostring(f1))
-    assert '<display_locations/>' in str(ET.tostring(f2))
+    assert '<display_locations />' in str(ET.tostring(f2))
-    assert '<display_locations/>' in str(ET.tostring(f3))
+    assert '<display_locations />' in str(ET.tostring(f3))
 
     formdef2 = assert_xml_import_export_works(formdef)
     assert formdef2.fields[0].display_locations == []
@@ -105,9 +106,9 @@ def test_empty_display_locations_tag(pub):
     f1 = formdef2_xml.findall('fields/field')[0]
     f2 = formdef2_xml.findall('fields/field')[1]
     f3 = formdef2_xml.findall('fields/field')[2]
-    assert '<display_locations/>' in str(ET.tostring(f1))
+    assert '<display_locations />' in str(ET.tostring(f1))
-    assert '<display_locations/>' in str(ET.tostring(f2))
+    assert '<display_locations />' in str(ET.tostring(f2))
-    assert '<display_locations/>' in str(ET.tostring(f3))
+    assert '<display_locations />' in str(ET.tostring(f3))
 
 
 def test_boolean_attributes(pub):

tests/test_mail_templates.py

@@ -1,9 +1,9 @@
 import io
 import os
 import re
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from quixote import cleanup
 from webtest import Upload
 
@@ -13,6 +13,7 @@ from wcs.formdef import FormDef
 from wcs.mail_templates import MailTemplate
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.ident.password_accounts import PasswordAccount
+from wcs.qommon.misc import indent_xml as indent
 from wcs.qommon.upload_storage import PicklableUpload
 from wcs.workflows import Workflow
 
@@ -502,7 +503,7 @@ def test_mail_templates_duplicate(pub, superuser, mail_template):
 
 def export_to_indented_xml(mail_template, include_id=False):
     mail_template_xml = mail_template.export_to_xml(include_id=include_id)
-    ET.indent(mail_template_xml)
+    indent(mail_template_xml)
     return mail_template_xml
 
 

tests/test_publisher.py

@@ -8,6 +8,7 @@ import re
 import shutil
 import sys
 import time
+import xml.etree.ElementTree as ET
 import zipfile
 from unittest import mock
 
@@ -16,7 +17,6 @@ from django.core.management import call_command
 from django.http import Http404
 from django.test import override_settings
 from django.utils.timezone import localtime
-from lxml import etree as ET
 from quixote import cleanup, get_publisher
 from quixote.http_request import Upload
 

tests/test_snapshots.py

@@ -1,9 +1,9 @@
 import io
 import os
 import shutil
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from quixote.http_request import Upload
 
 from wcs.blocks import BlockDef

tests/test_workflow_import.py

@@ -1,8 +1,8 @@
 import io
 import re
+import xml.etree.ElementTree as ET
 
 import pytest
-from lxml import etree as ET
 from quixote.http_request import Upload
 
 from wcs.blocks import BlockDef
@@ -12,6 +12,7 @@ from wcs.fields import BlockField, BoolField, EmailField, FileField, ItemField,
 from wcs.formdef import FormDef
 from wcs.mail_templates import MailTemplate
 from wcs.qommon.form import UploadedFile
+from wcs.qommon.misc import indent_xml as indent
 from wcs.wf.create_formdata import Mapping
 from wcs.wf.form import WorkflowFormFieldsFormDef
 from wcs.workflows import (
@@ -36,7 +37,7 @@ def teardown_module(module):
 
 def export_to_indented_xml(workflow, include_id=False):
     workflow_xml = workflow.export_to_xml(include_id=include_id)
-    ET.indent(workflow_xml)
+    indent(workflow_xml)
     return workflow_xml
 
 
@@ -167,11 +168,9 @@ def test_status_actions_named_existing_role(pub):
     commentable.by = [2]
 
     wf2 = assert_import_export_works(wf)
-    wf_xml = wf.export_to_xml()
-    ET.indent(wf_xml)
     assert re.findall(
         '<item.*role_id="2".*>Test Role named existing role</item>',
-        ET.tostring(wf_xml).decode(),
+        ET.tostring(indent(wf.export_to_xml())).decode(),
     )
     assert wf2.possible_status[0].items[0].by == ['2']
 

wcs/admin/forms.py

@@ -16,9 +16,9 @@
 
 import difflib
 import io
+import xml.etree.ElementTree as ET
 from collections import defaultdict
 
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import AccessControlled, Directory
 from quixote.html import TemplateIO, htmlescape, htmltext

wcs/admin/settings.py

@@ -24,10 +24,10 @@ try:
     import lasso
 except ImportError:
     lasso = None
+import xml.etree.ElementTree as ET
 import zipfile
 
 from django.utils.encoding import force_bytes
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import AccessControlled, Directory
 from quixote.html import TemplateIO, htmltext
@@ -38,7 +38,7 @@ from wcs.carddef import CardDef
 from wcs.data_sources import NamedDataSource
 from wcs.fields import MapOptionsMixin
 from wcs.formdef import FormDef, FormdefImportError
-from wcs.qommon import _, errors, get_cfg, ident, template
+from wcs.qommon import _, errors, get_cfg, ident, misc, template
 from wcs.qommon.admin.cfg import cfg_submit
 from wcs.qommon.admin.emails import EmailsDirectory
 from wcs.qommon.admin.texts import TextsDirectory
@@ -821,7 +821,7 @@ class SettingsDirectory(AccessControlled, Directory):
                             if ds.external == 'agenda':
                                 continue
                             node = ds.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('datasources', str(ds.id)),
                                 ET.tostring(node),
@@ -830,7 +830,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if 'formdefs' in self.dirs:
                         for formdef in FormDef.select():
                             node = formdef.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('formdefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
@@ -839,7 +839,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if 'carddefs' in self.dirs:
                         for formdef in CardDef.select():
                             node = formdef.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('carddefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
@@ -848,7 +848,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if 'workflows' in self.dirs:
                         for workflow in Workflow.select():
                             node = workflow.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('workflows_xml', str(workflow.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
@@ -857,7 +857,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if 'blockdefs' in self.dirs:
                         for blockdef in BlockDef.select():
                             node = blockdef.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('blockdefs_xml', str(blockdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
@@ -866,7 +866,7 @@ class SettingsDirectory(AccessControlled, Directory):
                     if 'roles' in self.dirs:
                         for role in get_publisher().role_class.select():
                             node = role.export_to_xml(include_id=True)
-                            ET.indent(node)
+                            misc.indent_xml(node)
                             z.writestr(
                                 os.path.join('roles_xml', str(role.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),

wcs/admin/workflows.py

@@ -19,10 +19,10 @@ import itertools
 import json
 import textwrap
 import time
+import xml.etree.ElementTree as ET
 from subprocess import PIPE, Popen
 
 from django.utils.encoding import force_bytes
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response, get_session, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext

wcs/api_export_import.py

@@ -17,11 +17,11 @@
 import io
 import json
 import tarfile
+import xml.etree.ElementTree as ET
 
 from django.http import Http404, HttpResponse, HttpResponseForbidden, JsonResponse
 from django.shortcuts import redirect
 from django.urls import reverse
-from lxml import etree as ET
 
 from wcs.api_utils import is_url_signed
 from wcs.applications import Application, ApplicationElement
@@ -46,7 +46,7 @@ from wcs.wscalls import NamedWsCall
 
 from .qommon import _
 from .qommon.afterjobs import AfterJob
-from .qommon.misc import xml_node_text
+from .qommon.misc import indent_xml, xml_node_text
 
 klasses = {
     'blocks': BlockDef,
@@ -208,7 +208,7 @@ def object_export(request, objects, slug):
         content, content_type = obj.export_for_application()
     else:
         etree = obj.export_to_xml(include_id=True)
-        ET.indent(etree)
+        indent_xml(etree)
         content = ET.tostring(etree)
         content_type = 'text/xml'
     return HttpResponse(content, content_type=content_type)

wcs/backoffice/i18n.py

@@ -15,9 +15,9 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import io
+import xml.etree.ElementTree as ET
 import zipfile
 
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response, redirect
 from quixote.directory import Directory
 from quixote.html import TemplateIO, htmltext
@@ -345,8 +345,8 @@ class ExportAfterJob(AfterJob):
             self.file_name = 'catalog.ods'
             self.content_type = 'application/vnd.oasis.opendocument.spreadsheet'
         elif self.file_format == 'xliff':
-            ET.indent(root)
+            misc.indent_xml(root)
-            output.write(ET.tostring(root, encoding='utf-8'))
+            output.write(ET.tostring(root, 'utf-8'))
             self.file_name = 'catalog.xliff'
             self.content_type = 'text/xml'
 

wcs/blocks.py

@@ -18,9 +18,9 @@ import collections
 import itertools
 import types
 import uuid
+import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response
 from quixote.html import htmltag, htmltext
 

wcs/categories.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_publisher
 from quixote.html import htmltext
 

wcs/custom_views.py

@@ -15,9 +15,9 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import urllib.parse
+import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_str
-from lxml import etree as ET
 from quixote import get_publisher
 
 from wcs.backoffice.data_management import CardPage

wcs/data_sources.py

@@ -19,11 +19,11 @@ import collections.abc
 import hashlib
 import json
 import urllib.parse
+import xml.etree.ElementTree as ET
 
 from django.core.cache import cache
 from django.template import TemplateSyntaxError, VariableDoesNotExist
 from django.utils.encoding import force_bytes, force_str
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_session
 from quixote.errors import RequestError
 from quixote.html import TemplateIO

wcs/fields.py

@@ -24,11 +24,11 @@ import random
 import re
 import sys
 import time
+import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_bytes, force_str, smart_str
 from django.utils.formats import date_format as django_date_format
 from django.utils.html import strip_tags, urlize
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_session
 from quixote.html import TemplateIO, htmlescape, htmltag, htmltext
 

wcs/formdef.py

@@ -28,10 +28,10 @@ import sys
 import time
 import types
 import uuid
+import xml.etree.ElementTree as ET
 from operator import itemgetter
 
 from django.utils.encoding import force_bytes, force_str
-from lxml import etree as ET
 from quixote import get_publisher, get_session
 from quixote.html import htmltext
 from quixote.http_request import Upload

wcs/qommon/misc.py

@@ -29,6 +29,7 @@ import subprocess
 import time
 import unicodedata
 import urllib.parse
+import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 
 import phonenumbers
@@ -44,7 +45,6 @@ from django.utils.html import strip_tags
 from django.utils.safestring import mark_safe
 from django.utils.text import Truncator
 from django.utils.timezone import is_aware, make_naive
-from lxml import etree as ET
 from PIL import Image
 from quixote import get_publisher, get_request, get_response, redirect
 from quixote.errors import RequestError
@@ -558,6 +558,23 @@ def get_foreground_colour(background_colour):
     return fg_colour
 
 
+def indent_xml(elem, level=0):
+    # in-place prettyprint formatter
+    # http://effbot.org/zone/element-lib.htm#prettyprint
+    i = "\n" + level * "  "
+    if len(elem):
+        if not elem.text or not elem.text.strip():
+            elem.text = i + "  "
+        for elem in elem:
+            indent_xml(elem, level + 1)
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+    else:
+        if level and (not elem.tail or not elem.tail.strip()):
+            elem.tail = i
+    return elem
+
+
 def xml_node_text(node):
     if node is None or node.text is None:
         return None
@@ -1167,7 +1184,7 @@ def xml_response(obj, filename, content_type='text/xml'):
     etree = obj.export_to_xml(include_id=True)
     if hasattr(obj, 'get_admin_url'):
         etree.attrib['url'] = obj.get_admin_url()
-    ET.indent(etree)
+    indent_xml(etree)
     response = get_response()
     response.set_content_type(content_type)
     response.set_header('content-disposition', 'attachment; filename=%s' % filename)

wcs/qommon/ods.py

@@ -15,10 +15,10 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import re
+import xml.etree.ElementTree as ET
 import zipfile
 
 from django.utils.encoding import force_str
-from lxml import etree as ET
 
 from .evalutils import make_date, make_datetime
 from .misc import date_format, datetime_format, strftime
@@ -149,10 +149,10 @@ class Workbook:
         return root
 
     def get_styles(self):
-        return ET.tostring(self.get_styles_node())
+        return ET.tostring(self.get_styles_node(), 'utf-8')
 
     def get_content(self):
-        return ET.tostring(self.get_content_node())
+        return ET.tostring(self.get_content_node(), 'utf-8')
 
     def save(self, output):
         with zipfile.ZipFile(output, 'w') as z:

wcs/qommon/xml_storage.py

@@ -15,12 +15,12 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import datetime
+import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_str
-from lxml import etree as ET
 from quixote import get_publisher
 
-from .misc import xml_node_text
+from .misc import indent_xml, xml_node_text
 from .storage import Equal, Or, StorableObject
 
 
@@ -74,7 +74,7 @@ class XmlStorableObject(StorableObject):
 
     def export_to_xml_string(self, include_id=False):
         x = self.export_to_xml(include_id=include_id)
-        ET.indent(x)
+        indent_xml(x)
         return ET.tostring(x)
 
     def export_roles_to_xml(self, element, attribute_name, include_id=False, **kwargs):
@@ -88,8 +88,7 @@ class XmlStorableObject(StorableObject):
     @classmethod
     def import_from_xml(cls, fd, charset=None, include_id=False):
         try:
-            parser = ET.XMLParser(encoding='utf-8', huge_tree=True)
+            tree = ET.parse(fd)
-            tree = ET.parse(fd, parser=parser)
         except Exception:
             raise ValueError()
         return cls.import_from_xml_tree(tree, charset=charset, include_id=include_id)

wcs/roles.py

@@ -16,9 +16,9 @@
 
 import json
 import urllib.parse
+import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_str
-from lxml import etree as ET
 from quixote import get_publisher
 from quixote.html import htmltext
 

wcs/snapshots.py

@@ -16,9 +16,9 @@
 
 import difflib
 import re
+import xml.etree.ElementTree as ET
 
 from django.utils.timezone import now
-from lxml import etree as ET
 from quixote import get_publisher, get_response, get_session
 
 from wcs.qommon import _, misc
@@ -29,6 +29,43 @@ class UnknownUser:
         return str(_('unknown user'))
 
 
+def indent(tree, space="  ", level=0):
+    # backport from Lib/xml/etree/ElementTree.py python 3.9
+    if isinstance(tree, ET.ElementTree):
+        tree = tree.getroot()
+    if level < 0:
+        raise ValueError(f"Initial indentation level must be >= 0, got {level}")
+    if len(tree) == 0:
+        return
+
+    # Reduce the memory consumption by reusing indentation strings.
+    indentations = ["\n" + level * space]
+
+    def _indent_children(elem, level):
+        # Start a new indentation level for the first child.
+        child_level = level + 1
+        try:
+            child_indentation = indentations[child_level]
+        except IndexError:
+            child_indentation = indentations[level] + space
+            indentations.append(child_indentation)
+
+        if not elem.text or not elem.text.strip():
+            elem.text = child_indentation
+
+        for child in elem:
+            if len(child):
+                _indent_children(child, child_level)
+            if not child.tail or not child.tail.strip():
+                child.tail = child_indentation
+
+        # Dedent after the last child by overwriting the previous indentation.
+        if not child.tail.strip():
+            child.tail = indentations[level]
+
+    _indent_children(tree, 0)
+
+
 _no_eol = "\\ No newline at end of file"
 _hdr_pat = re.compile(r"^@@ -(\d+),?(\d+)? \+(\d+),?(\d+)? @@$")
 
@@ -121,8 +158,8 @@ class Snapshot:
         # get patch beetween latest serialization and current instance
         # indent xml to minimize patch
         latest_tree = ET.fromstring(latest_complete.serialization)
-        ET.indent(tree)
+        indent(tree)
-        ET.indent(latest_tree)
+        indent(latest_tree)
         patch = make_patch(ET.tostring(latest_tree).decode('utf-8'), ET.tostring(tree).decode('utf-8'))
         # should we store a snapshot ?
         store_snapshot = True
@@ -180,7 +217,7 @@ class Snapshot:
                 return self.serialization
 
             tree = ET.fromstring(self.serialization)
-            ET.indent(tree)
+            indent(tree)
             return ET.tostring(tree).decode('utf-8')
 
         # get latest version with serialization
@@ -188,7 +225,7 @@ class Snapshot:
             self.object_type, self.object_id, complete=True, max_timestamp=self.timestamp
         )
         latest_tree = ET.fromstring(latest_complete.serialization)
-        ET.indent(latest_tree)
+        indent(latest_tree)
         serialization = apply_patch(ET.tostring(latest_tree).decode('utf-8'), self.patch or '')
         return serialization
 

wcs/wf/attachment.py

@@ -16,8 +16,8 @@
 
 import os
 import urllib.parse
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_request, redirect
 
 from wcs.forms.common import FileDirectory, FormStatusPage

wcs/wf/backoffice_fields.py

@@ -15,8 +15,8 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import copy
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_publisher
 from quixote.html import htmltext
 

wcs/wf/comment.py

@@ -15,9 +15,9 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import html
+import xml.etree.ElementTree as ET
 
 from django.utils.html import strip_tags
-from lxml import etree as ET
 from quixote.html import htmltext
 
 from wcs.qommon import _

wcs/wf/create_formdata.py

@@ -16,9 +16,9 @@
 
 import collections
 import time
+import xml.etree.ElementTree as ET
 
 from django.utils.functional import cached_property
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_session
 from quixote.html import TemplateIO, htmltext
 

wcs/wf/dispatch.py

@@ -15,8 +15,8 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import collections
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_publisher
 from quixote.html import htmltext
 

wcs/wf/export_to_model.py

@@ -24,10 +24,10 @@ import subprocess
 import tempfile
 import time
 import zipfile
+from xml.etree import ElementTree as ET
 
 from django.template.defaultfilters import filesizeformat
 from django.utils.encoding import force_bytes, force_str
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response
 from quixote.directory import Directory
 from quixote.errors import PublishError
@@ -144,6 +144,19 @@ def transform_opendocument(instream, outstream, process):
             root = ET.fromstring(content)
             process(root, new_images)
             content = ET.tostring(root)
+            if (
+                root.find(f'{{{OO_OFFICE_NS}}}body/{{{OO_OFFICE_NS}}}spreadsheet')
+                and b'xmlns:of=' not in content
+            ):
+                # force xmlns:of namespace inclusion in spreadsheet files, as it may be
+                # required for proper handling of table:formula attributes.
+                # (there is no easy way to have ElementTree include namespace declarations
+                # if there are no elements of that namespace)
+                content = content.replace(
+                    b':document-content ',
+                    b':document-content xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2" ',
+                    1,
+                )
             zout.writestr(filename, content)
 
         for filename in zin.namelist():

wcs/wf/form.py

@@ -14,8 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_publisher, get_session
 from quixote.html import TemplateIO, htmltext
 

wcs/wf/profile.py

@@ -18,8 +18,8 @@ import datetime
 import json
 import time
 import urllib.parse
+import xml.etree.ElementTree as ET
 
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response
 
 from wcs.api_utils import MissingSecret, get_secret_and_orig, sign_url

wcs/wf/wscall.py

@@ -21,9 +21,9 @@ import io
 import mimetypes
 import sys
 import traceback
+import xml.etree.ElementTree as ET
 
 from django.utils.encoding import force_str
-from lxml import etree as ET
 from quixote import get_publisher, get_request
 from quixote.html import TemplateIO, htmltext
 

wcs/workflows.py

@@ -24,12 +24,12 @@ import os
 import random
 import time
 import uuid
+import xml.etree.ElementTree as ET
 from contextlib import contextmanager
 from importlib import import_module
 
 from django.utils.encoding import force_str
 from django.utils.timezone import is_aware, localtime, make_aware, now
-from lxml import etree as ET
 from quixote import get_publisher, get_request, get_response
 from quixote.html import TemplateIO, htmltext
 

wcs/wscalls.py

@@ -17,10 +17,10 @@
 import collections
 import json
 import urllib.parse
+import xml.etree.ElementTree as ET
 
 from django.conf import settings
 from django.utils.encoding import force_str
-from lxml import etree as ET
 from quixote import get_publisher, get_request
 
 from wcs.api_utils import MissingSecret, get_secret_and_orig, sign_url