Fix default values decoding as reported by issue #108

- Default and fixed values are inserted during the decode or
    encode process
  - Update tox.ini to include coverage and flake8 to environments
    and an optional environment to build source and wheel packages
This commit is contained in:
Davide Brunato 2019-05-31 09:29:41 +02:00
parent 0e9a3cb4c0
commit d21ac11dde
25 changed files with 278 additions and 174 deletions

4
.coveragerc Normal file
View File

@ -0,0 +1,4 @@
[run]
branch = True
source = xmlschema/
omit = xmlschema/tests/*

1
.gitignore vendored
View File

@ -6,6 +6,7 @@
*.json
.idea/
.tox/
.coverage
.ipynb_checkpoints/
doc/_*/
dist/

View File

@ -1,6 +1,7 @@
# Requirements for setup a development environment for the xmlschema package.
setuptools
tox
coverage
elementpath~=1.1.7
lxml
memory_profiler

40
tox.ini
View File

@ -4,14 +4,20 @@
# and then run "tox" from this directory.
[tox]
envlist = py27, py34, py35, py36, py37
envlist = py27, py35, py36, py37, py38, docs, flake8, coverage
skip_missing_interpreters = true
toxworkdir = {homedir}/.tox/xmlschema
[testenv]
deps =
lxml
elementpath~=1.1.7
docs: Sphinx
docs: sphinx_rtd_theme
flake8: flake8
coverage: coverage
commands = python xmlschema/tests/test_all.py {posargs}
whitelist_externals = make
[testenv:py27]
deps =
@ -19,3 +25,35 @@ deps =
elementpath~=1.1.7
pathlib2
commands = python xmlschema/tests/test_all.py {posargs}
[testenv:py38]
deps = elementpath~=1.1.7
commands = python xmlschema/tests/test_all.py {posargs}
[testenv:docs]
commands =
make -C doc html
make -C doc latexpdf
make -C doc doctest
[flake8]
max-line-length = 119
[testenv:flake8]
commands =
flake8 --ignore=F401,F403,F405,F811,F821 xmlschema
[testenv:coverage]
commands =
coverage run -p -m unittest
coverage combine
coverage report -m
[testenv:build]
deps =
setuptools
wheel
commands =
python setup.py clean --all
python setup.py sdist --dist-dir {toxinidir}/dist
python setup.py bdist_wheel --universal --dist-dir {toxinidir}/dist

View File

@ -314,6 +314,7 @@ class ParkerConverter(XMLSchemaConverter):
XML Schema based converter class for Parker convention.
ref: http://wiki.open311.org/JSON_and_XML_Conversion/#the-parker-convention
ref: https://developer.mozilla.org/en-US/docs/Archive/JXON#The_Parker_Convention
:param namespaces: Map from namespace prefixes to URI.
:param dict_class: Dictionary class to use for decoded data. Default is `dict` for \
@ -780,8 +781,8 @@ class JsonMLConverter(XMLSchemaConverter):
if data_len <= content_index:
return ElementData(xsd_element.name, None, [], attributes)
elif data_len == content_index + 1 and (xsd_element.type.is_simple()
or xsd_element.type.has_simple_content()):
elif data_len == content_index + 1 and \
(xsd_element.type.is_simple() or xsd_element.type.has_simple_content()):
return ElementData(xsd_element.name, obj[content_index], [], attributes)
else:
cdata_num = iter(range(1, data_len))

View File

@ -56,4 +56,3 @@ class XMLSchemaRegexError(XMLSchemaException, ValueError):
class XMLSchemaWarning(Warning):
"""Base warning class for the XMLSchema package."""

View File

@ -51,8 +51,8 @@ I_SHORTCUT_SET = UnicodeSubset(I_SHORTCUT_REPLACE)
C_SHORTCUT_SET = UnicodeSubset(C_SHORTCUT_REPLACE)
W_SHORTCUT_SET = UnicodeSubset()
W_SHORTCUT_SET._code_points = sorted(
UNICODE_CATEGORIES['P'].code_points + UNICODE_CATEGORIES['Z'].code_points +
UNICODE_CATEGORIES['C'].code_points, key=lambda x: x if isinstance(x, int) else x[0]
UNICODE_CATEGORIES['P'].code_points + UNICODE_CATEGORIES['Z'].code_points + UNICODE_CATEGORIES['C'].code_points,
key=lambda x: x if isinstance(x, int) else x[0]
)
# Single and Multi character escapes

View File

@ -121,11 +121,8 @@ class TestPackaging(unittest.TestCase):
message = "\nFound a debug missing statement at line %d or file %r: %r"
filename = None
file_excluded = []
files = (
glob.glob(os.path.join(self.source_dir, '*.py')) +
files = glob.glob(os.path.join(self.source_dir, '*.py')) + \
glob.glob(os.path.join(self.source_dir, 'validators/*.py'))
)
for line in fileinput.input(files):
if fileinput.isfirstline():
filename = fileinput.filename()

View File

@ -95,7 +95,6 @@ class TestXMLSchema10(XMLSchemaTestCase):
<annotation/>
<list itemType="string"/>
</simpleType>
<simpleType name="test_union">
<annotation/>
<union memberTypes="string integer boolean"/>
@ -156,7 +155,6 @@ class TestXMLSchema10(XMLSchemaTestCase):
<totalDigits value="20" />
</restriction>
</simpleType>
<simpleType name="ntype">
<restriction base="ns:dtype">
<totalDigits value="3" />
@ -360,11 +358,9 @@ class TestXMLSchema10(XMLSchemaTestCase):
<maxInclusive value="100"/>
</restriction>
</simpleType>
<simpleType name="Integer">
<union memberTypes="int ns:IntegerString"/>
</simpleType>
<simpleType name="IntegerString">
<restriction base="string">
<pattern value="-?[0-9]+(\.[0-9]+)?%"/>
@ -421,7 +417,7 @@ class TestXMLSchema10(XMLSchemaTestCase):
def test_base_schemas(self):
from xmlschema.validators.schema import XML_SCHEMA_FILE
schema = self.schema_class(XML_SCHEMA_FILE)
self.schema_class(XML_SCHEMA_FILE)
def test_recursive_complex_type(self):
schema = self.schema_class("""
@ -567,7 +563,7 @@ class TestXMLSchema11(TestXMLSchema10):
self.assertTrue(xsd_type.is_valid(etree_element('a', attrib={'min': '25', 'max': '100'})))
def test_open_content(self):
schema = self.check_schema("""
self.check_schema("""
<element name="Book">
<complexType>
<openContent mode="interleave">
@ -583,6 +579,7 @@ class TestXMLSchema11(TestXMLSchema10):
</complexType>
</element>""")
def make_schema_test_class(test_file, test_args, test_num, schema_class, check_with_lxml):
"""
Creates a schema test class.

View File

@ -24,7 +24,7 @@ from elementpath import datatypes
import xmlschema
from xmlschema import (
XMLSchemaEncodeError, XMLSchemaValidationError, XMLSchema, ParkerConverter,
XMLSchemaEncodeError, XMLSchemaValidationError, ParkerConverter,
BadgerFishConverter, AbderaConverter, JsonMLConverter
)
from xmlschema.compat import unicode_type, ordered_dict_class
@ -912,6 +912,47 @@ class TestDecoding(XMLSchemaTestCase):
self.check_decode(decimal_or_nan, '95.0', Decimal('95.0'))
self.check_decode(decimal_or_nan, 'NaN', u'NaN')
def test_default_values(self):
# From issue #108
xsd_text = """<?xml version="1.0" encoding="utf-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="root" type="root" default="default_value"/>
<xs:complexType name="root">
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="attr" type="xs:string"/>
<xs:attribute name="attrWithDefault" type="xs:string" default="default_value"/>
<xs:attribute name="attrWithFixed" type="xs:string" fixed="fixed_value"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:element name="simple_root" type="xs:string" default="default_value"/>
</xs:schema>"""
schema = self.schema_class(xsd_text)
self.assertEqual(schema.to_dict("<root>text</root>"),
{'@attrWithDefault': 'default_value',
'@attrWithFixed': 'fixed_value',
'$': 'text'})
self.assertEqual(schema.to_dict("<root/>"),
{'@attrWithDefault': 'default_value',
'@attrWithFixed': 'fixed_value',
'$': 'default_value'})
self.assertEqual(schema.to_dict("""<root attr="attr_value">text</root>"""),
{'$': 'text',
'@attr': 'attr_value',
'@attrWithDefault': 'default_value',
'@attrWithFixed': 'fixed_value'})
self.assertEqual(schema.to_dict("<root>text</root>", use_defaults=False),
{'@attrWithFixed': 'fixed_value', '$': 'text'})
self.assertEqual(schema.to_dict("""<root attr="attr_value">text</root>""", use_defaults=False),
{'$': 'text', '@attr': 'attr_value', '@attrWithFixed': 'fixed_value'})
self.assertEqual(schema.to_dict("<root/>", use_defaults=False), {'@attrWithFixed': 'fixed_value'})
self.assertEqual(schema.to_dict("<simple_root/>"), 'default_value')
self.assertIsNone(schema.to_dict("<simple_root/>", use_defaults=False))
class TestDecoding11(TestDecoding):
schema_class = XMLSchema11

View File

@ -230,7 +230,7 @@ class XsdAttribute(XsdComponent, ValidationMixin):
yield obj
def iter_decode(self, text, validation='lax', **kwargs):
if not text and kwargs.get('use_defaults', True) and self.default is not None:
if not text and self.default is not None:
text = self.default
if self.fixed is not None and text != self.fixed and validation != 'skip':
yield self.validation_error(validation, "value differs from fixed value", text, **kwargs)
@ -548,6 +548,20 @@ class XsdAttributeGroup(MutableMapping, XsdComponent, ValidationMixin):
if k is not None and v.use == 'required':
yield k
def iter_predefined(self, use_defaults=True):
if use_defaults:
for k, v in self._attribute_group.items():
if k is None:
continue
elif v.fixed is not None:
yield k, v.fixed
elif v.default is not None:
yield k, v.default
else:
for k, v in self._attribute_group.items():
if k is not None and v.fixed is not None:
yield k, v.fixed
def iter_components(self, xsd_classes=None):
if xsd_classes is None or isinstance(self, xsd_classes):
yield self
@ -561,8 +575,18 @@ class XsdAttributeGroup(MutableMapping, XsdComponent, ValidationMixin):
if not attrs and not self:
return
if validation != 'skip' and any(k not in attrs for k in self.iter_required()):
missing_attrs = {k for k in self.iter_required() if k not in attrs}
reason = "missing required attributes: %r" % missing_attrs
yield self.validation_error(validation, reason, attrs, **kwargs)
use_defaults = kwargs.get('use_defaults', True)
additional_attrs = {k: v for k, v in self.iter_predefined(use_defaults) if k not in attrs}
if additional_attrs:
attrs = {k: v for k, v in attrs.items()}
attrs.update(additional_attrs)
result_list = []
required_attributes = {a for a in self.iter_required()}
for name, value in attrs.items():
try:
xsd_attribute = self[name]
@ -584,8 +608,6 @@ class XsdAttributeGroup(MutableMapping, XsdComponent, ValidationMixin):
reason = "%r attribute not allowed for element." % name
yield self.validation_error(validation, reason, attrs, **kwargs)
continue
else:
required_attributes.discard(name)
for result in xsd_attribute.iter_decode(value, validation, **kwargs):
if isinstance(result, XMLSchemaValidationError):
@ -594,21 +616,22 @@ class XsdAttributeGroup(MutableMapping, XsdComponent, ValidationMixin):
result_list.append((name, result))
break
if required_attributes and validation != 'skip':
reason = "missing required attributes: %r" % required_attributes
yield self.validation_error(validation, reason, attrs, **kwargs)
yield result_list
def iter_encode(self, attrs, validation='lax', **kwargs):
result_list = []
required_attributes = {a for a in self.iter_required()}
try:
attrs = attrs.items()
except AttributeError:
pass
if validation != 'skip' and any(k not in attrs for k in self.iter_required()):
missing_attrs = {k for k in self.iter_required() if k not in attrs}
reason = "missing required attributes: %r" % missing_attrs
yield self.validation_error(validation, reason, attrs, **kwargs)
for name, value in attrs:
use_defaults = kwargs.get('use_defaults', True)
additional_attrs = {k: v for k, v in self.iter_predefined(use_defaults) if k not in attrs}
if additional_attrs:
attrs = {k: v for k, v in attrs.items()}
attrs.update(additional_attrs)
result_list = []
for name, value in attrs.items():
try:
xsd_attribute = self[name]
except KeyError:
@ -630,17 +653,13 @@ class XsdAttributeGroup(MutableMapping, XsdComponent, ValidationMixin):
reason = "%r attribute not allowed for element." % name
yield self.validation_error(validation, reason, attrs, **kwargs)
continue
else:
required_attributes.discard(name)
for result in xsd_attribute.iter_encode(value, validation, **kwargs):
if isinstance(result, XMLSchemaValidationError):
yield result
else:
if result is not None:
result_list.append((name, result))
break
if required_attributes and validation != 'skip':
reason = "missing required attributes %r" % required_attributes
yield self.validation_error(validation, reason, attrs, **kwargs)
yield result_list

View File

@ -486,9 +486,15 @@ class XsdElement(XsdComponent, ValidationMixin, ParticleMixin, ElementPathMixin)
reason = "xsi:nil attribute must has a boolean value."
yield self.validation_error(validation, reason, elem, **kwargs)
if xsd_type.is_simple():
if not xsd_type.has_simple_content():
for result in xsd_type.content_type.iter_decode(elem, validation, converter, level=level + 1, **kwargs):
if isinstance(result, XMLSchemaValidationError):
yield self.validation_error(validation, result, elem, **kwargs)
else:
content = result
else:
if len(elem) and validation != 'skip':
reason = "a simpleType element can't has child elements."
reason = "a simple content element can't has child elements."
yield self.validation_error(validation, reason, elem, **kwargs)
text = elem.text
@ -501,6 +507,9 @@ class XsdElement(XsdComponent, ValidationMixin, ParticleMixin, ElementPathMixin)
elif not text and use_defaults and self.default is not None:
text = self.default
if not xsd_type.is_simple():
xsd_type = xsd_type.content_type
if text is None:
for result in xsd_type.iter_decode('', validation, **kwargs):
if isinstance(result, XMLSchemaValidationError):
@ -512,25 +521,6 @@ class XsdElement(XsdComponent, ValidationMixin, ParticleMixin, ElementPathMixin)
else:
value = result
elif xsd_type.has_simple_content():
if len(elem) and validation != 'skip':
reason = "a simple content element can't has child elements."
yield self.validation_error(validation, reason, elem, **kwargs)
if elem.text is not None:
text = elem.text or self.default if use_defaults else elem.text
for result in xsd_type.content_type.iter_decode(text, validation, **kwargs):
if isinstance(result, XMLSchemaValidationError):
yield self.validation_error(validation, result, elem, **kwargs)
else:
value = result
else:
for result in xsd_type.content_type.iter_decode(elem, validation, converter, level=level + 1, **kwargs):
if isinstance(result, XMLSchemaValidationError):
yield self.validation_error(validation, result, elem, **kwargs)
else:
content = result
if isinstance(value, Decimal):
try:
value = kwargs['decimal_type'](value)

View File

@ -9,8 +9,7 @@
# @author Davide Brunato <brunato@sissa.it>
#
"""
This module contains functions and classes for managing namespaces's
XSD declarations/definitions.
This module contains functions and classes for namespaces XSD declarations/definitions.
"""
from __future__ import unicode_literals
import re

View File

@ -204,8 +204,8 @@ class XsdIdentity(XsdComponent):
values[v] += 1
for value, count in values.items():
if count > 1:
yield XMLSchemaValidationError(self, elem, reason="duplicated value %r." % value)
if value and count > 1:
yield XMLSchemaValidationError(self, elem, reason="duplicated value {!r}.".format(value))
class XsdUnique(XsdIdentity):
@ -300,6 +300,6 @@ class XsdKeyref(XsdIdentity):
continue
if v not in refer_values:
reason = "Key %r with value %r not found for identity constraint of element %r." \
% (self.prefixed_name, v, qname_to_prefixed(elem.tag, self.namespaces))
reason = "Key {!r} with value {!r} not found for identity constraint of element {!r}." \
.format(self.prefixed_name, v, qname_to_prefixed(elem.tag, self.namespaces))
yield XMLSchemaValidationError(validator=self, obj=elem, reason=reason)

View File

@ -25,6 +25,9 @@ from .xsdbase import ValidationMixin, XsdComponent, ParticleMixin
class XsdWildcard(XsdComponent, ValidationMixin):
names = {}
namespace = '##any'
not_namespace = ()
not_qname = ()
def __init__(self, elem, schema, parent):
if parent is None:
@ -40,15 +43,15 @@ class XsdWildcard(XsdComponent, ValidationMixin):
super(XsdWildcard, self)._parse()
# Parse namespace and processContents
namespace = self.elem.get('namespace', '##any')
items = namespace.strip().split()
if len(items) == 1 and items[0] in ('##any', '##other', '##local', '##targetNamespace'):
self.namespace = namespace.strip()
elif not all(not s.startswith('##') or s in {'##local', '##targetNamespace'} for s in items):
namespace = self.elem.get('namespace', '##any').strip()
if namespace == '##any':
pass
elif namespace in {'##other', '##local', '##targetNamespace'}:
self.namespace = namespace
elif not all(not s.startswith('##') or s in {'##local', '##targetNamespace'} for s in namespace.split()):
self.parse_error("wrong value %r for 'namespace' attribute." % namespace)
self.namespace = '##any'
else:
self.namespace = namespace.strip()
self.namespace = namespace
self.process_contents = self.elem.get('processContents', 'strict')
if self.process_contents not in {'lax', 'skip', 'strict'}:
@ -99,7 +102,15 @@ class XsdWildcard(XsdComponent, ValidationMixin):
return self.is_namespace_allowed(default_namespace)
def is_namespace_allowed(self, namespace):
if self.namespace == '##any' or namespace == XSI_NAMESPACE:
if self.not_namespace:
if '##local' in self.not_namespace and namespace == '':
return False
elif '##targetNamespace' in self.not_namespace and namespace == self.target_namespace:
return False
else:
return namespace not in self.not_namespace
elif self.namespace == '##any' or namespace == XSI_NAMESPACE:
return True
elif self.namespace == '##other':
if namespace:
@ -351,54 +362,6 @@ class XsdAnyAttribute(XsdWildcard):
yield self.validation_error(validation, reason, attribute, **kwargs)
class Xsd11Wildcard(XsdWildcard):
def _parse(self):
super(Xsd11Wildcard, self)._parse()
# Parse notNamespace attribute
try:
not_namespace = self.elem.attrib['notNamespace'].strip()
except KeyError:
self.not_namespace = None
else:
if 'namespace' in self.elem.attrib:
self.not_namespace = None
self.parse_error("'namespace' and 'notNamespace' attributes are mutually exclusive.")
elif not_namespace in ('##local', '##targetNamespace'):
self.not_namespace = not_namespace
else:
self.not_namespace = not_namespace.split()
# Parse notQName attribute
try:
not_qname = self.elem.attrib['notQName'].strip()
except KeyError:
self.not_qname = None
else:
if not_qname in ('##defined', '##definedSibling'):
self.not_qname = not_qname
else:
self.not_qname = not_qname.split()
def is_namespace_allowed(self, namespace):
if self.namespace == '##any' or namespace == XSI_NAMESPACE:
return True
elif self.namespace == '##other':
if namespace:
return namespace != self.target_namespace
else:
return False
else:
any_namespaces = self.namespace.split()
if '##local' in any_namespaces and namespace == '':
return True
elif '##targetNamespace' in any_namespaces and namespace == self.target_namespace:
return True
else:
return namespace in any_namespaces
class Xsd11AnyElement(XsdAnyElement):
"""
Class for XSD 1.1 'any' declarations.
@ -415,7 +378,32 @@ class Xsd11AnyElement(XsdAnyElement):
Content: (annotation?)
</any>
"""
def _parse(self):
super(Xsd11AnyElement, self)._parse()
# Parse notNamespace attribute
try:
not_namespace = self.elem.attrib['notNamespace'].strip().split()
except KeyError:
pass
else:
if 'namespace' in self.elem.attrib:
self.parse_error("'namespace' and 'notNamespace' attributes are mutually exclusive.")
elif not all(not s.startswith('##') or s in {'##local', '##targetNamespace'} for s in not_namespace):
self.parse_error("wrong value %r for 'notNamespace' attribute." % self.elem.attrib['notNamespace'])
else:
self.not_namespace = not_namespace
# Parse notQName attribute
try:
not_qname = self.elem.attrib['notQName'].strip().split()
except KeyError:
pass
else:
if not all(not s.startswith('##') or s in {'##defined', '##definedSibling'} for s in not_qname):
self.parse_error("wrong value %r for 'notQName' attribute." % self.elem.attrib['notQName'])
else:
self.not_qname = not_qname
class Xsd11AnyAttribute(XsdAnyAttribute):
@ -432,7 +420,32 @@ class Xsd11AnyAttribute(XsdAnyAttribute):
Content: (annotation?)
</anyAttribute>
"""
def _parse(self):
super(Xsd11AnyAttribute, self)._parse()
# Parse notNamespace attribute
try:
not_namespace = self.elem.attrib['notNamespace'].strip().split()
except KeyError:
pass
else:
if 'namespace' in self.elem.attrib:
self.parse_error("'namespace' and 'notNamespace' attributes are mutually exclusive.")
elif not all(not s.startswith('##') or s in {'##local', '##targetNamespace'} for s in not_namespace):
self.parse_error("wrong value %r for 'notNamespace' attribute." % self.elem.attrib['notNamespace'])
else:
self.not_namespace = not_namespace
# Parse notQName attribute
try:
not_qname = self.elem.attrib['notQName'].strip().split()
except KeyError:
pass
else:
if not all(not s.startswith('##') or s == '##defined' for s in not_qname):
self.parse_error("wrong value %r for 'notQName' attribute." % self.elem.attrib['notQName'])
else:
self.not_qname = not_qname
class XsdOpenContent(XsdComponent):
@ -467,6 +480,10 @@ class XsdOpenContent(XsdComponent):
if child is not None and child.tag == XSD_ANY:
self.any_element = Xsd11AnyElement(child, self.schema, self)
@property
def built(self):
return True
class XsdDefaultOpenContent(XsdOpenContent):
"""