From 2fda1556ef7714d562317049f46db6ff59db5c2f Mon Sep 17 00:00:00 2001 From: "Kostis Anagnostopoulos @ STUW025" Date: Wed, 24 Sep 2014 03:20:07 +0200 Subject: [PATCH 01/27] issue #158: TRY to speed-up scope & $ref url-handling by keeping fragments separated from URL (and avoid redunant frag/defrag). Conflicts: jsonschema/tests/test_benchmarks.py issue #158: Use try-finally to ensure resolver scopes_stack empty when iteration breaks (no detectable performance penalty). * Replace non-python-2.6 DefragResult with named-tuple. * Add test-case checking scopes_stack empty. Conflicts: jsonschema/tests/test_validators.py jsonschema/validators.py --- jsonschema/compat.py | 10 +++++-- jsonschema/validators.py | 58 +++++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/jsonschema/compat.py b/jsonschema/compat.py index 6ca49ab..0afd9ea 100644 --- a/jsonschema/compat.py +++ b/jsonschema/compat.py @@ -1,6 +1,9 @@ from __future__ import unicode_literals -import sys + +from collections import namedtuple import operator +import sys + try: from collections import MutableMapping, Sequence # noqa @@ -40,6 +43,9 @@ def urlsplit(url): return SplitResult(scheme, netloc, path, query, fragment) +DefragResult = namedtuple('DefragResult', 'url fragment') + + def urldefrag(url): if "#" in url: s, n, p, q, frag = urlsplit(url) @@ -47,7 +53,7 @@ def urldefrag(url): else: defrag = url frag = '' - return defrag, frag + return DefragResult(defrag, frag) # flake8: noqa diff --git a/jsonschema/validators.py b/jsonschema/validators.py index c347bf1..8f48062 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -11,7 +11,8 @@ except ImportError: from jsonschema import _utils, _validators from jsonschema.compat import ( - Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, + Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, DefragResult, + str_types, int_types, iteritems, ) from jsonschema.exceptions import ErrorTree # Backwards compatibility # noqa @@ -79,7 +80,10 @@ def create(meta_schema, validators=(), version=None, default_types=None): # noq if _schema is None: _schema = self.schema - with self.resolver.in_scope(_schema.get(u"id", u"")): + scope = _schema.get(u"id") + if scope: + self.resolver.push_scope(scope) + try: ref = _schema.get(u"$ref") if ref is not None: validators = [(u"$ref", ref)] @@ -103,6 +107,9 @@ def create(meta_schema, validators=(), version=None, default_types=None): # noq if k != u"$ref": error.schema_path.appendleft(k) yield error + finally: + if scope: + self.resolver.pop_scope() def descend(self, instance, schema, path=None, schema_path=None): for error in self.iter_errors(instance, schema): @@ -222,7 +229,7 @@ class RefResolver(object): :argument str base_uri: URI of the referring document :argument referrer: the actual referring document - :argument dict store: a mapping from URIs to documents to cache + :argument dict store: a mapping from URIs (without fragments!) to documents to cache :argument bool cache_remote: whether remote refs should be cached after first resolution :argument dict handlers: a mapping from URI schemes to functions that @@ -233,6 +240,7 @@ class RefResolver(object): def __init__( self, base_uri, referrer, store=(), cache_remote=True, handlers=(), ): + base_uri = urldefrag(base_uri) self.base_uri = base_uri self.resolution_scope = base_uri # This attribute is not used, it is for backwards compatibility @@ -240,12 +248,13 @@ class RefResolver(object): self.cache_remote = cache_remote self.handlers = dict(handlers) + self.scopes_stack = [] self.store = _utils.URIDict( (id, validator.META_SCHEMA) for id, validator in iteritems(meta_schemas) ) self.store.update(store) - self.store[base_uri] = referrer + self.store[base_uri.url] = referrer @classmethod def from_schema(cls, schema, *args, **kwargs): @@ -259,14 +268,19 @@ class RefResolver(object): return cls(schema.get(u"id", u""), schema, *args, **kwargs) - @contextlib.contextmanager - def in_scope(self, scope): + def push_scope(self, scope, is_defragged=False): old_scope = self.resolution_scope - self.resolution_scope = urljoin(old_scope, scope) - try: - yield - finally: - self.resolution_scope = old_scope + self.scopes_stack.append(old_scope) + if not is_defragged: + scope = urldefrag(scope) + self.resolution_scope = DefragResult( + urljoin(old_scope.url, scope.url, allow_fragments=False) + if scope.url else old_scope.url, + scope.fragment + ) + + def pop_scope(self): + self.resolution_scope = self.scopes_stack.pop() @contextlib.contextmanager def resolving(self, ref): @@ -278,24 +292,26 @@ class RefResolver(object): """ - full_uri = urljoin(self.resolution_scope, ref) - uri, fragment = urldefrag(full_uri) - if not uri: - uri = self.base_uri + ref = urldefrag(ref) - if uri in self.store: - document = self.store[uri] - else: + url = urljoin(self.resolution_scope.url, ref.url, allow_fragments=False) \ + if ref.url else self.resolution_scope.url + + try: + document = self.store[url] + except KeyError: try: - document = self.resolve_remote(uri) + document = self.resolve_remote(url) except Exception as exc: raise RefResolutionError(exc) + uri = DefragResult(url, ref.fragment) old_base_uri, self.base_uri = self.base_uri, uri + self.push_scope(uri, is_defragged=True) try: - with self.in_scope(uri): - yield self.resolve_fragment(document, fragment) + yield self.resolve_fragment(document, ref.fragment) finally: + self.pop_scope() self.base_uri = old_base_uri def resolve_fragment(self, document, fragment): From 22701dc6526433201d2781b77566a6dba42768a8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Feb 2015 19:53:54 -0500 Subject: [PATCH 02/27] Fix test failures --- jsonschema/tests/test_validators.py | 4 ++-- jsonschema/validators.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index 2b14372..b3512ed 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -815,7 +815,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id" : "foo"} resolver = RefResolver.from_schema(schema) - self.assertEqual(resolver.base_uri, "foo") + self.assertEqual(resolver.base_uri.url, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: @@ -828,7 +828,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) - self.assertEqual(resolver.base_uri, "") + self.assertEqual(resolver.base_uri.url, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 8f48062..d0431f3 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -229,7 +229,7 @@ class RefResolver(object): :argument str base_uri: URI of the referring document :argument referrer: the actual referring document - :argument dict store: a mapping from URIs (without fragments!) to documents to cache + :argument dict store: a mapping from URIs to documents to cache :argument bool cache_remote: whether remote refs should be cached after first resolution :argument dict handlers: a mapping from URI schemes to functions that @@ -275,7 +275,7 @@ class RefResolver(object): scope = urldefrag(scope) self.resolution_scope = DefragResult( urljoin(old_scope.url, scope.url, allow_fragments=False) - if scope.url else old_scope.url, + if scope.url else old_scope.url, scope.fragment ) @@ -294,8 +294,13 @@ class RefResolver(object): ref = urldefrag(ref) - url = urljoin(self.resolution_scope.url, ref.url, allow_fragments=False) \ - if ref.url else self.resolution_scope.url + if ref.url: + url = urljoin( + self.resolution_scope.url, + ref.url, + allow_fragments=False) + else: + url = self.resolution_scope.url try: document = self.store[url] From 812392bbd4359718eab86e892c32211823b229e5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 1 Mar 2015 17:58:35 -0500 Subject: [PATCH 03/27] Add benchmark script. --- bench.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 bench.py diff --git a/bench.py b/bench.py new file mode 100644 index 0000000..e7318ed --- /dev/null +++ b/bench.py @@ -0,0 +1,74 @@ +#!/usr/env/bin python +""" +Benchmark the performance of jsonschema. + +Example benchmark: + + wget http://swagger.io/v2/schema.json + wget http://petstore.swagger.io/v2/swagger.json + python bench.py -r 5 schema.json swagger.json + +""" +from __future__ import print_function +import argparse +import cProfile +import json +import time + +import jsonschema + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('schema', help="path to a schema used to benchmark") + parser.add_argument('document', help="document to validate with schema") + parser.add_argument('-r', '--repeat', type=int, help="number of iterations") + parser.add_argument('--profile', + help="Enable profiling, write profile to this filepath") + return parser.parse_args() + + +def run(filename, schema, document): + resolver = jsonschema.RefResolver( + 'file://{0}'.format(filename), + schema, + store={schema['id']: schema}) + jsonschema.validate(document, schema, resolver=resolver) + + +def format_time(time_): + return "%.3fms" % (time_ * 1000) + + +def run_timeit(schema_filename, document_filename, repeat, profile): + with open(schema_filename) as schema_file: + schema = json.load(schema_file) + + with open(document_filename) as fh: + document = json.load(fh) + + if profile: + profiler = cProfile.Profile() + profiler.enable() + + times = [] + for _ in range(repeat): + start_time = time.time() + run(schema_filename, schema, document) + times.append(time.time() - start_time) + + if profile: + profiler.disable() + profiler.dump_stats(profile) + + print(", ".join(map(format_time, sorted(times)))) + print("Mean: {0}".format(format_time(sum(times) / repeat))) + + +def main(): + args = parse_args() + run_timeit(args.schema, args.document, args.repeat, args.profile) + + +if __name__ == "__main__": + main() From 613cf3e1a0ed9c09f9a13634dd05416594694e40 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Feb 2015 21:15:08 -0500 Subject: [PATCH 04/27] Perf improvements by using a cache. --- jsonschema/_utils.py | 16 +++++++ jsonschema/tests/test_validators.py | 12 ++--- jsonschema/validators.py | 73 +++++++++++++---------------- 3 files changed, 54 insertions(+), 47 deletions(-) diff --git a/jsonschema/_utils.py b/jsonschema/_utils.py index ae7e2b5..c13711c 100644 --- a/jsonschema/_utils.py +++ b/jsonschema/_utils.py @@ -38,6 +38,22 @@ class URIDict(MutableMapping): return repr(self.store) +class Cache(object): + """Cache the result of a function, using the arguments to the function as + the key. + """ + + def __init__(self, func): + self.func = func + self._cache = {} + + def __call__(self, *args): + if args in self._cache: + return self._cache[args] + self._cache[args] = value = self.func(*args) + return value + + class Unset(object): """ An as-of-yet unset attribute or unprovided default parameter. diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index b3512ed..0f4c825 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -775,11 +775,11 @@ class TestRefResolver(unittest.TestCase): self.assertEqual(resolved, self.referrer["properties"]["foo"]) def test_it_resolves_local_refs_with_id(self): - schema = {"id": "foo://bar/schema#", "a": {"foo": "bar"}} + schema = {"id": "http://bar/schema#", "a": {"foo": "bar"}} resolver = RefResolver.from_schema(schema) with resolver.resolving("#/a") as resolved: self.assertEqual(resolved, schema["a"]) - with resolver.resolving("foo://bar/schema#/a") as resolved: + with resolver.resolving("http://bar/schema#/a") as resolved: self.assertEqual(resolved, schema["a"]) def test_it_retrieves_stored_refs(self): @@ -815,7 +815,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id" : "foo"} resolver = RefResolver.from_schema(schema) - self.assertEqual(resolver.base_uri.url, "foo") + self.assertEqual(resolver.resolution_scope, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: @@ -828,7 +828,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) - self.assertEqual(resolver.base_uri.url, "") + self.assertEqual(resolver.resolution_scope, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) with resolver.resolving("#") as resolved: @@ -863,9 +863,7 @@ class TestRefResolver(unittest.TestCase): ) with resolver.resolving(ref): pass - with resolver.resolving(ref): - pass - self.assertEqual(foo_handler.call_count, 2) + self.assertEqual(foo_handler.call_count, 1) def test_if_you_give_it_junk_you_get_a_resolution_error(self): ref = "foo://bar" diff --git a/jsonschema/validators.py b/jsonschema/validators.py index d0431f3..a69accb 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -11,8 +11,7 @@ except ImportError: from jsonschema import _utils, _validators from jsonschema.compat import ( - Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, DefragResult, - + Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, str_types, int_types, iteritems, ) from jsonschema.exceptions import ErrorTree # Backwards compatibility # noqa @@ -109,7 +108,7 @@ def create(meta_schema, validators=(), version=None, default_types=None): # noq yield error finally: if scope: - self.resolver.pop_scope() + self.resolver.scopes_stack.pop() def descend(self, instance, schema, path=None, schema_path=None): for error in self.iter_errors(instance, schema): @@ -240,21 +239,21 @@ class RefResolver(object): def __init__( self, base_uri, referrer, store=(), cache_remote=True, handlers=(), ): - base_uri = urldefrag(base_uri) - self.base_uri = base_uri - self.resolution_scope = base_uri # This attribute is not used, it is for backwards compatibility self.referrer = referrer self.cache_remote = cache_remote self.handlers = dict(handlers) - self.scopes_stack = [] + self.scopes_stack = [base_uri] self.store = _utils.URIDict( (id, validator.META_SCHEMA) for id, validator in iteritems(meta_schemas) ) self.store.update(store) - self.store[base_uri.url] = referrer + self.store[base_uri] = referrer + + self.urljoin_cache = _utils.Cache(urljoin) + self.resolve_cache = _utils.Cache(self.resolve_from_url) @classmethod def from_schema(cls, schema, *args, **kwargs): @@ -268,19 +267,21 @@ class RefResolver(object): return cls(schema.get(u"id", u""), schema, *args, **kwargs) - def push_scope(self, scope, is_defragged=False): - old_scope = self.resolution_scope - self.scopes_stack.append(old_scope) - if not is_defragged: - scope = urldefrag(scope) - self.resolution_scope = DefragResult( - urljoin(old_scope.url, scope.url, allow_fragments=False) - if scope.url else old_scope.url, - scope.fragment - ) + def push_scope(self, scope): + self.scopes_stack.append( + self.urljoin_cache(self.resolution_scope, scope)) - def pop_scope(self): - self.resolution_scope = self.scopes_stack.pop() + @property + def resolution_scope(self): + return self.scopes_stack[-1] + + @contextlib.contextmanager + def in_scope(self, scope): + self.push_scope(scope) + try: + yield + finally: + self.scopes_stack.pop() @contextlib.contextmanager def resolving(self, ref): @@ -291,33 +292,25 @@ class RefResolver(object): :argument str ref: reference to resolve """ + url = self.urljoin_cache(self.resolution_scope, ref) - ref = urldefrag(ref) - - if ref.url: - url = urljoin( - self.resolution_scope.url, - ref.url, - allow_fragments=False) - else: - url = self.resolution_scope.url - + self.push_scope(url) try: - document = self.store[url] + yield self.resolve_cache(url) + finally: + self.scopes_stack.pop() + + def resolve_from_url(self, url): + ref = urldefrag(url) + try: + document = self.store[ref.url] except KeyError: try: - document = self.resolve_remote(url) + document = self.resolve_remote(ref.url) except Exception as exc: raise RefResolutionError(exc) - uri = DefragResult(url, ref.fragment) - old_base_uri, self.base_uri = self.base_uri, uri - self.push_scope(uri, is_defragged=True) - try: - yield self.resolve_fragment(document, ref.fragment) - finally: - self.pop_scope() - self.base_uri = old_base_uri + return self.resolve_fragment(document, ref.fragment) def resolve_fragment(self, document, fragment): """ From d1e24483aabea8844bf7498b52649498f6b3ff5a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 1 Mar 2015 20:52:20 -0500 Subject: [PATCH 05/27] Remove context manager from ref() validation. --- jsonschema/_validators.py | 7 ++++++- jsonschema/tests/test_validators.py | 8 ++------ jsonschema/validators.py | 27 +++++++++++++++++++-------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/jsonschema/_validators.py b/jsonschema/_validators.py index 7e5956d..a51681e 100644 --- a/jsonschema/_validators.py +++ b/jsonschema/_validators.py @@ -190,9 +190,14 @@ def enum(validator, enums, instance, schema): def ref(validator, ref, instance, schema): - with validator.resolver.resolving(ref) as resolved: + scope, resolved = validator.resolver.resolve(ref) + validator.resolver.push_scope(scope) + + try: for error in validator.descend(instance, resolved): yield error + finally: + validator.resolver.pop_scope() def type_draft3(validator, types, instance, schema): diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index 0f4c825..f3bb854 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -633,12 +633,8 @@ class ValidatorTestMixin(object): resolver = RefResolver("", {}) schema = {"$ref" : mock.Mock()} - @contextmanager - def resolving(): - yield {"type": "integer"} - - with mock.patch.object(resolver, "resolving") as resolve: - resolve.return_value = resolving() + with mock.patch.object(resolver, "resolve") as resolve: + resolve.return_value = "url", {"type": "integer"} with self.assertRaises(ValidationError): self.validator_class(schema, resolver=resolver).validate(None) diff --git a/jsonschema/validators.py b/jsonschema/validators.py index a69accb..435b8db 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -108,7 +108,7 @@ def create(meta_schema, validators=(), version=None, default_types=None): # noq yield error finally: if scope: - self.resolver.scopes_stack.pop() + self.resolver.pop_scope() def descend(self, instance, schema, path=None, schema_path=None): for error in self.iter_errors(instance, schema): @@ -271,20 +271,36 @@ class RefResolver(object): self.scopes_stack.append( self.urljoin_cache(self.resolution_scope, scope)) + def pop_scope(self): + self.scopes_stack.pop() + @property def resolution_scope(self): return self.scopes_stack[-1] + + # Deprecated, this function is no longer used, but is preserved for + # backwards compatibility @contextlib.contextmanager def in_scope(self, scope): self.push_scope(scope) try: yield finally: - self.scopes_stack.pop() + self.pop_scope() + # Deprecated, this function is no longer used, but is preserved for + # backwards compatibility @contextlib.contextmanager def resolving(self, ref): + url, resolved = self.resolve(ref) + self.push_scope(url) + try: + yield resolved + finally: + self.pop_scope() + + def resolve(self, ref): """ Context manager which resolves a JSON ``ref`` and enters the resolution scope of this ref. @@ -293,12 +309,7 @@ class RefResolver(object): """ url = self.urljoin_cache(self.resolution_scope, ref) - - self.push_scope(url) - try: - yield self.resolve_cache(url) - finally: - self.scopes_stack.pop() + return url, self.resolve_cache(url) def resolve_from_url(self, url): ref = urldefrag(url) From ca59f3fa87cda8b0f101d3997dcbc8711e5a19c2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Mar 2015 16:09:42 -0500 Subject: [PATCH 06/27] Remove DefragResult. --- jsonschema/compat.py | 6 +----- jsonschema/validators.py | 8 ++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/jsonschema/compat.py b/jsonschema/compat.py index 0afd9ea..b3156f9 100644 --- a/jsonschema/compat.py +++ b/jsonschema/compat.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -from collections import namedtuple import operator import sys @@ -43,9 +42,6 @@ def urlsplit(url): return SplitResult(scheme, netloc, path, query, fragment) -DefragResult = namedtuple('DefragResult', 'url fragment') - - def urldefrag(url): if "#" in url: s, n, p, q, frag = urlsplit(url) @@ -53,7 +49,7 @@ def urldefrag(url): else: defrag = url frag = '' - return DefragResult(defrag, frag) + return defrag, frag # flake8: noqa diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 435b8db..2343908 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -312,16 +312,16 @@ class RefResolver(object): return url, self.resolve_cache(url) def resolve_from_url(self, url): - ref = urldefrag(url) + url, fragment = urldefrag(url) try: - document = self.store[ref.url] + document = self.store[url] except KeyError: try: - document = self.resolve_remote(ref.url) + document = self.resolve_remote(url) except Exception as exc: raise RefResolutionError(exc) - return self.resolve_fragment(document, ref.fragment) + return self.resolve_fragment(document, fragment) def resolve_fragment(self, document, fragment): """ From ee1a256fc856a616940fb1eddcb3f4d8f7cb2f22 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Mar 2015 15:57:31 -0500 Subject: [PATCH 07/27] Use lru_cache --- jsonschema/__init__.py | 4 +--- jsonschema/_utils.py | 16 --------------- jsonschema/compat.py | 2 ++ jsonschema/tests/test_validators.py | 8 +++++++- jsonschema/validators.py | 32 +++++++++++++++++++---------- jsonschema/version.py | 1 + setup.py | 15 ++++++++++++-- 7 files changed, 45 insertions(+), 33 deletions(-) create mode 100644 jsonschema/version.py diff --git a/jsonschema/__init__.py b/jsonschema/__init__.py index 16c9843..e23168d 100644 --- a/jsonschema/__init__.py +++ b/jsonschema/__init__.py @@ -19,8 +19,6 @@ from jsonschema.validators import ( Draft3Validator, Draft4Validator, RefResolver, validate ) - -__version__ = "2.5.0-dev" - +from jsonschema.version import __version__ # flake8: noqa diff --git a/jsonschema/_utils.py b/jsonschema/_utils.py index c13711c..ae7e2b5 100644 --- a/jsonschema/_utils.py +++ b/jsonschema/_utils.py @@ -38,22 +38,6 @@ class URIDict(MutableMapping): return repr(self.store) -class Cache(object): - """Cache the result of a function, using the arguments to the function as - the key. - """ - - def __init__(self, func): - self.func = func - self._cache = {} - - def __call__(self, *args): - if args in self._cache: - return self._cache[args] - self._cache[args] = value = self.func(*args) - return value - - class Unset(object): """ An as-of-yet unset attribute or unprovided default parameter. diff --git a/jsonschema/compat.py b/jsonschema/compat.py index b3156f9..9f52ded 100644 --- a/jsonschema/compat.py +++ b/jsonschema/compat.py @@ -13,6 +13,7 @@ PY3 = sys.version_info[0] >= 3 if PY3: zip = zip + from functools import lru_cache from io import StringIO from urllib.parse import ( unquote, urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit @@ -23,6 +24,7 @@ if PY3: iteritems = operator.methodcaller("items") else: from itertools import izip as zip # noqa + from repoze.lru import lru_cache from StringIO import StringIO from urlparse import ( urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index f3bb854..1f03294 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -1,5 +1,4 @@ from collections import deque -from contextlib import contextmanager import json from jsonschema import FormatChecker, ValidationError @@ -870,6 +869,13 @@ class TestRefResolver(unittest.TestCase): pass self.assertEqual(str(err.exception), "Oh no! What's this?") + def test_helpful_error_message_on_failed_pop_scope(self): + resolver = RefResolver("", {}) + resolver.pop_scope() + with self.assertRaises(RefResolutionError) as exc: + resolver.pop_scope() + self.assertIn("Failed to pop the scope", str(exc.exception)) + def sorted_errors(errors): def key(error): diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 2343908..c84a3db 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -12,7 +12,7 @@ except ImportError: from jsonschema import _utils, _validators from jsonschema.compat import ( Sequence, urljoin, urlsplit, urldefrag, unquote, urlopen, - str_types, int_types, iteritems, + str_types, int_types, iteritems, lru_cache, ) from jsonschema.exceptions import ErrorTree # Backwards compatibility # noqa from jsonschema.exceptions import RefResolutionError, SchemaError, UnknownType @@ -233,18 +233,22 @@ class RefResolver(object): first resolution :argument dict handlers: a mapping from URI schemes to functions that should be used to retrieve them - + :arguments callable cache_func: a function decorator used to cache + expensive calls. Should support the `functools.lru_cache` interface. + :argument int cache_maxsize: number of items to store in the cache. Set + this to 0 to disable caching. Defaults to 1000. """ def __init__( self, base_uri, referrer, store=(), cache_remote=True, handlers=(), + cache_func=lru_cache, cache_maxsize=1000, ): # This attribute is not used, it is for backwards compatibility self.referrer = referrer self.cache_remote = cache_remote self.handlers = dict(handlers) - self.scopes_stack = [base_uri] + self._scopes_stack = [base_uri] self.store = _utils.URIDict( (id, validator.META_SCHEMA) for id, validator in iteritems(meta_schemas) @@ -252,8 +256,8 @@ class RefResolver(object): self.store.update(store) self.store[base_uri] = referrer - self.urljoin_cache = _utils.Cache(urljoin) - self.resolve_cache = _utils.Cache(self.resolve_from_url) + self._urljoin_cache = cache_func(cache_maxsize)(urljoin) + self._resolve_cache = cache_func(cache_maxsize)(self.resolve_from_url) @classmethod def from_schema(cls, schema, *args, **kwargs): @@ -268,15 +272,21 @@ class RefResolver(object): return cls(schema.get(u"id", u""), schema, *args, **kwargs) def push_scope(self, scope): - self.scopes_stack.append( - self.urljoin_cache(self.resolution_scope, scope)) + self._scopes_stack.append( + self._urljoin_cache(self.resolution_scope, scope)) def pop_scope(self): - self.scopes_stack.pop() + try: + self._scopes_stack.pop() + except IndexError: + raise RefResolutionError( + "Failed to pop the scope from an empty stack. " + "`pop_scope()` should only be called once for every " + "`push_scope()`") @property def resolution_scope(self): - return self.scopes_stack[-1] + return self._scopes_stack[-1] # Deprecated, this function is no longer used, but is preserved for @@ -308,8 +318,8 @@ class RefResolver(object): :argument str ref: reference to resolve """ - url = self.urljoin_cache(self.resolution_scope, ref) - return url, self.resolve_cache(url) + url = self._urljoin_cache(self.resolution_scope, ref) + return url, self._resolve_cache(url) def resolve_from_url(self, url): url, fragment = urldefrag(url) diff --git a/jsonschema/version.py b/jsonschema/version.py new file mode 100644 index 0000000..9509105 --- /dev/null +++ b/jsonschema/version.py @@ -0,0 +1 @@ +__version__ = "2.5.0-dev" diff --git a/setup.py b/setup.py index 6b47714..ffd61f1 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,12 @@ +import os.path from setuptools import setup +import sys -from jsonschema import __version__ - +# Load __version__ info globals without importing anything +with open( + os.path.join(os.path.dirname(__file__), 'jsonschema', 'version.py') +) as fh: + exec(fh.read()) with open("README.rst") as readme: long_description = readme.read() @@ -21,6 +26,11 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] +install_requires = [] + +if sys.version_info < (3, 2): + install_requires.append('repoze.lru >= 0.6') + setup( name="jsonschema", version=__version__, @@ -34,4 +44,5 @@ setup( long_description=long_description, url="http://github.com/Julian/jsonschema", entry_points={"console_scripts": ["jsonschema = jsonschema.cli:main"]}, + install_requires=install_requires, ) From b40f902441afb70dfb99556d054452e1097953ff Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sat, 14 Mar 2015 21:23:13 -0400 Subject: [PATCH 08/27] It's hard to be enthusiastic about tox anymore. --- README.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 55e5e8d..148dec2 100644 --- a/README.rst +++ b/README.rst @@ -63,12 +63,8 @@ now uses setuptools. Running the Test Suite ---------------------- -``jsonschema`` uses the wonderful `Tox `_ for its -test suite. (It really is wonderful, if for some reason you haven't heard of -it, you really should use it for your projects). - -Assuming you have ``tox`` installed (perhaps via ``pip install tox`` or your -package manager), just run ``tox`` in the directory of your source checkout to +If you have ``tox`` installed (perhaps via ``pip install tox`` or your +package manager), running``tox`` in the directory of your source checkout will run ``jsonschema``'s test suite on all of the versions of Python ``jsonschema`` supports. Note that you'll need to have all of those versions installed in order to run the tests on each of them, otherwise ``tox`` will skip (and fail) From 6093485f50afcc558b4333db00e9e18fbdc78818 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:39:47 -0400 Subject: [PATCH 09/27] Newer tox is slightly saner. --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index a231b9c..f330efc 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,6 @@ envlist = py{26,27,34,py,py3}, docs, style [testenv] -# by default tox runs with --pre which tickles this bug: -# https://bitbucket.org/pypy/pypy/issue/1894/keyerror-core-dumped-on-unicode-triple -install_command = pip install {opts} {packages} commands = py.test [] jsonschema {envpython} -m doctest README.rst From 5f91ee687601d97db46bc60ad359e7b30f950556 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:41:34 -0400 Subject: [PATCH 10/27] Run tests on the installed package. --- tox.ini | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index f330efc..d267bd3 100644 --- a/tox.ini +++ b/tox.ini @@ -3,10 +3,11 @@ envlist = py{26,27,34,py,py3}, docs, style [testenv] +changedir = {envtmpdir} commands = - py.test [] jsonschema - {envpython} -m doctest README.rst - py{26,27,34,py}: sphinx-build -b doctest docs {envtmpdir}/html + py.test [] {toxinidir}/jsonschema + {envpython} -m doctest {toxinidir}/README.rst + py{26,27,34,py}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = pytest strict-rfc3339 @@ -24,7 +25,7 @@ deps = [testenv:coverage] commands = - coverage run --branch --source jsonschema [] {envbindir}/py.test jsonschema + coverage run --branch --source {toxinidir}/jsonschema [] {envbindir}/py.test {toxinidir}/jsonschema coverage report --show-missing coverage html deps = @@ -43,7 +44,7 @@ commands = [testenv:style] deps = flake8 commands = - flake8 [] --max-complexity 10 jsonschema + flake8 [] --max-complexity 10 {toxinidir}/jsonschema [flake8] From e2a604f7ebc230e811ec6af15ac20551553b8a81 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:45:34 -0400 Subject: [PATCH 11/27] Switch to vcversioner, use repoze.lru only on 2.6, and add extras_require for format. --- .gitignore | 23 ++--------------------- MANIFEST.in | 1 + jsonschema/__init__.py | 2 +- jsonschema/compat.py | 7 ++++++- jsonschema/version.py | 1 - setup.py | 28 +++++++++++++++------------- tox.ini | 6 ++---- 7 files changed, 27 insertions(+), 41 deletions(-) delete mode 100644 jsonschema/version.py diff --git a/.gitignore b/.gitignore index 4ea7454..a4dc846 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,7 @@ -.DS_Store -.idea - -*.pyc -*.pyo - -*.egg-info -_build -build -dist -MANIFEST - -.coverage -.coveragerc -coverage -htmlcov - _cache _static _templates -_trial_temp - -.tox - TODO + +jsonschema/_version.py diff --git a/MANIFEST.in b/MANIFEST.in index a951c8a..a122503 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include *.rst include COPYING include tox.ini +include version.txt recursive-include json * diff --git a/jsonschema/__init__.py b/jsonschema/__init__.py index e23168d..baf1d89 100644 --- a/jsonschema/__init__.py +++ b/jsonschema/__init__.py @@ -19,6 +19,6 @@ from jsonschema.validators import ( Draft3Validator, Draft4Validator, RefResolver, validate ) -from jsonschema.version import __version__ +from jsonschema._version import __version__ # flake8: noqa diff --git a/jsonschema/compat.py b/jsonschema/compat.py index 9f52ded..8ea8d31 100644 --- a/jsonschema/compat.py +++ b/jsonschema/compat.py @@ -10,6 +10,7 @@ except ImportError: from collections.abc import MutableMapping, Sequence # noqa PY3 = sys.version_info[0] >= 3 +PY26 = sys.version_info[:2] == (2, 6) if PY3: zip = zip @@ -24,7 +25,6 @@ if PY3: iteritems = operator.methodcaller("items") else: from itertools import izip as zip # noqa - from repoze.lru import lru_cache from StringIO import StringIO from urlparse import ( urljoin, urlunsplit, SplitResult, urlsplit as _urlsplit # noqa @@ -35,6 +35,11 @@ else: int_types = int, long iteritems = operator.methodcaller("iteritems") + if PY26: + from repoze.lru import lru_cache + else: + from functools32 import lru_cache + # On python < 3.3 fragments are not handled properly with unknown schemes def urlsplit(url): diff --git a/jsonschema/version.py b/jsonschema/version.py deleted file mode 100644 index 9509105..0000000 --- a/jsonschema/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "2.5.0-dev" diff --git a/setup.py b/setup.py index ffd61f1..bbd2b2e 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,10 @@ -import os.path -from setuptools import setup +import os import sys -# Load __version__ info globals without importing anything -with open( - os.path.join(os.path.dirname(__file__), 'jsonschema', 'version.py') -) as fh: - exec(fh.read()) +from setuptools import setup -with open("README.rst") as readme: + +with open(os.path.join(os.path.dirname(__file__), "README.rst")) as readme: long_description = readme.read() classifiers = [ @@ -26,16 +22,22 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] -install_requires = [] -if sys.version_info < (3, 2): - install_requires.append('repoze.lru >= 0.6') +extras_require = {"format" : ["rfc3987", "strict-rfc3339", "webcolors"]} +if sys.version_info[:2] == (2, 6): + install_requires = ["argparse", "repoze.lru"] +elif sys.version_info[:2] == (2, 7): + install_requires = ["functools32"] +else: + install_requires = [] setup( name="jsonschema", - version=__version__, packages=["jsonschema", "jsonschema.tests"], package_data={"jsonschema": ["schemas/*.json"]}, + setup_requires=["vcversioner"], + install_requires=install_requires, + extras_require=extras_require, author="Julian Berman", author_email="Julian@GrayVines.com", classifiers=classifiers, @@ -44,5 +46,5 @@ setup( long_description=long_description, url="http://github.com/Julian/jsonschema", entry_points={"console_scripts": ["jsonschema = jsonschema.cli:main"]}, - install_requires=install_requires, + vcversioner={"version_module_paths" : ["jsonschema/_version.py"]}, ) diff --git a/tox.ini b/tox.ini index d267bd3..e3c69cf 100644 --- a/tox.ini +++ b/tox.ini @@ -10,11 +10,9 @@ commands = py{26,27,34,py}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = pytest - strict-rfc3339 - webcolors - py{27,34,py,py3}: rfc3987 - py26: argparse + -e{toxinidir}[format] + py26: unittest2 py{26,27,py,py3}: mock From 504f978da37e1fdaa6815bc92eede357055252d8 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:45:51 -0400 Subject: [PATCH 12/27] Fix a non-type in the docs. --- jsonschema/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsonschema/validators.py b/jsonschema/validators.py index c84a3db..2ef8d52 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -264,7 +264,7 @@ class RefResolver(object): """ Construct a resolver from a JSON schema object. - :argument schema schema: the referring schema + :argument schema: the referring schema :rtype: :class:`RefResolver` """ From 2b7b85e206deba6dd1ecbf9e5c08615a757fa8a2 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:46:08 -0400 Subject: [PATCH 13/27] lxml-cffi is giving obscure errors again. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e3c69cf..b08e0cd 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ changedir = {envtmpdir} commands = py.test [] {toxinidir}/jsonschema {envpython} -m doctest {toxinidir}/README.rst - py{26,27,34,py}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html + py{26,27,34}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = pytest From 9b051e3124d3c15924c4066ffbb6b3369e59fd29 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:52:33 -0400 Subject: [PATCH 14/27] Remove 3.3, use pip for installs, use green here too. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b2cb28..21b6bc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,10 @@ python: - "pypy3" - "2.6" - "2.7" - - "3.3" - "3.4" install: - - python setup.py -q install + - pip install --quiet . green script: - if [[ "$(python -c 'import sys; print(sys.version_info[:2])')" == "(2, 6)" ]]; then pip install unittest2; fi - - py.test --tb=native jsonschema + - green jsonschema - python -m doctest README.rst From e5664b639fe41b0774f328bb99009071bf2663dc Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 11:52:45 -0400 Subject: [PATCH 15/27] Begone py.test. --- tox.ini | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index b08e0cd..39a1638 100644 --- a/tox.ini +++ b/tox.ini @@ -5,11 +5,11 @@ envlist = py{26,27,34,py,py3}, docs, style [testenv] changedir = {envtmpdir} commands = - py.test [] {toxinidir}/jsonschema + green [] {toxinidir}/jsonschema {envpython} -m doctest {toxinidir}/README.rst py{26,27,34}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = - pytest + green -e{toxinidir}[format] @@ -23,7 +23,7 @@ deps = [testenv:coverage] commands = - coverage run --branch --source {toxinidir}/jsonschema [] {envbindir}/py.test {toxinidir}/jsonschema + coverage run --branch --source {toxinidir}/jsonschema [] {envbindir}/green {toxinidir}/jsonschema coverage report --show-missing coverage html deps = @@ -47,7 +47,3 @@ commands = [flake8] ignore = E203,E302,E303,E701,F811 - - -[pytest] -addopts = -r s -s From 159dd1f0ea7631cced423db0e11dcd39fce232a4 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 12:24:33 -0400 Subject: [PATCH 16/27] Really run on the installed package. --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 39a1638..7365f31 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,10 @@ envlist = py{26,27,34,py,py3}, docs, style [testenv] changedir = {envtmpdir} +setenv = + JSON_SCHEMA_TEST_SUITE = {toxinidir}/json commands = - green [] {toxinidir}/jsonschema + green [] jsonschema {envpython} -m doctest {toxinidir}/README.rst py{26,27,34}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = @@ -23,7 +25,7 @@ deps = [testenv:coverage] commands = - coverage run --branch --source {toxinidir}/jsonschema [] {envbindir}/green {toxinidir}/jsonschema + coverage run --branch --source {toxinidir}/jsonschema [] {envbindir}/green jsonschema coverage report --show-missing coverage html deps = From feea2329b11cce0ec39c7921b59b5fa84c279acd Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 12:27:15 -0400 Subject: [PATCH 17/27] Probably should combine these at some point, but for now move them. --- bench.py => benchmarks/bench.py | 0 perftest => benchmarks/perftest | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bench.py => benchmarks/bench.py (100%) rename perftest => benchmarks/perftest (100%) diff --git a/bench.py b/benchmarks/bench.py similarity index 100% rename from bench.py rename to benchmarks/bench.py diff --git a/perftest b/benchmarks/perftest similarity index 100% rename from perftest rename to benchmarks/perftest From b74a7fd9b86f0f75280be7c48e41850f56ca5144 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 12:37:08 -0400 Subject: [PATCH 18/27] Wait wat. Remove insanity. --- jsonschema/compat.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jsonschema/compat.py b/jsonschema/compat.py index 8ea8d31..0789f1e 100644 --- a/jsonschema/compat.py +++ b/jsonschema/compat.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import operator import sys From 2c59b7192e6ea9a53f498620d1f55d04ccbe2ae1 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 12:39:35 -0400 Subject: [PATCH 19/27] Add back assertions for backwards compat. --- jsonschema/tests/test_validators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index 1f03294..cc4bea8 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -810,6 +810,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id" : "foo"} resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "foo") self.assertEqual(resolver.resolution_scope, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) @@ -823,6 +824,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "") self.assertEqual(resolver.resolution_scope, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) From 2fda68994160309d316f6fa13b3cf3e944c550dd Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 13:45:38 -0400 Subject: [PATCH 20/27] Wrong docstring. --- jsonschema/validators.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 2ef8d52..64d70ce 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -288,7 +288,6 @@ class RefResolver(object): def resolution_scope(self): return self._scopes_stack[-1] - # Deprecated, this function is no longer used, but is preserved for # backwards compatibility @contextlib.contextmanager @@ -303,6 +302,14 @@ class RefResolver(object): # backwards compatibility @contextlib.contextmanager def resolving(self, ref): + """ + Context manager which resolves a JSON ``ref`` and enters the + resolution scope of this ref. + + :argument str ref: reference to resolve + + """ + url, resolved = self.resolve(ref) self.push_scope(url) try: @@ -311,13 +318,6 @@ class RefResolver(object): self.pop_scope() def resolve(self, ref): - """ - Context manager which resolves a JSON ``ref`` and enters the - resolution scope of this ref. - - :argument str ref: reference to resolve - - """ url = self._urljoin_cache(self.resolution_scope, ref) return url, self._resolve_cache(url) From 10ea4fa104d4b33bf60078acf06629c92ade2372 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 14:00:10 -0400 Subject: [PATCH 21/27] Er, green doesn't work on 2.6, and make running right out of a checkout easier. --- .gitignore | 2 -- jsonschema/_version.py | 5 +++++ tox.ini | 9 ++++++--- version.txt | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 jsonschema/_version.py create mode 100644 version.txt diff --git a/.gitignore b/.gitignore index a4dc846..31236db 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,3 @@ _static _templates TODO - -jsonschema/_version.py diff --git a/jsonschema/_version.py b/jsonschema/_version.py new file mode 100644 index 0000000..2525dd0 --- /dev/null +++ b/jsonschema/_version.py @@ -0,0 +1,5 @@ + +# This file is automatically generated by setup.py. +__version__ = '2.3.0.post128' +__sha__ = 'g2fda689' +__revision__ = 'g2fda689' diff --git a/tox.ini b/tox.ini index 7365f31..5fc1207 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,17 @@ changedir = {envtmpdir} setenv = JSON_SCHEMA_TEST_SUITE = {toxinidir}/json commands = - green [] jsonschema + py26: trial [] jsonschema + py{27,34,py,py3}: green [] jsonschema + {envpython} -m doctest {toxinidir}/README.rst py{26,27,34}: sphinx-build -b doctest {toxinidir}/docs {envtmpdir}/html deps = - green - -e{toxinidir}[format] + py26: twisted + py{27,34,py,py3}: green + py26: unittest2 py{26,27,py,py3}: mock diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..6baff11 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +v2.3.0-128-g2fda689 \ No newline at end of file From 7241db0d5e0be9a43acde2743f68a5182b3f97ed Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 15 Mar 2015 17:51:07 -0400 Subject: [PATCH 22/27] Fix base_uri backwards compatibility. --- jsonschema/tests/test_validators.py | 2 ++ jsonschema/validators.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index 1f03294..cc4bea8 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -810,6 +810,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema(self): schema = {"id" : "foo"} resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "foo") self.assertEqual(resolver.resolution_scope, "foo") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) @@ -823,6 +824,7 @@ class TestRefResolver(unittest.TestCase): def test_it_can_construct_a_base_uri_from_a_schema_without_id(self): schema = {} resolver = RefResolver.from_schema(schema) + self.assertEqual(resolver.base_uri, "") self.assertEqual(resolver.resolution_scope, "") with resolver.resolving("") as resolved: self.assertEqual(resolved, schema) diff --git a/jsonschema/validators.py b/jsonschema/validators.py index c84a3db..8f35d2b 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -288,6 +288,11 @@ class RefResolver(object): def resolution_scope(self): return self._scopes_stack[-1] + # backwards compatibility + @property + def base_uri(self): + uri, _ = urldefrag(self.resolution_scope) + return uri # Deprecated, this function is no longer used, but is preserved for # backwards compatibility From b7aab95f434bf5b386af1929f5bf5dbd7dbb4159 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 18:19:20 -0400 Subject: [PATCH 23/27] Not deprecating these for now, just not used internally. --- jsonschema/_version.py | 6 +++--- jsonschema/validators.py | 18 +++++++++++------- version.txt | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/jsonschema/_version.py b/jsonschema/_version.py index 2525dd0..ae262cf 100644 --- a/jsonschema/_version.py +++ b/jsonschema/_version.py @@ -1,5 +1,5 @@ # This file is automatically generated by setup.py. -__version__ = '2.3.0.post128' -__sha__ = 'g2fda689' -__revision__ = 'g2fda689' +__version__ = '2.3.0.post131' +__sha__ = 'gfec4647' +__revision__ = 'gfec4647' diff --git a/jsonschema/validators.py b/jsonschema/validators.py index f280be5..e6c5aa5 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -237,11 +237,18 @@ class RefResolver(object): expensive calls. Should support the `functools.lru_cache` interface. :argument int cache_maxsize: number of items to store in the cache. Set this to 0 to disable caching. Defaults to 1000. + """ def __init__( - self, base_uri, referrer, store=(), cache_remote=True, handlers=(), - cache_func=lru_cache, cache_maxsize=1000, + self, + base_uri, + referrer, + store=(), + cache_remote=True, + handlers=(), + cache_func=lru_cache, + cache_maxsize=1000, ): # This attribute is not used, it is for backwards compatibility self.referrer = referrer @@ -282,7 +289,8 @@ class RefResolver(object): raise RefResolutionError( "Failed to pop the scope from an empty stack. " "`pop_scope()` should only be called once for every " - "`push_scope()`") + "`push_scope()`", + ) @property def resolution_scope(self): @@ -293,8 +301,6 @@ class RefResolver(object): uri, _ = urldefrag(self.resolution_scope) return uri - # Deprecated, this function is no longer used, but is preserved for - # backwards compatibility @contextlib.contextmanager def in_scope(self, scope): self.push_scope(scope) @@ -303,8 +309,6 @@ class RefResolver(object): finally: self.pop_scope() - # Deprecated, this function is no longer used, but is preserved for - # backwards compatibility @contextlib.contextmanager def resolving(self, ref): """ diff --git a/version.txt b/version.txt index 6baff11..1d29f07 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v2.3.0-128-g2fda689 \ No newline at end of file +v2.3.0-131-gfec4647 \ No newline at end of file From 8ebd5bcb0a6d7a7679b7d0e90d35e6fcc51ae2ff Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 15 Mar 2015 18:20:00 -0400 Subject: [PATCH 24/27] I give up. --- .travis.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21b6bc1..51e2255 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,5 @@ language: python python: - - "pypy" - - "pypy3" - - "2.6" - "2.7" - - "3.4" -install: - - pip install --quiet . green script: - - if [[ "$(python -c 'import sys; print(sys.version_info[:2])')" == "(2, 6)" ]]; then pip install unittest2; fi - - green jsonschema - - python -m doctest README.rst + - tox From 33973dc9fa22e47ebd15b42025c4a1284b48ce9c Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sat, 21 Mar 2015 21:56:50 -0400 Subject: [PATCH 25/27] Pass in caches instead of arguments. --- jsonschema/_version.py | 6 +++--- jsonschema/validators.py | 27 ++++++++++++++++----------- version.txt | 2 +- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/jsonschema/_version.py b/jsonschema/_version.py index ae262cf..1ba07cc 100644 --- a/jsonschema/_version.py +++ b/jsonschema/_version.py @@ -1,5 +1,5 @@ # This file is automatically generated by setup.py. -__version__ = '2.3.0.post131' -__sha__ = 'gfec4647' -__revision__ = 'gfec4647' +__version__ = '2.3.0.post133' +__sha__ = 'g8ebd5bc' +__revision__ = 'g8ebd5bc' diff --git a/jsonschema/validators.py b/jsonschema/validators.py index e6c5aa5..0945949 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -233,10 +233,10 @@ class RefResolver(object): first resolution :argument dict handlers: a mapping from URI schemes to functions that should be used to retrieve them - :arguments callable cache_func: a function decorator used to cache - expensive calls. Should support the `functools.lru_cache` interface. - :argument int cache_maxsize: number of items to store in the cache. Set - this to 0 to disable caching. Defaults to 1000. + :arguments functools.lru_cache urljoin_cache: a cache that will be used for + caching the results of joining the resolution scope to subscopes. + :arguments functools.lru_cache remote_cache: a cache that will be used for + caching the results of resolved remote URLs. """ @@ -247,10 +247,14 @@ class RefResolver(object): store=(), cache_remote=True, handlers=(), - cache_func=lru_cache, - cache_maxsize=1000, + urljoin_cache=None, + remote_cache=None, ): - # This attribute is not used, it is for backwards compatibility + if urljoin_cache is None: + urljoin_cache = lru_cache(1024)(urljoin) + if remote_cache is None: + remote_cache = lru_cache(1024)(self.resolve_from_url) + self.referrer = referrer self.cache_remote = cache_remote self.handlers = dict(handlers) @@ -263,8 +267,8 @@ class RefResolver(object): self.store.update(store) self.store[base_uri] = referrer - self._urljoin_cache = cache_func(cache_maxsize)(urljoin) - self._resolve_cache = cache_func(cache_maxsize)(self.resolve_from_url) + self._urljoin_cache = urljoin_cache + self._remote_cache = remote_cache @classmethod def from_schema(cls, schema, *args, **kwargs): @@ -280,7 +284,8 @@ class RefResolver(object): def push_scope(self, scope): self._scopes_stack.append( - self._urljoin_cache(self.resolution_scope, scope)) + self._urljoin_cache(self.resolution_scope, scope), + ) def pop_scope(self): try: @@ -328,7 +333,7 @@ class RefResolver(object): def resolve(self, ref): url = self._urljoin_cache(self.resolution_scope, ref) - return url, self._resolve_cache(url) + return url, self._remote_cache(url) def resolve_from_url(self, url): url, fragment = urldefrag(url) diff --git a/version.txt b/version.txt index 1d29f07..bab441e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v2.3.0-131-gfec4647 \ No newline at end of file +v2.3.0-133-g8ebd5bc \ No newline at end of file From f672988e2cb68aec04877d288057c37bf9587e44 Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sat, 21 Mar 2015 22:08:54 -0400 Subject: [PATCH 26/27] Need to preserve backwards compat for RefResolvers without the new methods. --- jsonschema/_validators.py | 20 +++++++++++++------- jsonschema/tests/test_validators.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/jsonschema/_validators.py b/jsonschema/_validators.py index a51681e..f566f6d 100644 --- a/jsonschema/_validators.py +++ b/jsonschema/_validators.py @@ -190,14 +190,20 @@ def enum(validator, enums, instance, schema): def ref(validator, ref, instance, schema): - scope, resolved = validator.resolver.resolve(ref) - validator.resolver.push_scope(scope) + resolve = getattr(validator.resolver, "resolve", None) + if resolve is None: + with validator.resolver.resolving(ref) as resolved: + for error in validator.descend(instance, resolved): + yield error + else: + scope, resolved = validator.resolver.resolve(ref) + validator.resolver.push_scope(scope) - try: - for error in validator.descend(instance, resolved): - yield error - finally: - validator.resolver.pop_scope() + try: + for error in validator.descend(instance, resolved): + yield error + finally: + validator.resolver.pop_scope() def type_draft3(validator, types, instance, schema): diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index cc4bea8..e34efca 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -1,4 +1,5 @@ from collections import deque +from contextlib import contextmanager import json from jsonschema import FormatChecker, ValidationError @@ -639,6 +640,25 @@ class ValidatorTestMixin(object): resolve.assert_called_once_with(schema["$ref"]) + def test_it_delegates_to_a_legacy_ref_resolver(self): + """ + Legacy RefResolvers support only the context manager form of + resolution. + + """ + + class LegacyRefResolver(object): + @contextmanager + def resolving(this, ref): + self.assertEqual(ref, "the ref") + yield {"type" : "integer"} + + resolver = LegacyRefResolver() + schema = {"$ref" : "the ref"} + + with self.assertRaises(ValidationError): + self.validator_class(schema, resolver=resolver).validate(None) + def test_is_type_is_true_for_valid_type(self): self.assertTrue(self.validator.is_type("foo", "string")) From e886fd27a6a276c784f4e8af99412950208f3f7b Mon Sep 17 00:00:00 2001 From: Julian Berman Date: Sun, 22 Mar 2015 13:31:49 -0400 Subject: [PATCH 27/27] Squashed 'json/' changes from 9208016..0b657e8 0b657e8 Merge pull request #86 from kylef/patch-1 1bd4151 [README] JSONSchema.swift uses these tests too 8f86716 Revert "Add jon, JSON parser for the fishshell." db9c629 Merge pull request #82 from bucaran/patch-1 875fa42 Add jon, JSON parser for the fishshell. 64c556c Merge pull request #81 from s-panferov/patch-1 43105d4 Add new Rust library to the list aa4d927 Merge pull request #80 from seagreen/implementations 20ef067 Add new haskell implementation. 1274070 Merge pull request #79 from Muscula/json-schema-benchmark 6d8cf45 Merge pull request #78 from JamesNK/patch-1 55c4992 Add json-schema-benchmark to list of users of this test suite 645623d Added Newtonsoft.Json.Schema implementation a7944d1 Merge pull request #76 from Prestaul/patch-1 5729cdf Added skeemas to list of suite users 4600fe3 Make the implementation list a bit less unwieldy now that it's so long (hooray!) 11d6905 Merge remote-tracking branch 'mafintosh/patch-1' into develop 689b80f Merge pull request #74 from bugventure/develop c36f888 Add request-validator as a user of the test suite 41876b1 Update README.md aabcb34 Merge pull request #71 from seagreen/additionalproperties b3d160b Add tests for additionalProperties by itself. git-subtree-dir: json git-subtree-split: 0b657e8b0d21a4099d3ce378ba7208073a9295e6 --- README.md | 80 ++++++++++++++++++++------ tests/draft3/additionalProperties.json | 19 ++++++ tests/draft4/additionalProperties.json | 19 ++++++ 3 files changed, 101 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7c4cfbf..aa63a6c 100644 --- a/README.md +++ b/README.md @@ -60,23 +60,69 @@ Who Uses the Test Suite This suite is being used by: - * [jsck (a fast JSON validator in CoffeeScript)](https://github.com/pandastrike/jsck) - * [json-schema-validator (Java)](https://github.com/fge/json-schema-validator) - * [jsonschema (python)](https://github.com/Julian/jsonschema) - * [aeson-schema (haskell)](https://github.com/timjb/aeson-schema) - * [direct-schema (javascript)](https://github.com/IreneKnapp/direct-schema) - * [jsonschema (javascript)](https://github.com/tdegrunt/jsonschema) - * [JaySchema (javascript)](https://github.com/natesilva/jayschema) - * [z-schema (javascript)](https://github.com/zaggino/z-schema) - * [jassi (javascript)](https://github.com/iclanzan/jassi) - * [json-schema-valid (javascript)](https://github.com/ericgj/json-schema-valid) - * [jesse (Erlang)](https://github.com/klarna/jesse) - * [json-schema (PHP)](https://github.com/justinrainbow/json-schema) - * [gojsonschema (Go)](https://github.com/sigu-399/gojsonschema) - * [json_schema (Dart)](https://github.com/patefacio/json_schema) - * [tv4 (JavaScript)](https://github.com/geraintluff/tv4) - * [Jsonary (JavaScript)](https://github.com/jsonary-js/jsonary) - * [json-schema (Ruby)](https://github.com/hoxworth/json-schema) +### Coffeescript ### + +* [jsck](https://github.com/pandastrike/jsck) + +### Dart ### + +* [json_schema](https://github.com/patefacio/json_schema) + +### Erlang ### + +* [jesse](https://github.com/klarna/jesse) + +### Go ### + +* [gojsonschema](https://github.com/sigu-399/gojsonschema) + +### Haskell ### + +* [aeson-schema](https://github.com/timjb/aeson-schema) +* [hjsonschema](https://github.com/seagreen/hjsonschema) + +### Java ### + +* [json-schema-validator](https://github.com/fge/json-schema-validator) + +### Javascript ### + +* [json-schema-benchmark](https://github.com/Muscula/json-schema-benchmark) +* [direct-schema](https://github.com/IreneKnapp/direct-schema) +* [is-my-json-valid](https://github.com/mafintosh/is-my-json-valid) +* [jassi](https://github.com/iclanzan/jassi) +* [JaySchema](https://github.com/natesilva/jayschema) +* [json-schema-valid](https://github.com/ericgj/json-schema-valid) +* [Jsonary](https://github.com/jsonary-js/jsonary) +* [jsonschema](https://github.com/tdegrunt/jsonschema) +* [request-validator](https://github.com/bugventure/request-validator) +* [skeemas](https://github.com/Prestaul/skeemas) +* [tv4](https://github.com/geraintluff/tv4) +* [z-schema](https://github.com/zaggino/z-schema) + +### .NET ### + +* [Newtonsoft.Json.Schema](https://github.com/JamesNK/Newtonsoft.Json.Schema) + +### PHP ### + +* [json-schema](https://github.com/justinrainbow/json-schema) + +### Python ### + +* [jsonschema](https://github.com/Julian/jsonschema) + +### Ruby ### + +* [json-schema](https://github.com/hoxworth/json-schema) + +### Rust ### + +* [valico](https://github.com/rustless/valico) + +### Swift ### + +* [JSONSchema](https://github.com/kylef/JSONSchema.swift) If you use it as well, please fork and send a pull request adding yourself to the list :). diff --git a/tests/draft3/additionalProperties.json b/tests/draft3/additionalProperties.json index eb334c9..40831f9 100644 --- a/tests/draft3/additionalProperties.json +++ b/tests/draft3/additionalProperties.json @@ -55,6 +55,25 @@ } ] }, + { + "description": + "additionalProperties can exist by itself", + "schema": { + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, { "description": "additionalProperties are allowed by default", "schema": {"properties": {"foo": {}, "bar": {}}}, diff --git a/tests/draft4/additionalProperties.json b/tests/draft4/additionalProperties.json index eb334c9..40831f9 100644 --- a/tests/draft4/additionalProperties.json +++ b/tests/draft4/additionalProperties.json @@ -55,6 +55,25 @@ } ] }, + { + "description": + "additionalProperties can exist by itself", + "schema": { + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, { "description": "additionalProperties are allowed by default", "schema": {"properties": {"foo": {}, "bar": {}}},