diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bf332085..3d60494de 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: args: [--fix, lf] - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.1.1" + rev: "v0.1.5" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -25,7 +25,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v3.0.3" + rev: "v3.1.0" hooks: - id: prettier exclude: "^jsonschema/benchmarks/issue232/issue.json$" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4965f5a75..e2627ebd9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,10 @@ +v4.20.0 +======= + +* Properly consider items (and properties) to be evaluated by ``unevaluatedItems`` (resp. ``unevaluatedProperties``) when behind a ``$dynamicRef`` as specified by the 2020 and 2019 specifications. +* ``jsonschema.exceptions.ErrorTree.__setitem__`` is now deprecated. + More broadly, in general users of ``jsonschema`` should never be mutating objects owned by the library. + v4.19.2 ======= diff --git a/json/tests/draft-next/unevaluatedItems.json b/json/tests/draft-next/unevaluatedItems.json index 7379afb41..8dda001f2 100644 --- a/json/tests/draft-next/unevaluatedItems.json +++ b/json/tests/draft-next/unevaluatedItems.json @@ -461,6 +461,48 @@ } ] }, + { + "description": "unevaluatedItems with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://example.com/derived", + + "$ref": "/baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "prefixItems": [ + true, + { "type": "string" } + ] + }, + "baseSchema": { + "$id": "/baseSchema", + + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "type": "array", + "prefixItems": [ + { "type": "string" } + ], + "$dynamicRef": "#addons" + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, { "description": "unevaluatedItems can't see inside cousins", "schema": { diff --git a/json/tests/draft-next/unevaluatedProperties.json b/json/tests/draft-next/unevaluatedProperties.json index 69fe8a00c..4fe7986d6 100644 --- a/json/tests/draft-next/unevaluatedProperties.json +++ b/json/tests/draft-next/unevaluatedProperties.json @@ -715,6 +715,54 @@ } ] }, + { + "description": "unevaluatedProperties with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/next/schema", + "$id": "https://example.com/derived", + + "$ref": "/baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "properties": { + "bar": { "type": "string" } + } + }, + "baseSchema": { + "$id": "/baseSchema", + + "$comment": "unevaluatedProperties comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "$dynamicRef": "#addons" + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, { "description": "unevaluatedProperties can't see inside cousins", "schema": { diff --git a/json/tests/draft2019-09/unevaluatedItems.json b/json/tests/draft2019-09/unevaluatedItems.json index 53565a0b9..9c115ab3c 100644 --- a/json/tests/draft2019-09/unevaluatedItems.json +++ b/json/tests/draft2019-09/unevaluatedItems.json @@ -480,6 +480,51 @@ } ] }, + { + "description": "unevaluatedItems with $recursiveRef", + "schema": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://example.com/extended-tree", + + "$recursiveAnchor": true, + + "$ref": "/tree", + "items": [ + true, + true, + { "type": "string" } + ], + + "$defs": { + "tree": { + "$id": "/tree", + "$recursiveAnchor": true, + + "type": "array", + "items": [ + { "type": "number" }, + { + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "$recursiveRef": "#" + } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [1, [2, [], "b"], "a"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": [1, [2, [], "b", "too many"], "a"], + "valid": false + } + ] + }, { "description": "unevaluatedItems can't see inside cousins", "schema": { diff --git a/json/tests/draft2019-09/unevaluatedProperties.json b/json/tests/draft2019-09/unevaluatedProperties.json index a6cce8bb6..4e0d3ec83 100644 --- a/json/tests/draft2019-09/unevaluatedProperties.json +++ b/json/tests/draft2019-09/unevaluatedProperties.json @@ -715,6 +715,64 @@ } ] }, + { + "description": "unevaluatedProperties with $recursiveRef", + "schema": { + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://example.com/extended-tree", + + "$recursiveAnchor": true, + + "$ref": "/tree", + "properties": { + "name": { "type": "string" } + }, + + "$defs": { + "tree": { + "$id": "/tree", + "$recursiveAnchor": true, + + "type": "object", + "properties": { + "node": true, + "branches": { + "$comment": "unevaluatedProperties comes first so it's more likely to bugs errors with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "$recursiveRef": "#" + } + }, + "required": ["node"] + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "name": "a", + "node": 1, + "branches": { + "name": "b", + "node": 2 + } + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "name": "a", + "node": 1, + "branches": { + "foo": "b", + "node": 2 + } + }, + "valid": false + } + ] + }, { "description": "unevaluatedProperties can't see inside cousins", "schema": { diff --git a/json/tests/draft2020-12/unevaluatedItems.json b/json/tests/draft2020-12/unevaluatedItems.json index 2615c4c41..ddc35da28 100644 --- a/json/tests/draft2020-12/unevaluatedItems.json +++ b/json/tests/draft2020-12/unevaluatedItems.json @@ -461,6 +461,55 @@ } ] }, + { + "description": "unevaluatedItems with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/derived", + + "$ref": "/baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "prefixItems": [ + true, + { "type": "string" } + ] + }, + "baseSchema": { + "$id": "/baseSchema", + + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "type": "array", + "prefixItems": [ + { "type": "string" } + ], + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, { "description": "unevaluatedItems can't see inside cousins", "schema": { diff --git a/json/tests/draft2020-12/unevaluatedProperties.json b/json/tests/draft2020-12/unevaluatedProperties.json index f7fb420ff..023e84a5d 100644 --- a/json/tests/draft2020-12/unevaluatedProperties.json +++ b/json/tests/draft2020-12/unevaluatedProperties.json @@ -715,6 +715,61 @@ } ] }, + { + "description": "unevaluatedProperties with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/derived", + + "$ref": "/baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "properties": { + "bar": { "type": "string" } + } + }, + "baseSchema": { + "$id": "/baseSchema", + + "$comment": "unevaluatedProperties comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, { "description": "unevaluatedProperties can't see inside cousins", "schema": { diff --git a/jsonschema/_legacy_keywords.py b/jsonschema/_legacy_keywords.py index e76a84f9c..7265511de 100644 --- a/jsonschema/_legacy_keywords.py +++ b/jsonschema/_legacy_keywords.py @@ -1,3 +1,5 @@ +import re + from referencing.jsonschema import lookup_recursive_ref from jsonschema import _utils @@ -249,8 +251,22 @@ def find_evaluated_item_indexes_by_schema(validator, instance, schema): return [] evaluated_indexes = [] - if "$ref" in schema: - resolved = validator._resolver.lookup(schema["$ref"]) + ref = schema.get("$ref") + if ref is not None: + resolved = validator._resolver.lookup(ref) + evaluated_indexes.extend( + find_evaluated_item_indexes_by_schema( + validator.evolve( + schema=resolved.contents, + _resolver=resolved.resolver, + ), + instance, + resolved.contents, + ), + ) + + if "$recursiveRef" in schema: + resolved = lookup_recursive_ref(validator._resolver) evaluated_indexes.extend( find_evaluated_item_indexes_by_schema( validator.evolve( @@ -316,3 +332,121 @@ def unevaluatedItems_draft2019(validator, unevaluatedItems, instance, schema): if unevaluated_items: error = "Unevaluated items are not allowed (%s %s unexpected)" yield ValidationError(error % _utils.extras_msg(unevaluated_items)) + + +def find_evaluated_property_keys_by_schema(validator, instance, schema): + if validator.is_type(schema, "boolean"): + return [] + evaluated_keys = [] + + ref = schema.get("$ref") + if ref is not None: + resolved = validator._resolver.lookup(ref) + evaluated_keys.extend( + find_evaluated_property_keys_by_schema( + validator.evolve( + schema=resolved.contents, + _resolver=resolved.resolver, + ), + instance, + resolved.contents, + ), + ) + + if "$recursiveRef" in schema: + resolved = lookup_recursive_ref(validator._resolver) + evaluated_keys.extend( + find_evaluated_property_keys_by_schema( + validator.evolve( + schema=resolved.contents, + _resolver=resolved.resolver, + ), + instance, + resolved.contents, + ), + ) + + for keyword in [ + "properties", "additionalProperties", "unevaluatedProperties", + ]: + if keyword in schema: + schema_value = schema[keyword] + if validator.is_type(schema_value, "boolean") and schema_value: + evaluated_keys += instance.keys() + + elif validator.is_type(schema_value, "object"): + for property in schema_value: + if property in instance: + evaluated_keys.append(property) + + if "patternProperties" in schema: + for property in instance: + for pattern in schema["patternProperties"]: + if re.search(pattern, property): + evaluated_keys.append(property) + + if "dependentSchemas" in schema: + for property, subschema in schema["dependentSchemas"].items(): + if property not in instance: + continue + evaluated_keys += find_evaluated_property_keys_by_schema( + validator, instance, subschema, + ) + + for keyword in ["allOf", "oneOf", "anyOf"]: + if keyword in schema: + for subschema in schema[keyword]: + errs = next(validator.descend(instance, subschema), None) + if errs is None: + evaluated_keys += find_evaluated_property_keys_by_schema( + validator, instance, subschema, + ) + + if "if" in schema: + if validator.evolve(schema=schema["if"]).is_valid(instance): + evaluated_keys += find_evaluated_property_keys_by_schema( + validator, instance, schema["if"], + ) + if "then" in schema: + evaluated_keys += find_evaluated_property_keys_by_schema( + validator, instance, schema["then"], + ) + else: + if "else" in schema: + evaluated_keys += find_evaluated_property_keys_by_schema( + validator, instance, schema["else"], + ) + + return evaluated_keys + + +def unevaluatedProperties_draft2019(validator, uP, instance, schema): + if not validator.is_type(instance, "object"): + return + evaluated_keys = find_evaluated_property_keys_by_schema( + validator, instance, schema, + ) + unevaluated_keys = [] + for property in instance: + if property not in evaluated_keys: + for _ in validator.descend( + instance[property], + uP, + path=property, + schema_path=property, + ): + # FIXME: Include context for each unevaluated property + # indicating why it's invalid under the subschema. + unevaluated_keys.append(property) + + if unevaluated_keys: + if uP is False: + error = "Unevaluated properties are not allowed (%s %s unexpected)" + extras = sorted(unevaluated_keys, key=str) + yield ValidationError(error % _utils.extras_msg(extras)) + else: + error = ( + "Unevaluated properties are not valid under " + "the given schema (%s %s unevaluated and invalid)" + ) + yield ValidationError(error % _utils.extras_msg(unevaluated_keys)) diff --git a/jsonschema/_utils.py b/jsonschema/_utils.py index b08d590ae..9d274fd93 100644 --- a/jsonschema/_utils.py +++ b/jsonschema/_utils.py @@ -28,10 +28,10 @@ def __delitem__(self, uri): def __iter__(self): return iter(self.store) - def __len__(self): + def __len__(self): # pragma: no cover -- untested, but to be removed return len(self.store) - def __repr__(self): + def __repr__(self): # pragma: no cover -- untested, but to be removed return repr(self.store) @@ -197,8 +197,23 @@ def find_evaluated_item_indexes_by_schema(validator, instance, schema): if "items" in schema: return list(range(0, len(instance))) - if "$ref" in schema: - resolved = validator._resolver.lookup(schema["$ref"]) + ref = schema.get("$ref") + if ref is not None: + resolved = validator._resolver.lookup(ref) + evaluated_indexes.extend( + find_evaluated_item_indexes_by_schema( + validator.evolve( + schema=resolved.contents, + _resolver=resolved.resolver, + ), + instance, + resolved.contents, + ), + ) + + dynamicRef = schema.get("$dynamicRef") + if dynamicRef is not None: + resolved = validator._resolver.lookup(dynamicRef) evaluated_indexes.extend( find_evaluated_item_indexes_by_schema( validator.evolve( @@ -258,8 +273,23 @@ def find_evaluated_property_keys_by_schema(validator, instance, schema): return [] evaluated_keys = [] - if "$ref" in schema: - resolved = validator._resolver.lookup(schema["$ref"]) + ref = schema.get("$ref") + if ref is not None: + resolved = validator._resolver.lookup(ref) + evaluated_keys.extend( + find_evaluated_property_keys_by_schema( + validator.evolve( + schema=resolved.contents, + _resolver=resolved.resolver, + ), + instance, + resolved.contents, + ), + ) + + dynamicRef = schema.get("$dynamicRef") + if dynamicRef is not None: + resolved = validator._resolver.lookup(dynamicRef) evaluated_keys.extend( find_evaluated_property_keys_by_schema( validator.evolve( diff --git a/jsonschema/exceptions.py b/jsonschema/exceptions.py index 80281057e..a1b3ad867 100644 --- a/jsonschema/exceptions.py +++ b/jsonschema/exceptions.py @@ -4,6 +4,7 @@ from __future__ import annotations from collections import defaultdict, deque +from collections.abc import Iterable, Mapping, MutableMapping from pprint import pformat from textwrap import dedent, indent from typing import ClassVar @@ -209,14 +210,14 @@ class _RefResolutionError(Exception): def __eq__(self, other): if self.__class__ is not other.__class__: - return NotImplemented + return NotImplemented # pragma: no cover -- uncovered but deprecated # noqa: E501 return self._cause == other._cause def __str__(self): return str(self._cause) -class _WrappedReferencingError(_RefResolutionError, _Unresolvable): +class _WrappedReferencingError(_RefResolutionError, _Unresolvable): # pragma: no cover -- partially uncovered but to be removed # noqa: E501 def __init__(self, cause: _Unresolvable): object.__setattr__(self, "_wrapped", cause) @@ -297,9 +298,9 @@ class ErrorTree: _instance = _unset - def __init__(self, errors=()): - self.errors = {} - self._contents = defaultdict(self.__class__) + def __init__(self, errors: Iterable[ValidationError] = ()): + self.errors: MutableMapping[str, ValidationError] = {} + self._contents: Mapping[str, ErrorTree] = defaultdict(self.__class__) for error in errors: container = self @@ -309,7 +310,7 @@ def __init__(self, errors=()): container._instance = error.instance - def __contains__(self, index): + def __contains__(self, index: str | int): """ Check whether ``instance[index]`` has any errors. """ @@ -328,11 +329,22 @@ def __getitem__(self, index): self._instance[index] return self._contents[index] - def __setitem__(self, index, value): + def __setitem__(self, index: str | int, value: ErrorTree): """ Add an error to the tree at the given ``index``. + + .. deprecated:: v4.20.0 + + Setting items on an `ErrorTree` is deprecated without replacement. + To populate a tree, provide all of its sub-errors when you + construct the tree. """ - self._contents[index] = value + warnings.warn( + "ErrorTree.__setitem__ is deprecated without replacement.", + DeprecationWarning, + stacklevel=2, + ) + self._contents[index] = value # type: ignore[index] def __iter__(self): """ diff --git a/jsonschema/tests/test_deprecations.py b/jsonschema/tests/test_deprecations.py index fdb3b7b0f..6e69785a6 100644 --- a/jsonschema/tests/test_deprecations.py +++ b/jsonschema/tests/test_deprecations.py @@ -51,6 +51,22 @@ def test_import_ErrorTree(self): self.assertEqual(ErrorTree, exceptions.ErrorTree) self.assertEqual(w.filename, __file__) + def test_ErrorTree_setitem(self): + """ + As of v4.20.0, setting items on an ErrorTree is deprecated. + """ + + e = exceptions.ValidationError("some error", path=["foo"]) + tree = exceptions.ErrorTree() + subtree = exceptions.ErrorTree(errors=[e]) + + message = "ErrorTree.__setitem__ is " + with self.assertWarnsRegex(DeprecationWarning, message) as w: + tree["foo"] = subtree + + self.assertEqual(tree["foo"], subtree) + self.assertEqual(w.filename, __file__) + def test_import_FormatError(self): """ As of v4.18.0, importing FormatError from the package root is diff --git a/jsonschema/tests/test_exceptions.py b/jsonschema/tests/test_exceptions.py index 00ff30091..cb9a598b0 100644 --- a/jsonschema/tests/test_exceptions.py +++ b/jsonschema/tests/test_exceptions.py @@ -396,6 +396,22 @@ def test_if_its_in_the_tree_anyhow_it_does_not_raise_an_error(self): tree = exceptions.ErrorTree([error]) self.assertIsInstance(tree["foo"], exceptions.ErrorTree) + def test_iter(self): + e1, e2 = ( + exceptions.ValidationError( + "1", + validator="foo", + path=["bar", "bar2"], + instance="i1"), + exceptions.ValidationError( + "2", + validator="quux", + path=["foobar", 2], + instance="i2"), + ) + tree = exceptions.ErrorTree([e1, e2]) + self.assertEqual(set(tree), {"bar", "foobar"}) + def test_repr_single(self): error = exceptions.ValidationError( "1", diff --git a/jsonschema/tests/test_format.py b/jsonschema/tests/test_format.py index 371eb90da..d829f9848 100644 --- a/jsonschema/tests/test_format.py +++ b/jsonschema/tests/test_format.py @@ -54,6 +54,7 @@ def test_it_catches_registered_errors(self): self.assertIs(cm.exception.cause, BOOM) self.assertIs(cm.exception.__cause__, BOOM) + self.assertEqual(str(cm.exception), "12 is not a 'boom'") # Unregistered errors should not be caught with self.assertRaises(type(BANG)): diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 740658bab..b8f6fcb26 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -782,7 +782,9 @@ def extend( "required": _keywords.required, "type": _keywords.type, "unevaluatedItems": _legacy_keywords.unevaluatedItems_draft2019, - "unevaluatedProperties": _keywords.unevaluatedProperties, + "unevaluatedProperties": ( + _legacy_keywords.unevaluatedProperties_draft2019 + ), "uniqueItems": _keywords.uniqueItems, }, type_checker=_types.draft201909_type_checker, diff --git a/pyproject.toml b/pyproject.toml index 26d299a36..efb5b9fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,6 @@ exclude = ["jsonschema/benchmarks/*"] [tool.ruff] line-length = 79 -target-version = "py38" select = ["B", "D", "D204", "E", "F", "Q", "RUF", "SIM", "UP", "W"] ignore = [ # Wat, type annotations for self and cls, why is this a thing?