Change filter behavior when applied to JSON objects.

Author: jg-rp

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Python JSONPath Change Log
 
+## Version 0.4.0
+
+**IETF JSONPath Draft compliance**
+
+- **Behavioral change.** When applied to a JSON object, filters now have an implicit preceding wildcard selector. This is now consistent with applying filters to arrays and adheres to the IETF JSONPath Internet Draft.
+
 ## Version 0.3.0
 
 **IETF JSONPath Draft compliance**

README.md

@@ -290,7 +290,7 @@ And this is a list of areas where we deviate from the [IETF JSONPath draft](http
 - The root token (default `$`) is optional.
 - Paths starting with a dot (`.`) are OK. `.thing` is the same as `$.thing`, as is `thing`, `$[thing]` and `$["thing"]`.
 - Nested filters are not supported.
-- We don't treat filter expressions without a comparison as existence test, but an "is truthy" test. See the "Existence of non-singular queries" example in the IETF JSONPath draft.
+- We don't treat filter expressions without a comparison as existence test, but an "is truthy" test.
 
 And this is a list of features that are uncommon or unique to Python JSONPath.
 

jsonpath/filter.py

@@ -1,6 +1,7 @@
 """Filter expression nodes."""
 from __future__ import annotations
 
+import json
 import re
 from abc import ABC
 from abc import abstractmethod
@@ -262,25 +263,15 @@ def __eq__(self, other: object) -> bool:
     def evaluate(self, context: FilterContext) -> bool:
         if isinstance(self.left, Undefined) and isinstance(self.right, Undefined):
             return True
-
         left = self.left.evaluate(context)
         right = self.right.evaluate(context)
-
-        if left is UNDEFINED and right is UNDEFINED:
-            return False
-
         return context.env.compare(left, self.operator, right)
 
     async def evaluate_async(self, context: FilterContext) -> bool:
         if isinstance(self.left, Undefined) and isinstance(self.right, Undefined):
             return True
-
         left = await self.left.evaluate_async(context)
         right = await self.right.evaluate_async(context)
-
-        if left is UNDEFINED and right is UNDEFINED:
-            return False
-
         return context.env.compare(left, self.operator, right)
 
 
@@ -328,7 +319,11 @@ def evaluate(self, context: FilterContext) -> object:
                 return UNDEFINED
             return context.current
 
-        matches = self.path.findall(context.current)
+        try:
+            matches = self.path.findall(context.current)
+        except json.JSONDecodeError:
+            return UNDEFINED
+
         if not matches:
             return UNDEFINED
         if len(matches) == 1:
@@ -341,7 +336,11 @@ async def evaluate_async(self, context: FilterContext) -> object:
                 return UNDEFINED
             return context.current
 
-        matches = await self.path.findall_async(context.current)
+        try:
+            matches = await self.path.findall_async(context.current)
+        except json.JSONDecodeError:
+            return UNDEFINED
+
         if not matches:
             return UNDEFINED
         if len(matches) == 1:

jsonpath/path.py

@@ -76,6 +76,7 @@ def findall(
                 an incompatible way.
         """
         if isinstance(data, str):
+            # TODO: catch JSONDecodeError?
             _data = json.loads(data)
         elif isinstance(data, TextIO):
             _data = json.loads(data.read())
@@ -140,6 +141,7 @@ async def findall_async(
     ) -> List[object]:
         """An async version of `findall()`."""
         if isinstance(data, str):
+            # TODO: catch JSONDecodeError
             _data = json.loads(data)
         elif isinstance(data, TextIO):
             _data = json.loads(data.read())

jsonpath/selectors.py

@@ -447,22 +447,34 @@ def __init__(
     def __str__(self) -> str:
         return f"[?({self.expression})]"
 
-    def resolve(self, matches: Iterable[JSONPathMatch]) -> Iterable[JSONPathMatch]:
+    def resolve(  # noqa: PLR0912
+        self, matches: Iterable[JSONPathMatch]
+    ) -> Iterable[JSONPathMatch]:
         for match in matches:
             if isinstance(match.obj, Mapping):
-                context = FilterContext(
-                    env=self.env,
-                    current=match.obj,
-                    root=match.root,
-                    extra_context=match.filter_context(),
-                )
-                try:
-                    if self.expression.evaluate(context):
-                        yield match
-                except JSONPathTypeError as err:
-                    if not err.token:
-                        err.token = self.token
-                    raise
+                for key, val in match.obj.items():
+                    context = FilterContext(
+                        env=self.env,
+                        current=val,
+                        root=match.root,
+                        extra_context=match.filter_context(),
+                    )
+                    try:
+                        if self.expression.evaluate(context):
+                            _match = JSONPathMatch(
+                                filter_context=match.filter_context(),
+                                obj=val,
+                                parent=match,
+                                parts=match.parts + (key,),
+                                path=match.path + f"['{key}']",
+                                root=match.root,
+                            )
+                            match.add_child(_match)
+                            yield _match
+                    except JSONPathTypeError as err:
+                        if not err.token:
+                            err.token = self.token
+                        raise
 
             elif isinstance(match.obj, Sequence) and not isinstance(match.obj, str):
                 for i, obj in enumerate(match.obj):
@@ -489,25 +501,37 @@ def resolve(self, matches: Iterable[JSONPathMatch]) -> Iterable[JSONPathMatch]:
                             err.token = self.token
                         raise
 
-    async def resolve_async(
+    async def resolve_async(  # noqa: PLR0912
         self, matches: AsyncIterable[JSONPathMatch]
     ) -> AsyncIterable[JSONPathMatch]:
         async for match in matches:
             if isinstance(match.obj, Mapping):
-                context = FilterContext(
-                    env=self.env,
-                    current=match.obj,
-                    root=match.root,
-                    extra_context=match.filter_context(),
-                )
+                for key, val in match.obj.items():
+                    context = FilterContext(
+                        env=self.env,
+                        current=val,
+                        root=match.root,
+                        extra_context=match.filter_context(),
+                    )
+
+                    try:
+                        result = await self.expression.evaluate_async(context)
+                    except JSONPathTypeError as err:
+                        if not err.token:
+                            err.token = self.token
+                        raise
 
-                try:
-                    if await self.expression.evaluate_async(context):
-                        yield match
-                except JSONPathTypeError as err:
-                    if not err.token:
-                        err.token = self.token
-                    raise
+                    if result:
+                        _match = JSONPathMatch(
+                            filter_context=match.filter_context(),
+                            obj=val,
+                            parent=match,
+                            parts=match.parts + (key,),
+                            path=match.path + f"['{key}']",
+                            root=match.root,
+                        )
+                        match.add_child(_match)
+                        yield _match
 
             elif isinstance(match.obj, Sequence) and not isinstance(match.obj, str):
                 for i, obj in enumerate(match.obj):
@@ -517,22 +541,24 @@ async def resolve_async(
                         root=match.root,
                         extra_context=match.filter_context(),
                     )
+
                     try:
-                        if await self.expression.evaluate_async(context):
-                            _match = JSONPathMatch(
-                                filter_context=match.filter_context(),
-                                obj=obj,
-                                parent=match,
-                                parts=match.parts + (i,),
-                                path=f"{match.path}[{i}]",
-                                root=match.root,
-                            )
-                            match.add_child(_match)
-                            yield _match
+                        result = await self.expression.evaluate_async(context)
                     except JSONPathTypeError as err:
                         if not err.token:
                             err.token = self.token
                         raise
+                    if result:
+                        _match = JSONPathMatch(
+                            filter_context=match.filter_context(),
+                            obj=obj,
+                            parent=match,
+                            parts=match.parts + (i,),
+                            path=f"{match.path}[{i}]",
+                            root=match.root,
+                        )
+                        match.add_child(_match)
+                        yield _match
 
 
 class FilterContext:

tests/compliance.py

@@ -45,19 +45,9 @@ def cases() -> List[Case]:
 
 
 def valid_cases() -> List[Case]:
-    def mangle_filter(case: Case) -> Case:
-        # XXX: Insert wildcard in front of root :(
-        if (
-            case.name.startswith("filter")
-            and case.selector.startswith("$[?")
-            and isinstance(case.document, list)
-        ):
-            case.selector = case.selector.replace("$[?", "$.*[?")
-        return case
-
     # TODO: skipping filter functions. Not supported.
     return [
-        mangle_filter(case)
+        case
         for case in cases()
         if not case.invalid_selector and "function" not in case.name
     ]

tests/consensus.py

@@ -47,11 +47,6 @@ class Query:
     "bracket_notation_with_number_on_object": "Bad consensus",
     "bracket_notation_with_number_on_string": "Invalid document",
     "dot_notation_with_number_-1": "Unexpected token",
-    "filter_expression_with_equals_on_object_with_key_matching_query": "Bad consensus",
-    (
-        "filter_expression_with_value_after_"
-        "dot_notation_with_wildcard_on_array_of_objects"
-    ): "Bad consensus",
 }
 
 

tests/test_env.py

@@ -28,7 +28,7 @@ def test_find_all_with_extra_filter_context(env: JSONPathEnvironment) -> None:
     """Test that we can pass extra filter context to findall."""
     rv = env.findall(
         "$[?(@.some == #.other)]",
-        {"some": 1, "thing": 2},
+        {"foo": {"some": 1, "thing": 2}},
         filter_context={"other": 1},
     )
     assert rv == [{"some": 1, "thing": 2}]
@@ -50,7 +50,7 @@ def test_find_iter_with_extra_filter_context(env: JSONPathEnvironment) -> None:
     """Test that we can pass extra filter context to finditer."""
     matches = env.finditer(
         "$[?(@.some == #.other)]",
-        {"some": 1, "thing": 2},
+        {"foo": {"some": 1, "thing": 2}},
         filter_context={"other": 1},
     )
     assert [match.obj for match in matches] == [{"some": 1, "thing": 2}]
@@ -80,7 +80,7 @@ def test_find_all_async_with_extra_filter_context(env: JSONPathEnvironment) -> N
     async def coro() -> List[object]:
         return await env.findall_async(
             "$[?(@.some == #.other)]",
-            {"some": 1, "thing": 2},
+            {"foo": {"some": 1, "thing": 2}},
             filter_context={"other": 1},
         )
 
@@ -113,7 +113,7 @@ def test_find_iter_async_with_extra_filter_context(env: JSONPathEnvironment) ->
     async def coro() -> List[object]:
         matches = await env.finditer_async(
             "$[?(@.some == #.other)]",
-            {"some": 1, "thing": 2},
+            {"foo": {"some": 1, "thing": 2}},
             filter_context={"other": 1},
         )
         return [match.obj async for match in matches]

tests/test_ietf.py

@@ -157,15 +157,15 @@ class Case:
         data=FILTER_SELECTOR_DATA,
         want=[{"b": "j"}, {"b": "k"}, {"b": {}}, {"b": "kilo"}],
     ),
-    # Case(
-    #     description="filter selector - Existence of non-singular queries",
-    #     path="$[?(@.*)]",
-    #     data=FILTER_SELECTOR_DATA,
-    #     want=[
-    #         [3, 5, 1, 2, 4, 6, {"b": "j"}, {"b": "k"}, {"b": {}}, {"b": "kilo"}],
-    #         {"p": 1, "q": 2, "r": 3, "s": 5, "t": {"u": 6}},
-    #     ],
-    # ),
+    Case(
+        description="filter selector - Existence of non-singular queries",
+        path="$[?(@.*)]",
+        data=FILTER_SELECTOR_DATA,
+        want=[
+            [3, 5, 1, 2, 4, 6, {"b": "j"}, {"b": "k"}, {"b": {}}, {"b": "kilo"}],
+            {"p": 1, "q": 2, "r": 3, "s": 5, "t": {"u": 6}},
+        ],
+    ),
     # Case(
     #     description="filter selector - Nested filters",
     #     path="$[?(@[?(@.b)])]	",
@@ -190,24 +190,24 @@ class Case:
     #     data=FILTER_SELECTOR_DATA,
     #     want=[{"b": "j"}, {"b": "k"}, {"b": "kilo"}],
     # ),
-    # Case(
-    #     description="filter selector - Object value logical AND",
-    #     path="$.o[?(@>1 && @<4)]",
-    #     data=FILTER_SELECTOR_DATA,
-    #     want=[2, 3],
-    # ),
-    # Case(
-    #     description="filter selector - Object value logical OR",
-    #     path="$.o[?(@.u || @.x)]",
-    #     data=FILTER_SELECTOR_DATA,
-    #     want=[{"u": 6}],
-    # ),
-    # Case(
-    #     description="filter selector - Comparison of queries with no values",
-    #     path="$.a[?(@.b == $.x)]",
-    #     data=FILTER_SELECTOR_DATA,
-    #     want=[3, 5, 1, 2, 4, 6],
-    # ),
+    Case(
+        description="filter selector - Object value logical AND",
+        path="$.o[?(@>1 && @<4)]",
+        data=FILTER_SELECTOR_DATA,
+        want=[2, 3],
+    ),
+    Case(
+        description="filter selector - Object value logical OR",
+        path="$.o[?(@.u || @.x)]",
+        data=FILTER_SELECTOR_DATA,
+        want=[{"u": 6}],
+    ),
+    Case(
+        description="filter selector - Comparison of queries with no values",
+        path="$.a[?(@.b == $.x)]",
+        data=FILTER_SELECTOR_DATA,
+        want=[3, 5, 1, 2, 4, 6],
+    ),
     Case(
         description=(
             "filter selector - Comparisons of primitive and of structured values"

tests/test_re.py

@@ -24,19 +24,19 @@ class Case:
     Case(
         description="match a regex",
         path="$.some[?(@.thing =~ /fo[a-z]/)]",
-        data={"some": {"thing": "foo"}},
+        data={"some": [{"thing": "foo"}]},
         want=[{"thing": "foo"}],
     ),
     Case(
         description="regex with no match",
         path="$.some[?(@.thing =~ /fo[a-z]/)]",
-        data={"some": {"thing": "foO"}},
+        data={"some": [{"thing": "foO"}]},
         want=[],
     ),
     Case(
         description="case insensitive match",
         path="$.some[?(@.thing =~ /fo[a-z]/i)]",
-        data={"some": {"thing": "foO"}},
+        data={"some": [{"thing": "foO"}]},
         want=[{"thing": "foO"}],
     ),
 ]