From d20b13b7b02b24b5b791070f06e302f9309fbd74 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Thu, 25 Jun 2026 09:54:34 +0200 Subject: [PATCH] Use identity comparison ('is') for no_default sentinel in itertoolz All comparisons of function arguments against the no_default sentinel in itertoolz used equality ('==') rather than identity ('is'). This means any value that happens to equal the sentinel string '__no__default__' (e.g. obtained via pickle round-trip) would be silently treated as "no argument provided" instead of being used as the user's intended default. For example: from pickle import dumps, loads sentinel_copy = loads(dumps('__no__default__')) get('missing', {}, default=sentinel_copy) # raised KeyError pluck(1, [[0]], sentinel_copy) # raised IndexError accumulate(add, [1,2,3], sentinel_copy) # silently ignored initial Replace every `== no_default` / `!= no_default` with `is no_default` / `is not no_default` throughout itertoolz.py, matching the pattern already used for the no_pad sentinel in the same file. Update tests to reflect that a value equal to but not identical to the sentinel is correctly treated as a real default/initial value. --- toolz/itertoolz.py | 22 +++++++++---------- toolz/tests/test_itertoolz.py | 41 ++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/toolz/itertoolz.py b/toolz/itertoolz.py index 354ecf25..96279826 100644 --- a/toolz/itertoolz.py +++ b/toolz/itertoolz.py @@ -55,7 +55,7 @@ def accumulate(binop, seq, initial=no_default): itertools.accumulate : In standard itertools for Python 3.2+ """ seq = iter(seq) - if initial == no_default: + if initial is no_default: try: result = next(seq) except StopIteration: @@ -448,7 +448,7 @@ def get(ind, seq, default=no_default): return seq[ind] except TypeError: # `ind` may be a list if isinstance(ind, list): - if default == no_default: + if default is no_default: if len(ind) > 1: return operator.itemgetter(*ind)(seq) elif ind: @@ -457,12 +457,12 @@ def get(ind, seq, default=no_default): return () else: return tuple(_get(i, seq, default) for i in ind) - elif default != no_default: + elif default is not no_default: return default else: raise except (KeyError, IndexError): # we know `ind` is not a list - if default == no_default: + if default is no_default: raise else: return default @@ -605,7 +605,7 @@ def reduceby(key, binop, seq, init=no_default): {True: set([2, 4]), False: set([1, 3])} """ - is_no_default = init == no_default + is_no_default = init is no_default if not is_no_default and not callable(init): _init = init init = lambda: _init @@ -787,7 +787,7 @@ def pluck(ind, seqs, default=no_default): get map """ - if default == no_default: + if default is no_default: get = getter(ind) return map(get, seqs) elif isinstance(ind, list): @@ -876,14 +876,14 @@ def join(leftkey, leftseq, rightkey, rightseq, d = groupby(leftkey, leftseq) - if left_default == no_default and right_default == no_default: + if left_default is no_default and right_default is no_default: # Inner Join for item in rightseq: key = rightkey(item) if key in d: for left_match in d[key]: yield (left_match, item) - elif left_default != no_default and right_default == no_default: + elif left_default is not no_default and right_default is no_default: # Right Join for item in rightseq: key = rightkey(item) @@ -892,11 +892,11 @@ def join(leftkey, leftseq, rightkey, rightseq, yield (left_match, item) else: yield (left_default, item) - elif right_default != no_default: + elif right_default is not no_default: seen_keys = set() seen = seen_keys.add - if left_default == no_default: + if left_default is no_default: # Left Join for item in rightseq: key = rightkey(item) @@ -945,7 +945,7 @@ def diff(*seqs, **kwargs): if N < 2: raise TypeError('Too few sequences given (min 2 required)') default = kwargs.get('default', no_default) - if default == no_default: + if default is no_default: iters = zip(*seqs) else: iters = zip_longest(*seqs, fillvalue=default) diff --git a/toolz/tests/test_itertoolz.py b/toolz/tests/test_itertoolz.py index c8640917..78daa261 100644 --- a/toolz/tests/test_itertoolz.py +++ b/toolz/tests/test_itertoolz.py @@ -17,7 +17,8 @@ from operator import add, mul -# is comparison will fail between this and no_default +# A string equal to (but not identical to) the no_default sentinel. +# Identity-based ('is') comparisons correctly treat this as a real value. no_default2 = loads(dumps('__no__default__')) @@ -217,7 +218,9 @@ def test_get(): assert raises(KeyError, lambda: get(10, {'a': 1})) assert raises(TypeError, lambda: get({}, [1, 2, 3])) assert raises(TypeError, lambda: get([1, 2, 3], 1, None)) - assert raises(KeyError, lambda: get('foo', {}, default=no_default2)) + # no_default2 is equal to but not identical to the sentinel, so it is used + # as a real default value (identity comparison means no false collision). + assert get('foo', {}, default=no_default2) == no_default2 def test_mapcat(): @@ -285,8 +288,11 @@ def test_reduceby(): def test_reduce_by_init(): assert reduceby(iseven, add, [1, 2, 3, 4]) == {True: 2 + 4, False: 1 + 3} - assert reduceby(iseven, add, [1, 2, 3, 4], no_default2) == {True: 2 + 4, - False: 1 + 3} + # no_default2 is a real value (not the sentinel), so it is used as the + # literal init; confirm it is passed through, not silently ignored. + assert reduceby(iseven, lambda acc, x: acc, [1, 2, 3, 4], no_default2) == { + True: no_default2, False: no_default2 + } def test_reduce_by_callable_default(): @@ -314,7 +320,11 @@ def binop(a, b): start = object() assert list(accumulate(binop, [], start)) == [start] assert list(accumulate(binop, [])) == [] - assert list(accumulate(add, [1, 2, 3], no_default2)) == [1, 3, 6] + # no_default2 is a real value (not the sentinel), so accumulate uses it as + # the actual initial value; confirm it appears first in the output. + assert list(accumulate(lambda a, b: b, [1, 2, 3], no_default2)) == [ + no_default2, 1, 2, 3 + ] def test_accumulate_works_on_consumable_iterables(): @@ -394,8 +404,10 @@ def test_pluck(): assert raises(IndexError, lambda: list(pluck(1, [[0]]))) assert raises(KeyError, lambda: list(pluck('name', [{'id': 1}]))) + # no_default2 is a real value (not the sentinel), so it is used as the + # fallback for missing items rather than triggering "no default" semantics. assert list(pluck(0, [[0, 1], [2, 3], [4, 5]], no_default2)) == [0, 2, 4] - assert raises(IndexError, lambda: list(pluck(1, [[0]], no_default2))) + assert list(pluck(1, [[0]], no_default2)) == [no_default2] def test_join(): @@ -414,10 +426,19 @@ def addpair(pair): assert result == expected - result = set(starmap(add, join(first, names, second, fruit, - left_default=no_default2, - right_default=no_default2))) - assert result == expected + # no_default2 is a real value (not the sentinel), so passing it as + # left_default/right_default activates outer-join semantics. + # Items with key 3 in names have no match in fruit → paired with no_default2. + result_outer = list(join(first, names, second, fruit, + left_default=no_default2, + right_default=no_default2)) + matched = {pair for pair in result_outer if pair[0] != no_default2 and pair[1] != no_default2} + assert matched == {((1, 'one'), ('apple', 1)), + ((1, 'one'), ('orange', 1)), + ((2, 'two'), ('banana', 2)), + ((2, 'two'), ('coconut', 2))} + # Unmatched left item (3, 'three') should appear paired with no_default2 + assert ((3, 'three'), no_default2) in result_outer def test_getter():