diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8da84256d8b6..1b058550477e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -261,6 +261,7 @@ class Envs: SGLANG_REQ_WAITING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds + SGLANG_SWA_EVICTION_INTERVAL_MULTIPLIER = EnvFloat(1.0) SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120) # Test: pd-disaggregation @@ -519,6 +520,7 @@ class Envs: SGLANG_OPT_ALLOW_SHARED_EXPERT_DUAL_STREAM = EnvBool(False) # verified in journal 2026-04-21-017 SGLANG_OPT_CACHE_SWA_TRANSLATION = EnvBool(False) SGLANG_OPT_SWA_RADIX_CACHE_COMPACT = EnvBool(False) + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False) SGLANG_OPT_MXFP4_FUSE_RSF_SHARED_ADD = EnvBool(False) SGLANG_OPT_MXFP4_STATIC_SCALE_ONES = EnvBool(False) SGLANG_OPT_MXFP4_SKIP_DISPATCHER_MAPPING = EnvBool(False) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 6f5a7ea2597d..ec2509985969 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -643,6 +643,7 @@ def __init__( self.storage_hit_length = 0 # The node to lock until for swa radix tree lock ref self.swa_uuid_for_lock: Optional[int] = None + self.swa_prefix_lock_released: bool = False # The prefix length that is inserted into the tree cache self.cache_protected_len: int = 0 @@ -1091,6 +1092,7 @@ def reset_for_retract(self): self.indexer_topk = None self.last_node = None self.swa_uuid_for_lock = None + self.swa_prefix_lock_released = False self.extend_input_len = 0 self.is_retracted = True self.retracted_stain = True @@ -2269,24 +2271,39 @@ def maybe_evict_swa(self): sliding_window_size = self.tree_cache.sliding_window_size server_args = get_global_server_args() - if ( - self.forward_mode.is_decode() - and server_args.enable_piecewise_cuda_graph - and not self.tree_cache.is_chunk_cache() - ): - return + release_leaf_lock = ( + envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.get() + and hasattr(self.tree_cache, "dec_swa_lock_only") + ) + page_size = self.tree_cache.page_size + eviction_interval = max( + page_size, + int( + sliding_window_size + * envs.SGLANG_SWA_EVICTION_INTERVAL_MULTIPLIER.get() + ), + ) + eviction_interval = (eviction_interval // page_size) * page_size for idx, req in enumerate(self.reqs): if self.forward_mode.is_decode(): - # We set evict_swa condition here with two reasons: - # 1. In overlap scheduler, we cannot evict swa when req.decode_batch_idx == 0 since the prev extend batch is still running. - # 2. Evict swa every window_size tokens to reduce the overhead. - if req.decode_batch_idx % sliding_window_size == 1: + if req.decode_batch_idx % eviction_interval == 1: self._evict_swa(req, req.seqlen - 1) + + if ( + release_leaf_lock + and not req.swa_prefix_lock_released + and req.swa_uuid_for_lock is not None + and req.last_node is not None + and req.decode_batch_idx >= sliding_window_size + ): + self.tree_cache.dec_swa_lock_only( + req.last_node, req.swa_uuid_for_lock + ) + req.swa_prefix_lock_released = True elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache(): pre_len = self.prefix_lens[idx] if self.enable_overlap: - # In chunked prefill case, when the second extend batch is scheduling, the first extend batch is still running, so we cannot evict swa tokens if req.extend_batch_idx < 2: continue else: @@ -2309,6 +2326,7 @@ def _evict_swa(self, req: Req, pre_len: int): ), "cache_protected_len must be page aligned" req.swa_evicted_seqlen = max(req.swa_evicted_seqlen, req.cache_protected_len) + # -page_size: keep one extra page inside window to avoid evicting a partially-filled page new_swa_evicted_seqlen = max( req.swa_evicted_seqlen, pre_len - sliding_window_size - self.tree_cache.page_size, diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index d1baae96181a..72005f4de14f 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -37,6 +37,7 @@ from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, InsertParams, MatchPrefixParams, ) @@ -573,11 +574,9 @@ def _add_dllm_req(self, req: Req, prefix_len: int): self._update_prefill_budget(prefix_len, trunc_len, 0) def _req_inc_lock_ref(self, req: Req): + result = self.tree_cache.inc_lock_ref(req.last_node) if self.is_hybrid_swa: - swa_uuid_for_lock = self.tree_cache.inc_lock_ref(req.last_node) - req.swa_uuid_for_lock = swa_uuid_for_lock - else: - self.tree_cache.inc_lock_ref(req.last_node) + req.swa_uuid_for_lock = result.swa_uuid_for_lock def add_dllm_staging_req(self, req: Req): assert self.dllm_config is not None @@ -642,14 +641,14 @@ def add_chunked_req(self, req: Req): @contextmanager def _lock_node(self, last_node: TreeNode): try: - if self.tree_cache.supports_swa() and self.tree_cache.is_tree_cache(): - swa_uuid_for_lock = self.tree_cache.inc_lock_ref(last_node) - else: - self.tree_cache.inc_lock_ref(last_node) + result = self.tree_cache.inc_lock_ref(last_node) + swa_uuid_for_lock = result.swa_uuid_for_lock yield None finally: if self.tree_cache.supports_swa() and self.tree_cache.is_tree_cache(): - self.tree_cache.dec_lock_ref(last_node, swa_uuid_for_lock) + self.tree_cache.dec_lock_ref( + last_node, DecLockRefParams(swa_uuid_for_lock=swa_uuid_for_lock) + ) else: self.tree_cache.dec_lock_ref(last_node) diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index a383c5aa2c63..1bb37265e0e7 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -70,11 +70,27 @@ class InsertResult: mamba_exist: bool = False +@dataclasses.dataclass +class IncLockRefResult: + delta: Optional[int] = None + swa_uuid_for_lock: Optional[int] = None + + +@dataclasses.dataclass +class DecLockRefParams: + swa_uuid_for_lock: Optional[int] = None + + +@dataclasses.dataclass +class DecLockRefResult: + delta: Optional[int] = None + + @dataclasses.dataclass class EvictParams: """Unified parameters for evict across different cache types""" - num_tokens: int + num_tokens: int = 0 swa_num_tokens: int = 0 mamba_num: int = 0 @@ -155,11 +171,13 @@ def evict(self, params: EvictParams) -> EvictResult: pass @abstractmethod - def inc_lock_ref(self, node: Any): + def inc_lock_ref(self, node: Any) -> "IncLockRefResult": pass @abstractmethod - def dec_lock_ref(self, node: Any, swa_uuid_for_lock: Optional[str] = None): + def dec_lock_ref( + self, node: Any, params: Optional["DecLockRefParams"] = None, **kwargs + ) -> "DecLockRefResult": pass def evictable_size(self): diff --git a/python/sglang/srt/mem_cache/chunk_cache.py b/python/sglang/srt/mem_cache/chunk_cache.py index 7ddee32ac1b4..203998e91fee 100644 --- a/python/sglang/srt/mem_cache/chunk_cache.py +++ b/python/sglang/srt/mem_cache/chunk_cache.py @@ -9,8 +9,11 @@ from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, InsertParams, InsertResult, MatchPrefixParams, @@ -79,11 +82,13 @@ def cache_unfinished_req(self, req: Req, chunked=False): def evict(self, params: EvictParams) -> EvictResult: return EvictResult() - def inc_lock_ref(self, node: Any): - return 0 + def inc_lock_ref(self, node: Any) -> IncLockRefResult: + return IncLockRefResult(delta=0) - def dec_lock_ref(self, node: Any, swa_uuid_for_lock: Optional[str] = None): - return 0 + def dec_lock_ref( + self, node: Any, params: Optional[DecLockRefParams] = None, **kwargs + ) -> DecLockRefResult: + return DecLockRefResult(delta=0) def protected_size(self): # NOTE: no protected size in chunk cache. Chunk cache's eviction is the same with request's lifecycle. diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index b95ff3033009..5f5d6780167b 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -14,8 +14,11 @@ from sglang.srt.managers.cache_controller import HiCacheController, PrefetchOperation from sglang.srt.mem_cache.base_prefix_cache import ( + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, InsertParams, InsertResult, MatchPrefixParams, @@ -720,9 +723,9 @@ def loading_check(self): def evictable_size(self): return self.evictable_size_ - def inc_lock_ref(self, node: TreeNode): + def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: if self.disable: - return 0 + return IncLockRefResult(delta=0) delta = 0 while node != self.root_node: @@ -734,11 +737,13 @@ def inc_lock_ref(self, node: TreeNode): self._update_leaf_status(node) self._update_host_leaf_status(node) node = node.parent - return delta + return IncLockRefResult(delta=delta) - def dec_lock_ref(self, node: TreeNode): + def dec_lock_ref( + self, node: TreeNode, params: Optional[DecLockRefParams] = None, **kwargs + ) -> DecLockRefResult: if self.disable: - return 0 + return DecLockRefResult(delta=0) delta = 0 while node != self.root_node: @@ -754,7 +759,7 @@ def dec_lock_ref(self, node: TreeNode): node is self.root_node ), f"This request holds the node from another tree" node = node.parent - return delta + return DecLockRefResult(delta=delta) def _update_host_leaf_status(self, node: TreeNode): if not node.evicted or node.lock_ref > 0: @@ -887,7 +892,8 @@ def load_back( ancester_node = node # protect the ancestor nodes from eviction - delta = self.inc_lock_ref(ancester_node) + result = self.inc_lock_ref(ancester_node) + delta = result.delta # load it all or not at all host_indices = torch.cat([n.host_value for n in nodes_to_load]) diff --git a/python/sglang/srt/mem_cache/mamba_radix_cache.py b/python/sglang/srt/mem_cache/mamba_radix_cache.py index 07f5278d964c..fe9e5dff0cf3 100644 --- a/python/sglang/srt/mem_cache/mamba_radix_cache.py +++ b/python/sglang/srt/mem_cache/mamba_radix_cache.py @@ -35,8 +35,11 @@ ) from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, InsertParams, InsertResult, MatchPrefixParams, @@ -785,14 +788,14 @@ def evict_full(self, full_num_tokens: int) -> int: return full_num_evicted - def inc_lock_ref(self, node: TreeNode) -> Optional[int]: + def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: """ Increment the lock reference count for the node. It locks the full_lock_ref for nodes between the [last node, root), exclusive. It locks the mamba_lock_ref for current node if its mamba_value exists. """ if self.disable: - return None + return IncLockRefResult() # protect mamba value in current node if it exists if node.mamba_value is not None: @@ -811,16 +814,18 @@ def inc_lock_ref(self, node: TreeNode) -> Optional[int]: self.full_protected_size_ += len(node.value) node.full_lock_ref += 1 node = node.parent - return None + return IncLockRefResult() - def dec_lock_ref(self, node: TreeNode): + def dec_lock_ref( + self, node: TreeNode, params: Optional[DecLockRefParams] = None, **kwargs + ) -> DecLockRefResult: """ Decrement the lock reference count for the node. It unlocks the full_lock_ref for nodes between the [last node, root), exclusive. It unlocks the mamba_lock_ref for current node if its mamba_value exists. """ if self.disable: - return None + return DecLockRefResult() if node.mamba_value is not None: assert ( @@ -840,6 +845,7 @@ def dec_lock_ref(self, node: TreeNode): self.full_protected_size_ -= len(node.value) node.full_lock_ref -= 1 node = node.parent + return DecLockRefResult() def sanity_check(self): if self.disable: diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 42b169728ad5..0ab9f233f912 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -42,8 +42,11 @@ ) from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, InsertParams, InsertResult, MatchPrefixParams, @@ -591,9 +594,9 @@ def evict(self, params: EvictParams) -> EvictResult: self.update_eviction_metrics(num_evicted, start_time) return EvictResult(num_tokens_evicted=num_evicted) - def inc_lock_ref(self, node: TreeNode): + def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: if self.disable: - return 0 + return IncLockRefResult(delta=0) delta = 0 while node != self.root_node: @@ -604,11 +607,13 @@ def inc_lock_ref(self, node: TreeNode): node.lock_ref += 1 self._update_leaf_status(node) node = node.parent - return delta + return IncLockRefResult(delta=delta) - def dec_lock_ref(self, node: TreeNode): + def dec_lock_ref( + self, node: TreeNode, params: Optional[DecLockRefParams] = None, **kwargs + ) -> DecLockRefResult: if self.disable: - return 0 + return DecLockRefResult(delta=0) delta = 0 while node != self.root_node: @@ -623,7 +628,7 @@ def dec_lock_ref(self, node: TreeNode): node is self.root_node ), f"This request holds the node from another tree" node = node.parent - return delta + return DecLockRefResult(delta=delta) def evictable_size(self): return self.evictable_size_ diff --git a/python/sglang/srt/mem_cache/radix_cache_cpp.py b/python/sglang/srt/mem_cache/radix_cache_cpp.py index 6ed293d1b39d..48717d681eeb 100644 --- a/python/sglang/srt/mem_cache/radix_cache_cpp.py +++ b/python/sglang/srt/mem_cache/radix_cache_cpp.py @@ -8,8 +8,11 @@ from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, MatchPrefixParams, MatchResult, ) @@ -123,21 +126,15 @@ def _insert(self, key: RadixKey, value: torch.Tensor) -> int: raise NotImplementedError("Host cache is not supported yet") - def dec_lock_ref(self, node: TreeNodeCpp): - """ - Decrement the reference count of a node to root of the radix tree. - Args: - node (TreeNodeCpp): The handle of the node to decrement the reference count for. - """ - self.tree.lock_ref(node, False) # do not increment + def dec_lock_ref( + self, node: TreeNodeCpp, params=None, **kwargs + ) -> DecLockRefResult: + self.tree.lock_ref(node, False) + return DecLockRefResult() - def inc_lock_ref(self, node: TreeNodeCpp): - """ - Increment the reference count of from a node to root of the radix tree. - Args: - node (TreeNodeCpp): The handle of the node to increment the reference count for. - """ + def inc_lock_ref(self, node: TreeNodeCpp) -> IncLockRefResult: self.tree.lock_ref(node, True) + return IncLockRefResult() def evict(self, params: EvictParams) -> EvictResult: start_time = time.perf_counter() diff --git a/python/sglang/srt/mem_cache/swa_radix_cache.py b/python/sglang/srt/mem_cache/swa_radix_cache.py index 09468e754ffb..a4705cf6b246 100644 --- a/python/sglang/srt/mem_cache/swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/swa_radix_cache.py @@ -31,8 +31,11 @@ from sglang.srt.environ import envs from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, + DecLockRefParams, + DecLockRefResult, EvictParams, EvictResult, + IncLockRefResult, InsertParams, InsertResult, MatchPrefixParams, @@ -499,7 +502,12 @@ def cache_finished_req(self, req: Req, is_insert: bool = True) -> None: self.token_to_kv_pool_allocator.free(kv_indices[page_aligned_len:]) # Remove req slot release the cache lock - self.dec_lock_ref(req.last_node, req.swa_uuid_for_lock) + self.dec_lock_ref( + req.last_node, + DecLockRefParams(swa_uuid_for_lock=req.swa_uuid_for_lock), + skip_swa=req.swa_prefix_lock_released, + ) + req.swa_prefix_lock_released = False def cache_unfinished_req(self, req: Req, chunked=False) -> None: """Cache request when it is unfinished.""" @@ -573,8 +581,14 @@ def cache_unfinished_req(self, req: Req, chunked=False) -> None: req.cache_protected_len = len(new_indices) - self.dec_lock_ref(req.last_node, req.swa_uuid_for_lock) - swa_uuid_for_lock = self.inc_lock_ref(new_last_node) + self.dec_lock_ref( + req.last_node, + DecLockRefParams(swa_uuid_for_lock=req.swa_uuid_for_lock), + skip_swa=req.swa_prefix_lock_released, + ) + req.swa_prefix_lock_released = False + result = self.inc_lock_ref(new_last_node) + swa_uuid_for_lock = result.swa_uuid_for_lock # `req.prefix_indices` will be used in `PrefillAdder::add_chunked_req` later if self.page_size != 1: @@ -621,12 +635,14 @@ def evict(self, params: EvictParams) -> EvictResult: # 1. free node kv indices, evict full and swa tokens self.token_to_kv_pool_allocator.free(x.value) full_num_evicted += len(x.value) - swa_num_evicted += len(x.value) + if not x.swa_tombstone: + swa_num_evicted += len(x.value) # 2. get the next leaf, update the lru lists x_next = self.full_lru_list.get_prev_leaf_no_lock(x) self.full_lru_list.remove_node(x) - self.swa_lru_list.remove_node(x) + if not x.swa_tombstone: + self.swa_lru_list.remove_node(x) # 3. delete the leaf node self._delete_leaf(x) @@ -663,6 +679,15 @@ def evict(self, params: EvictParams) -> EvictResult: # 3. tombstone the node self._tombstone_internal_node(x) + elif x.full_lock_ref > 0: + self.token_to_kv_pool_allocator.free_swa(x.value) + swa_num_evicted += len(x.value) + + x_next = self.swa_lru_list.get_prev_no_lock(x) + self.swa_lru_list.remove_node(x) + + self.swa_evictable_size_ -= len(x.value) + x.swa_tombstone = True else: assert ( x.full_lock_ref == 0 @@ -690,7 +715,7 @@ def evict(self, params: EvictParams) -> EvictResult: num_tokens_evicted=full_num_evicted, swa_num_tokens_evicted=swa_num_evicted ) - def inc_lock_ref(self, node: TreeNode) -> Optional[int]: + def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: """ Increment the lock reference count for the node. Returns the swa_uuid_for_lock, which needs to be passed to dec_lock_ref. @@ -698,7 +723,7 @@ def inc_lock_ref(self, node: TreeNode) -> Optional[int]: It locks the swa_lock_ref for nodes between the [last node, swa_uuid_for_lock], inclusive. """ if self.disable: - return None + return IncLockRefResult() swa_lock_size = 0 swa_uuid_for_lock = None @@ -729,19 +754,29 @@ def inc_lock_ref(self, node: TreeNode) -> Optional[int]: node.swa_uuid = gen_swa_uuid() swa_uuid_for_lock = node.swa_uuid node = node.parent - return swa_uuid_for_lock + return IncLockRefResult(swa_uuid_for_lock=swa_uuid_for_lock) - def dec_lock_ref(self, node: TreeNode, swa_uuid_for_lock: Optional[int] = None): + def dec_lock_ref( + self, + node: TreeNode, + params: Optional[DecLockRefParams] = None, + skip_swa: bool = False, + ) -> DecLockRefResult: """ Decrement the lock reference count for the node. It unlocks the full_lock_ref for nodes between the [last node, root), exclusive. It unlocks the swa_lock_ref for nodes between the [last node, swa_uuid_for_lock], inclusive. If swa_uuid_for_lock is None, it unlocks to the root, exclusive. + + If skip_swa is True, only the full_lock_ref is decremented; the SWA lock is + assumed to have been released already (e.g. via `dec_swa_lock_only`). """ + swa_uuid_for_lock = params.swa_uuid_for_lock if params is not None else None + if self.disable: - return + return DecLockRefResult() - dec_lock_swa = True + dec_lock_swa = not skip_swa while node != self.root_node: assert ( node.full_lock_ref > 0 @@ -768,6 +803,46 @@ def dec_lock_ref(self, node: TreeNode, swa_uuid_for_lock: Optional[int] = None): node = node.parent + return DecLockRefResult() + + def dec_swa_lock_only( + self, node: TreeNode, swa_uuid_for_lock: Optional[int] = None + ): + """ + Decrement only the swa_lock_ref along the chain [node, swa_uuid_for_lock], + inclusive. The full_lock_ref is left untouched. + + Used to early-release the SWA portion of a request's tree lock once the + request's decode position has advanced past the sliding window. + + For leaf nodes, free the SWA pool slots immediately and mark as + swa_tombstone=True. For internal nodes, transition from protected to evictable. + """ + if self.disable: + return + + while node != self.root_node: + assert ( + not node.swa_tombstone + ), f"dec_swa_lock_only on swa_tombstone node, {node.id=}" + assert ( + node.swa_lock_ref > 0 + ), f"dec_swa_lock_only on node with {node.swa_lock_ref=}, {node.id=}" + + if node.swa_lock_ref == 1: + self.swa_protected_size_ -= len(node.value) + if len(node.children) == 0: + self.token_to_kv_pool_allocator.free_swa(node.value) + self.swa_lru_list.remove_node(node) + node.swa_tombstone = True + else: + self.swa_evictable_size_ += len(node.value) + node.swa_lock_ref -= 1 + + if swa_uuid_for_lock and node.swa_uuid == swa_uuid_for_lock: + break + node = node.parent + def sanity_check(self): self.full_lru_list.sanity_check(self) self.swa_lru_list.sanity_check(self) @@ -1149,15 +1224,13 @@ def _iteratively_delete_tombstone_leaf( return node, full_num_evicted def _delete_leaf(self, node: TreeNode) -> None: - assert ( - not node.swa_tombstone - ), f"Invariant violated: leaf node is a tombstone, {node.id=}" assert len(node.children) == 0, f"leaf node has children, {node.id=}" key = self.get_child_key_fn(node.key) v = node.parent.children.pop(key, None) assert v == node, f"parent does not have child key, {key}" self.full_evictable_size_ -= len(node.key) - self.swa_evictable_size_ -= len(node.key) + if not node.swa_tombstone: + self.swa_evictable_size_ -= len(node.key) def _tombstone_internal_node(self, node: TreeNode) -> None: assert len(node.children) != 0, f"Cannot tombstone a leaf node, {node.id=}"