Fix Memory leaks and add tests to prevent memory leaks during md_clear from passing (#1233)

Related issue number: #1232

Author: Vizonex

.pre-commit-config.yaml

@@ -130,6 +130,8 @@ repos:
     - pytest_codspeed
     - Sphinx >= 5.3.0
     - sphinxcontrib-spelling
+    - types-psutil
+    - psutil
     args:
     - --python-version=3.13
     - --txt-report=.tox/.tmp/.mypy/python-3.13
@@ -146,6 +148,8 @@ repos:
     - pytest_codspeed
     - Sphinx >= 5.3.0
     - sphinxcontrib-spelling
+    - types-psutil
+    - psutil
     args:
     - --python-version=3.11
     - --txt-report=.tox/.tmp/.mypy/python-3.11

CHANGES/1233.bugfix.rst

@@ -0,0 +1,2 @@
+Fix ``MutliDict`` & ``CIMultiDict`` memory leak when deleting values or clearing them 
+-- by :user:`Vizonex`

CHANGES/1233.contrib.rst

@@ -0,0 +1,2 @@
+Added memory leak test for popping or deleting attributes from a multidict to prevent future issues or bogus claims.
+-- by :user:`Vizonex`

multidict/_multilib/hashtable.h

@@ -1877,7 +1877,7 @@ md_traverse(MultiDictObject *md, visitproc visit, void *arg)
 static inline int
 md_clear(MultiDictObject *md)
 {
-    if (md->used == 0) {
+    if (md->keys == NULL || md->keys == &empty_htkeys) {
         return 0;
     }
     md->version = NEXT_VERSION(md->state);

pytest.ini

@@ -41,7 +41,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS
 # Marks tests with an empty parameterset as xfail(run=False)
 empty_parameter_set_mark = xfail
 
-faulthandler_timeout = 60
+faulthandler_timeout = 90
 
 filterwarnings =
   error

requirements/pytest.txt

@@ -2,3 +2,4 @@ objgraph==3.6.2
 pytest==8.4.0
 pytest-codspeed==3.2.0
 pytest-cov==6.1.0
+psutil==7.0.0

tests/isolated/multidict_pop.py

@@ -0,0 +1,80 @@
+# Test for memory leaks surrounding deletion of values or
+# bad cleanups.
+# SEE: https://github.com/aio-libs/multidict/issues/1232
+# We want to make sure that bad predictions or bougus claims
+# of memory leaks can be prevented in the future.
+
+import gc
+import psutil
+import os
+from multidict import MultiDict
+
+
+def trim_ram() -> None:
+    """Forces python garbage collection."""
+    gc.collect()
+
+
+process = psutil.Process(os.getpid())
+
+
+def get_memory_usage() -> int:
+    memory_info = process.memory_info()
+    return memory_info.rss / (1024 * 1024)  # type: ignore[no-any-return]
+
+
+keys = [f"X-Any-{i}" for i in range(100)]
+headers = {key: key * 2 for key in keys}
+
+
+def check_for_leak() -> None:
+    trim_ram()
+    usage = get_memory_usage()
+    assert usage < 50, f"Memory leaked at: {usage} MB"
+
+
+def _test_pop() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.pop(k)
+        check_for_leak()
+
+
+def _test_popall() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.popall(k)
+        check_for_leak()
+
+
+def _test_popone() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.popone(k)
+        check_for_leak()
+
+
+def _test_del() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                del result[k]
+        check_for_leak()
+
+
+def _run_isolated_case() -> None:
+    _test_pop()
+    _test_popall()
+    _test_popone()
+    _test_del()
+
+
+if __name__ == "__main__":
+    _run_isolated_case()

tests/test_leaks.py

@@ -15,6 +15,7 @@
         "multidict_extend_multidict.py",
         "multidict_extend_tuple.py",
         "multidict_update_multidict.py",
+        "multidict_pop.py",
     ),
 )
 @pytest.mark.leaks