From dcbc141996d00ff2eee557c5b4c6908a221fd339 Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 18:29:39 -0500 Subject: [PATCH 01/16] add atomic global counter system --- multidict/_multilib/state.h | 4 ++- tests/isolated/multidict_global_counter.py | 33 ++++++++++++++++++++++ tests/test_leaks.py | 1 + 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/isolated/multidict_global_counter.py diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index 4e2610b6c..b5839f9a7 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -5,6 +5,8 @@ extern "C" { #endif +#include + /* State of the _multidict module */ typedef struct { PyTypeObject *IStrType; @@ -128,7 +130,7 @@ get_mod_state_by_def(PyObject *self) static inline uint64_t NEXT_VERSION(mod_state *state) { - return ++state->global_version; + return atomic_fetch_add_explicit((_Atomic(uint64_t)*)&state->global_version, 1, memory_order_relaxed); } #ifdef __cplusplus diff --git a/tests/isolated/multidict_global_counter.py b/tests/isolated/multidict_global_counter.py new file mode 100644 index 000000000..4acbcecb9 --- /dev/null +++ b/tests/isolated/multidict_global_counter.py @@ -0,0 +1,33 @@ +import threading +import multidict +from multidict import MultiDict +import sysconfig + +FREETHREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +if not FREETHREADED: + raise SystemExit(0) + +md = MultiDict() +N, M = 3, 100 +baseline = multidict.getversion(md) + + +def worker(tid): + for i in range(M): + md[f"k{tid}_{i}"] = i + + +if __name__ == "__main__": + threads = [threading.Thread(target=worker, args=(tid,)) for tid in range(N)] + for t in threads: + t.start() + for t in threads: + t.join() + + observed = multidict.getversion(md) - baseline + expected = N * M + assert expected == observed, ( + f"expected delta: {expected}" + f" observed: {observed} " + f"lost: {expected - observed}" + ) diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 105853ddd..18dc3de6c 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -18,6 +18,7 @@ "multidict_type_leak.py", "multidict_type_leak_items_values.py", "multidict_pop.py", + "multidict_global_counter.py", ), ) @pytest.mark.leaks From df0477ae8f82b680e26d9098690b4287e0fb0cee Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 18:36:01 -0500 Subject: [PATCH 02/16] add change to timeline --- CHANGES/1328.bugfix.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 CHANGES/1328.bugfix.rst diff --git a/CHANGES/1328.bugfix.rst b/CHANGES/1328.bugfix.rst new file mode 100644 index 000000000..e90957a77 --- /dev/null +++ b/CHANGES/1328.bugfix.rst @@ -0,0 +1,2 @@ +Fixed global counter system using an atomic variable. +-- by :user:`Vizonex`. From 6a35f59d7aa5a6e9570cf76e29ddddbd5a43733b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:36:02 +0000 Subject: [PATCH 03/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- multidict/_multilib/state.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index b5839f9a7..acdf29be5 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -130,7 +130,8 @@ get_mod_state_by_def(PyObject *self) static inline uint64_t NEXT_VERSION(mod_state *state) { - return atomic_fetch_add_explicit((_Atomic(uint64_t)*)&state->global_version, 1, memory_order_relaxed); + return atomic_fetch_add_explicit( + (_Atomic(uint64_t) *)&state->global_version, 1, memory_order_relaxed); } #ifdef __cplusplus From adcec6893b316ba14e1ef93da1ddf0a4b5e2742b Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 18:37:31 -0500 Subject: [PATCH 04/16] add changes to setup.py for windows users --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 318229fcc..48ed3fee4 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,13 @@ "-Werror", ] ) +else: + CFLAGS.extend( + [ + "/std:c11", + "/experimental:c11atomics", + ] + ) extensions = [ Extension( From aaf0b257b0f73ac76a11f82e6f727ddcdcb22948 Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 18:55:40 -0500 Subject: [PATCH 05/16] fix mypy issues with global_counter test --- tests/isolated/multidict_global_counter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/isolated/multidict_global_counter.py b/tests/isolated/multidict_global_counter.py index 4acbcecb9..223f4ad34 100644 --- a/tests/isolated/multidict_global_counter.py +++ b/tests/isolated/multidict_global_counter.py @@ -7,12 +7,12 @@ if not FREETHREADED: raise SystemExit(0) -md = MultiDict() +md: MultiDict[int] = MultiDict() N, M = 3, 100 -baseline = multidict.getversion(md) +baseline = multidict.getversion(md) # type: ignore[arg-type] -def worker(tid): +def worker(tid: int) -> None: for i in range(M): md[f"k{tid}_{i}"] = i @@ -24,7 +24,7 @@ def worker(tid): for t in threads: t.join() - observed = multidict.getversion(md) - baseline + observed = multidict.getversion(md) - baseline # type: ignore[arg-type] expected = N * M assert expected == observed, ( f"expected delta: {expected}" From ae8b77614bc8a5f9f810dc56be4db7cde597d2db Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 19:36:57 -0500 Subject: [PATCH 06/16] fix global counter test again --- tests/isolated/multidict_global_counter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/isolated/multidict_global_counter.py b/tests/isolated/multidict_global_counter.py index 223f4ad34..ae2c01b2f 100644 --- a/tests/isolated/multidict_global_counter.py +++ b/tests/isolated/multidict_global_counter.py @@ -4,8 +4,7 @@ import sysconfig FREETHREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) -if not FREETHREADED: - raise SystemExit(0) + md: MultiDict[int] = MultiDict() N, M = 3, 100 @@ -17,7 +16,7 @@ def worker(tid: int) -> None: md[f"k{tid}_{i}"] = i -if __name__ == "__main__": +if (__name__ == "__main__") and FREETHREADED: threads = [threading.Thread(target=worker, args=(tid,)) for tid in range(N)] for t in threads: t.start() From d949bebd40439de781969614d10a31edf09e11f1 Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 21:23:55 -0500 Subject: [PATCH 07/16] fix compiler issues with mac --- multidict/_multilib/state.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index acdf29be5..d5d382b6c 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -28,7 +28,7 @@ typedef struct { PyObject *str_lower; PyObject *str_name; - uint64_t global_version; + _Atomic uint64_t global_version; } mod_state; static inline mod_state * @@ -131,7 +131,7 @@ static inline uint64_t NEXT_VERSION(mod_state *state) { return atomic_fetch_add_explicit( - (_Atomic(uint64_t) *)&state->global_version, 1, memory_order_relaxed); + &state->global_version, 1, memory_order_relaxed); } #ifdef __cplusplus From 89214fae55ef57f3db78f81ef059eb56ab1676ba Mon Sep 17 00:00:00 2001 From: Vizonex <114684698+Vizonex@users.noreply.github.com> Date: Tue, 21 Apr 2026 21:33:23 -0500 Subject: [PATCH 08/16] Update multidict/_multilib/state.h Co-authored-by: J. Nick Koston --- multidict/_multilib/state.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index d5d382b6c..0b862a4ae 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -131,7 +131,7 @@ static inline uint64_t NEXT_VERSION(mod_state *state) { return atomic_fetch_add_explicit( - &state->global_version, 1, memory_order_relaxed); + &state->global_version, 1, memory_order_relaxed) + 1; } #ifdef __cplusplus From 6024eaa35d590c0df05fd4db93e874f6a0beb719 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 02:34:41 +0000 Subject: [PATCH 09/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- multidict/_multilib/state.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index 0b862a4ae..7b5e59ed2 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -131,7 +131,8 @@ static inline uint64_t NEXT_VERSION(mod_state *state) { return atomic_fetch_add_explicit( - &state->global_version, 1, memory_order_relaxed) + 1; + &state->global_version, 1, memory_order_relaxed) + + 1; } #ifdef __cplusplus From d9e4b008054975cecf9372295624ddf32a73995a Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 22:10:14 -0500 Subject: [PATCH 10/16] fine tune atomics to enusre all tests pass and add comment about why it's here --- multidict/_multilib/state.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index 7b5e59ed2..03827bcea 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -130,9 +130,10 @@ get_mod_state_by_def(PyObject *self) static inline uint64_t NEXT_VERSION(mod_state *state) { - return atomic_fetch_add_explicit( - &state->global_version, 1, memory_order_relaxed) + - 1; + /* threads have the ability to screw with the global version, + * using an atomic variable ensures the global version is + * always accurate. */ + return atomic_fetch_add(&state->global_version, 1) + 1; } #ifdef __cplusplus From 3ac82ec78467c14540557df4fe7604c7dd735982 Mon Sep 17 00:00:00 2001 From: Vizonex <114684698+Vizonex@users.noreply.github.com> Date: Tue, 21 Apr 2026 22:22:20 -0500 Subject: [PATCH 11/16] Update multidict/_multilib/state.h Co-authored-by: J. Nick Koston --- multidict/_multilib/state.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index 03827bcea..d8269dbc4 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -130,10 +130,8 @@ get_mod_state_by_def(PyObject *self) static inline uint64_t NEXT_VERSION(mod_state *state) { - /* threads have the ability to screw with the global version, - * using an atomic variable ensures the global version is - * always accurate. */ - return atomic_fetch_add(&state->global_version, 1) + 1; + /* relaxed is fine here as we only care about the atomicity of the RMW itself */ + return atomic_fetch_add_explicit(&state->global_version, 1, memory_order_relaxed) + 1 } #ifdef __cplusplus From 9a73bd47b1797fdad62c38fded14aaf8880a0b79 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Apr 2026 03:23:37 +0000 Subject: [PATCH 12/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- multidict/_multilib/state.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index d8269dbc4..95c890605 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -130,8 +130,11 @@ get_mod_state_by_def(PyObject *self) static inline uint64_t NEXT_VERSION(mod_state *state) { - /* relaxed is fine here as we only care about the atomicity of the RMW itself */ - return atomic_fetch_add_explicit(&state->global_version, 1, memory_order_relaxed) + 1 + /* relaxed is fine here as we only care about the atomicity of the RMW + * itself */ + return atomic_fetch_add_explicit( + &state->global_version, 1, memory_order_relaxed) + + 1 } #ifdef __cplusplus From b5f675d7c0ea803a1c6da3a3fbc89e7d9eec1292 Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 22:26:06 -0500 Subject: [PATCH 13/16] fix syntax error --- multidict/_multilib/state.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multidict/_multilib/state.h b/multidict/_multilib/state.h index 95c890605..ec836e29b 100644 --- a/multidict/_multilib/state.h +++ b/multidict/_multilib/state.h @@ -134,7 +134,7 @@ NEXT_VERSION(mod_state *state) * itself */ return atomic_fetch_add_explicit( &state->global_version, 1, memory_order_relaxed) + - 1 + 1; } #ifdef __cplusplus From a7fad511f8a1c86003b209eada4758f1572c148e Mon Sep 17 00:00:00 2001 From: Vizonex Date: Tue, 21 Apr 2026 22:56:30 -0500 Subject: [PATCH 14/16] fix setup.py and customize flags a little bit more --- setup.py | 56 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 48ed3fee4..ddc3fbcad 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ import os -import platform import sys from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext NO_EXTENSIONS = bool(os.environ.get("MULTIDICT_NO_EXTENSIONS")) DEBUG_BUILD = bool(os.environ.get("MULTIDICT_DEBUG_BUILD")) @@ -10,33 +10,41 @@ if sys.implementation.name != "cpython": NO_EXTENSIONS = True -CFLAGS = ["-O0", "-g3", "-UNDEBUG"] if DEBUG_BUILD else ["-O3", "-DNDEBUG"] - -if platform.system() != "Windows": - CFLAGS.extend( - [ - "-std=c11", - "-Wall", - "-Wsign-compare", - "-Wconversion", - "-fno-strict-aliasing", - "-Wno-conversion", - "-Werror", - ] - ) -else: - CFLAGS.extend( - [ - "/std:c11", - "/experimental:c11atomics", - ] - ) +BASE_CFLAGS = ["O0", "g3", "UNDEBUG"] if DEBUG_BUILD else ["O3", "DNDEBUG"] + +UNIX_CFLAGS = [ + "-std=c11", + "-Wall", + "-Wsign-compare", + "-Wconversion", + "-fno-strict-aliasing", + "-Wno-conversion", + "-Werror", +] + +MSVC_CFLAGS = ["/std:c11", "/experimental:c11atomics"] + + +class BuildExt(build_ext): + def build_extensions(self): + if self.compiler.compiler_type == "msvc": + for ext in self.extensions: + ext.extra_compile_args.extend(MSVC_CFLAGS) + for flag in BASE_CFLAGS: + # XXX: MSVC Doesn't have a /O3 flag only O2 is possible... + ext.extra_compile_args.append("/O2" if flag == "O3" else f"/{flag}") + else: + for ext in self.extensions: + ext.extra_compile_args.extend(UNIX_CFLAGS) + for flag in BASE_CFLAGS: + ext.extra_compile_args.append(f"-{flag}") + super().build_extensions() + extensions = [ Extension( "multidict._multidict", ["multidict/_multidict.c"], - extra_compile_args=CFLAGS, ), ] @@ -45,7 +53,7 @@ print("*********************") print("* Accelerated build *") print("*********************") - setup(ext_modules=extensions) + setup(ext_modules=extensions, cmdclass={"build_ext": BuildExt}) else: print("*********************") print("* Pure Python build *") From e872f37eaa8f41ef4049320937a878246727a618 Mon Sep 17 00:00:00 2001 From: Vizonex <114684698+Vizonex@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:16:01 -0500 Subject: [PATCH 15/16] Update tests/isolated/multidict_global_counter.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/isolated/multidict_global_counter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/isolated/multidict_global_counter.py b/tests/isolated/multidict_global_counter.py index ae2c01b2f..fd8c25ff6 100644 --- a/tests/isolated/multidict_global_counter.py +++ b/tests/isolated/multidict_global_counter.py @@ -1,7 +1,8 @@ +import sysconfig import threading + import multidict from multidict import MultiDict -import sysconfig FREETHREADED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) From 8bb2038c04a69eded1a2bf7e24923513fdd20a73 Mon Sep 17 00:00:00 2001 From: Vizonex <114684698+Vizonex@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:16:14 -0500 Subject: [PATCH 16/16] Update setup.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ddc3fbcad..a5fbb899c 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ def build_extensions(self): Extension( "multidict._multidict", ["multidict/_multidict.c"], + extra_compile_args=[], ), ]