From cb97b7c2a31da0130a3340a8d5b1d9328f1f0f76 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 13 Jul 2026 12:45:17 -0300 Subject: [PATCH 1/2] Pin PyThreadStates so native extensions can cache them per thread Py.GIL() acquires the GIL via PyGILState_Ensure/Release; the outermost scope on a .NET thread creates a fresh PyThreadState and deletes it on dispose. pybind11-based extensions (matplotlib >= 3.10 _path/ft2font, scipy, PyTorch) cache the first PyThreadState* they see per OS thread in pybind11 internals TLS and never invalidate it, so once a scope deletes the thread state the next native GIL acquire on that thread restores a dangling pointer: 0xC0000005 access violation or heap corruption. Root cause of QuantConnect/Lean#9203 (ReportChartTests crashing the test host on the first matplotlib render). Pin each thread state with one extra never-released PyGILState_Ensure per thread per runtime run, keeping the gilstate counter >= 1 so the thread state lives until engine shutdown - the same lifetime CPython gives thread states it binds itself. The extra Ensure runs with the GIL already held (a bare counter increment): GIL acquire/release timing is unchanged and empty-scope microbenchmarks are within noise (0.212 -> 0.209 us/scope). Regression test: threading.local values live in the thread state dict, so a value stored in one Py.GIL scope survives a second scope on the same thread only if the thread state was not deleted. Co-Authored-By: Claude Fable 5 --- src/embed_tests/TestGILState.cs | 53 +++++++++++++++++++++++++++++++++ src/runtime/Py.cs | 16 ++++++++++ 2 files changed, 69 insertions(+) diff --git a/src/embed_tests/TestGILState.cs b/src/embed_tests/TestGILState.cs index bf6f02dc6..1f454c1b2 100644 --- a/src/embed_tests/TestGILState.cs +++ b/src/embed_tests/TestGILState.cs @@ -1,5 +1,7 @@ namespace Python.EmbeddingTest { + using System; + using System.Threading; using NUnit.Framework; using Python.Runtime; @@ -18,6 +20,57 @@ public void CanDisposeMultipleTimes() } } + /// + /// The thread's PyThreadState must survive between GIL scopes. Native extensions + /// built with pybind11 (e.g. matplotlib >= 3.10 _path/ft2font) cache the pointer + /// per OS thread and crash with an access violation if a later scope runs after + /// the thread state was deleted. threading.local values live in the thread state + /// dictionary, so they survive a second scope only if the thread state did. + /// + [Test] + public void ThreadStateIsPreservedBetweenGILScopes() + { + var result = 0; + Exception error = null; + var thread = new Thread(() => + { + try + { + PyModule scope; + using (Py.GIL()) + { + scope = Py.CreateScope(); + scope.Exec("import threading\nlocal = threading.local()\nlocal.value = 42"); + } + using (Py.GIL()) + using (scope) + { + using var value = scope.Eval("getattr(local, 'value', -1)"); + result = value.As(); + } + } + catch (Exception e) + { + error = e; + } + }); + // the fixture initializes the engine with the GIL held on this thread: + // release it so the worker thread can acquire it + var ts = PythonEngine.BeginAllowThreads(); + try + { + thread.Start(); + thread.Join(); + } + finally + { + PythonEngine.EndAllowThreads(ts); + } + + Assert.IsNull(error); + Assert.AreEqual(42, result); + } + [OneTimeSetUp] public void SetUp() { diff --git a/src/runtime/Py.cs b/src/runtime/Py.cs index 824cb9d15..65e6999bc 100644 --- a/src/runtime/Py.cs +++ b/src/runtime/Py.cs @@ -28,12 +28,28 @@ public void Dispose() public class GILState : IDisposable { + // Tracks the runtime run for which this thread's PyThreadState has been pinned. + // Native extensions built with pybind11 (e.g. matplotlib >= 3.10 _path/ft2font) + // cache the PyThreadState pointer per OS thread and reuse it later; if the + // outermost PyGILState_Release deletes the thread state, that cached pointer + // dangles and the next native GIL acquire crashes with an access violation. + // Pinning: one extra, never-released PyGILState_Ensure per thread keeps the + // gilstate counter >= 1 so the thread state lives until engine shutdown. + [ThreadStatic] private static int _pinnedOnRun; + private readonly PyGILState state; private bool isDisposed; internal GILState() { state = PythonEngine.AcquireLock(); + + var run = Runtime.GetRun(); + if (_pinnedOnRun != run) + { + _pinnedOnRun = run; + Runtime.PyGILState_Ensure(); + } } public virtual void Dispose() From aa263434224155420d4e25057210156537baff0d Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 13 Jul 2026 12:45:17 -0300 Subject: [PATCH 2/2] Update version to 2.0.62 Bump package , AssemblyVersion/AssemblyFileVersion and the perf-test baseline reference to 2.0.62. Co-Authored-By: Claude Fable 5 --- src/perf_tests/Python.PerformanceTests.csproj | 4 ++-- src/runtime/Properties/AssemblyInfo.cs | 4 ++-- src/runtime/Python.Runtime.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index 2655a5c10..6d38f74bc 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + compile @@ -25,7 +25,7 @@ - + diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index 9514b7ab6..ab4fddd1d 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,5 +4,5 @@ [assembly: InternalsVisibleTo("Python.EmbeddingTest, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] [assembly: InternalsVisibleTo("Python.Test, PublicKey=00240000048000009400000006020000002400005253413100040000110000005ffd8f49fb44ab0641b3fd8d55e749f716e6dd901032295db641eb98ee46063cbe0d4a1d121ef0bc2af95f8a7438d7a80a3531316e6b75c2dae92fb05a99f03bf7e0c03980e1c3cfb74ba690aca2f3339ef329313bcc5dccced125a4ffdc4531dcef914602cd5878dc5fbb4d4c73ddfbc133f840231343e013762884d6143189")] -[assembly: AssemblyVersion("2.0.61")] -[assembly: AssemblyFileVersion("2.0.61")] +[assembly: AssemblyVersion("2.0.62")] +[assembly: AssemblyFileVersion("2.0.62")] diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 66f746d08..359e42944 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -5,7 +5,7 @@ Python.Runtime Python.Runtime QuantConnect.pythonnet - 2.0.61 + 2.0.62 false LICENSE https://github.com/pythonnet/pythonnet