Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 26e88d7

Browse files
amos402Martin-Molinero
amos402
authored and
Martin-Molinero
committed
Finalizer for PyObject
1 parent f116429 commit 26e88d7

File tree

7 files changed

+319
-31
lines changed

7 files changed

+319
-31
lines changed

‎src/embed_tests/TestFinalizer.cs‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
using NUnit.Framework;
2+
using Python.Runtime;
3+
using System;
4+
using System.Threading;
5+
6+
namespace Python.EmbeddingTest
7+
{
8+
public class TestFinalizer
9+
{
10+
private string _PYTHONMALLOC = string.Empty;
11+
12+
[SetUp]
13+
public void SetUp()
14+
{
15+
try
16+
{
17+
_PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC");
18+
}
19+
catch (ArgumentNullException)
20+
{
21+
_PYTHONMALLOC = string.Empty;
22+
}
23+
Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc");
24+
PythonEngine.Initialize();
25+
}
26+
27+
[TearDown]
28+
public void TearDown()
29+
{
30+
PythonEngine.Shutdown();
31+
if (string.IsNullOrEmpty(_PYTHONMALLOC))
32+
{
33+
Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC);
34+
}
35+
}
36+
37+
private static void FullGCCollect()
38+
{
39+
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
40+
GC.WaitForFullGCComplete();
41+
GC.WaitForPendingFinalizers();
42+
}
43+
44+
[Test]
45+
public void CollectBasicObject()
46+
{
47+
int thId = Thread.CurrentThread.ManagedThreadId;
48+
Finalizer.Instance.Threshold = 1;
49+
bool called = false;
50+
EventHandler<Finalizer.CollectArgs> handler = (s, e) =>
51+
{
52+
Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId);
53+
Assert.GreaterOrEqual(e.ObjectCount, 1);
54+
called = true;
55+
};
56+
Finalizer.Instance.CollectOnce += handler;
57+
FullGCCollect();
58+
PyLong obj = new PyLong(1024);
59+
60+
WeakReference shortWeak = new WeakReference(obj);
61+
WeakReference longWeak = new WeakReference(obj, true);
62+
obj = null;
63+
FullGCCollect();
64+
// The object has been resurrected
65+
Assert.IsFalse(shortWeak.IsAlive);
66+
Assert.IsTrue(longWeak.IsAlive);
67+
68+
Assert.IsFalse(called);
69+
var garbage = Finalizer.Instance.GetCollectedObjects();
70+
// FIXME: If make some query for garbage,
71+
// the above case will failed Assert.IsFalse(shortWeak.IsAlive)
72+
//Assert.IsTrue(garbage.All(T => T.IsAlive));
73+
74+
Finalizer.Instance.CallPendingFinalizers();
75+
Assert.IsTrue(called);
76+
77+
FullGCCollect();
78+
//Assert.IsFalse(garbage.All(T => T.IsAlive));
79+
80+
Assert.IsNull(longWeak.Target);
81+
82+
Finalizer.Instance.CollectOnce -= handler;
83+
}
84+
85+
private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale)
86+
{
87+
// Must larger than 512 bytes make sure Python use
88+
string str = new string('1', 1024);
89+
Finalizer.Instance.Enable = true;
90+
FullGCCollect();
91+
FullGCCollect();
92+
pyCollect.Invoke();
93+
Finalizer.Instance.CallPendingFinalizers();
94+
Finalizer.Instance.Enable = enbale;
95+
96+
// Estimate unmanaged memory size
97+
long before = Environment.WorkingSet - GC.GetTotalMemory(true);
98+
for (int i = 0; i < 10000; i++)
99+
{
100+
// Memory will leak when disable Finalizer
101+
new PyString(str);
102+
}
103+
FullGCCollect();
104+
FullGCCollect();
105+
pyCollect.Invoke();
106+
if (enbale)
107+
{
108+
Finalizer.Instance.CallPendingFinalizers();
109+
}
110+
111+
FullGCCollect();
112+
FullGCCollect();
113+
long after = Environment.WorkingSet - GC.GetTotalMemory(true);
114+
return after - before;
115+
116+
}
117+
118+
/// <summary>
119+
/// Because of two vms both have their memory manager,
120+
/// this test only prove the finalizer has take effect.
121+
/// </summary>
122+
[Test]
123+
[Ignore("Too many uncertainties, only manual on when debugging")]
124+
public void SimpleTestMemory()
125+
{
126+
bool oldState = Finalizer.Instance.Enable;
127+
try
128+
{
129+
using (PyObject gcModule = PythonEngine.ImportModule("gc"))
130+
using (PyObject pyCollect = gcModule.GetAttr("collect"))
131+
{
132+
long span1 = CompareWithFinalizerOn(pyCollect, false);
133+
long span2 = CompareWithFinalizerOn(pyCollect, true);
134+
Assert.Less(span2, span1);
135+
}
136+
}
137+
finally
138+
{
139+
Finalizer.Instance.Enable = oldState;
140+
}
141+
}
142+
}
143+
}

‎src/runtime/delegatemanager.cs‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses
181181
too "special" for this to work. It would be more work, so for now
182182
the 80/20 rule applies :) */
183183

184-
public class Dispatcher
184+
public class Dispatcher : IDisposable
185185
{
186186
public IntPtr target;
187187
public Type dtype;
188+
private bool _disposed = false;
189+
private bool _finalized = false;
188190

189191
public Dispatcher(IntPtr target, Type dtype)
190192
{
@@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype)
195197

196198
~Dispatcher()
197199
{
198-
// We needs to disable Finalizers until it's valid implementation.
199-
// Current implementation can produce low probability floating bugs.
200-
return;
200+
if (_finalized || _disposed)
201+
{
202+
return;
203+
}
204+
_finalized = true;
205+
Finalizer.Instance.AddFinalizedObject(this);
206+
}
201207

202-
// Note: the managed GC thread can run and try to free one of
203-
// these *after* the Python runtime has been finalized!
204-
if (Runtime.Py_IsInitialized() > 0)
208+
public void Dispose()
209+
{
210+
if (_disposed)
205211
{
206-
IntPtr gs = PythonEngine.AcquireLock();
207-
Runtime.XDecref(target);
208-
PythonEngine.ReleaseLock(gs);
212+
return;
209213
}
214+
_disposed = true;
215+
Runtime.XDecref(target);
216+
target = IntPtr.Zero;
217+
dtype = null;
218+
GC.SuppressFinalize(this);
210219
}
211220

212221
public object Dispatch(ArrayList args)

‎src/runtime/finalizer.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
using System;
2+
using System.Collections.Concurrent;
3+
using System.Collections.Generic;
4+
using System.Linq;
5+
using System.Runtime.InteropServices;
6+
using System.Threading;
7+
8+
namespace Python.Runtime
9+
{
10+
public class Finalizer
11+
{
12+
public class CollectArgs : EventArgs
13+
{
14+
public int ObjectCount { get; set; }
15+
}
16+
17+
public static readonly Finalizer Instance = new Finalizer();
18+
19+
public event EventHandler<CollectArgs> CollectOnce;
20+
21+
private ConcurrentQueue<IDisposable> _objQueue = new ConcurrentQueue<IDisposable>();
22+
23+
[UnmanagedFunctionPointer(CallingConvention.Cdecl)]
24+
private delegate int PedingCall(IntPtr arg);
25+
private PedingCall _collectAction;
26+
private bool _pending = false;
27+
private readonly object _collectingLock = new object();
28+
public int Threshold { get; set; }
29+
public bool Enable { get; set; }
30+
31+
private Finalizer()
32+
{
33+
Enable = true;
34+
Threshold = 200;
35+
_collectAction = OnCollect;
36+
}
37+
38+
public void CallPendingFinalizers()
39+
{
40+
if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId)
41+
{
42+
throw new Exception("PendingCall should execute in main Python thread");
43+
}
44+
Runtime.Py_MakePendingCalls();
45+
}
46+
47+
public List<WeakReference> GetCollectedObjects()
48+
{
49+
return _objQueue.Select(T => new WeakReference(T)).ToList();
50+
}
51+
52+
internal void AddFinalizedObject(IDisposable obj)
53+
{
54+
if (!Enable)
55+
{
56+
return;
57+
}
58+
if (Runtime.Py_IsInitialized() == 0)
59+
{
60+
// XXX: Memory will leak if a PyObject finalized after Python shutdown,
61+
// for avoiding that case, user should call GC.Collect manual before shutdown.
62+
return;
63+
}
64+
_objQueue.Enqueue(obj);
65+
GC.ReRegisterForFinalize(obj);
66+
if (_objQueue.Count >= Threshold)
67+
{
68+
Collect();
69+
}
70+
}
71+
72+
internal static void Shutdown()
73+
{
74+
Instance.DisposeAll();
75+
Instance.CallPendingFinalizers();
76+
Runtime.PyErr_Clear();
77+
}
78+
79+
private void Collect()
80+
{
81+
lock (_collectingLock)
82+
{
83+
if (_pending)
84+
{
85+
return;
86+
}
87+
_pending = true;
88+
}
89+
IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction);
90+
if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0)
91+
{
92+
// Full queue, append next time
93+
_pending = false;
94+
}
95+
}
96+
97+
private int OnCollect(IntPtr arg)
98+
{
99+
CollectOnce?.Invoke(this, new CollectArgs()
100+
{
101+
ObjectCount = _objQueue.Count
102+
});
103+
DisposeAll();
104+
_pending = false;
105+
return 0;
106+
}
107+
108+
private void DisposeAll()
109+
{
110+
IDisposable obj;
111+
while (_objQueue.TryDequeue(out obj))
112+
{
113+
obj.Dispose();
114+
}
115+
}
116+
}
117+
}

‎src/runtime/pyobject.cs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable
1717
{
1818
protected internal IntPtr obj = IntPtr.Zero;
1919
private bool disposed = false;
20+
private bool _finalized = false;
2021

2122
/// <summary>
2223
/// PyObject Constructor
@@ -41,14 +42,15 @@ protected PyObject()
4142

4243
// Ensure that encapsulated Python object is decref'ed appropriately
4344
// when the managed wrapper is garbage-collected.
44-
4545
~PyObject()
4646
{
47-
// We needs to disable Finalizers until it's valid implementation.
48-
// Current implementation can produce low probability floating bugs.
49-
return;
50-
51-
Dispose();
47+
if (_finalized || disposed)
48+
{
49+
return;
50+
}
51+
// Prevent a infinity loop by calling GC.WaitForPendingFinalizers
52+
_finalized = true;
53+
Finalizer.Instance.AddFinalizedObject(this);
5254
}
5355

5456

‎src/runtime/pyscope.cs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable
3737
internal readonly IntPtr variables;
3838

3939
private bool _isDisposed;
40+
private bool _finalized = false;
4041

4142
/// <summary>
4243
/// The Manager this scope associated with.
@@ -527,11 +528,12 @@ public void Dispose()
527528

528529
~PyScope()
529530
{
530-
// We needs to disable Finalizers until it's valid implementation.
531-
// Current implementation can produce low probability floating bugs.
532-
return;
533-
534-
Dispose();
531+
if (_finalized || _isDisposed)
532+
{
533+
return;
534+
}
535+
_finalized = true;
536+
Finalizer.Instance.AddFinalizedObject(this);
535537
}
536538
}
537539

0 commit comments

Comments
 (0)