Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ff86507

Browse files
lostmsulostmsu
committed
reworked Finalizer to only store raw python object handles instead of PyObject instances
1 parent d888244 commit ff86507

File tree

9 files changed

+88
-158
lines changed

9 files changed

+88
-158
lines changed

‎src/embed_tests/TestDomainReload.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ void ExecTest()
288288

289289
GC.Collect();
290290
GC.WaitForPendingFinalizers(); // <- this will put former `num` into Finalizer queue
291-
Finalizer.Instance.Collect(forceDispose: true);
291+
Finalizer.Instance.Collect();
292292
// ^- this will call PyObject.Dispose, which will call XDecref on `num.Handle`,
293293
// but Python interpreter from "run" 1 is long gone, so it will corrupt memory instead.
294294
Assert.False(numRef.IsAlive);
@@ -333,7 +333,7 @@ void ExecTest()
333333
PythonEngine.Initialize(); // <- "run" 2 starts
334334
GC.Collect();
335335
GC.WaitForPendingFinalizers();
336-
Finalizer.Instance.Collect(forceDispose: true);
336+
Finalizer.Instance.Collect();
337337
Assert.False(objRef.IsAlive);
338338
}
339339
finally

‎src/embed_tests/TestFinalizer.cs

Lines changed: 4 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,7 @@ public void CollectBasicObject()
5252
Assert.IsFalse(called, "The event handler was called before it was installed");
5353
Finalizer.Instance.CollectOnce += handler;
5454

55-
WeakReference shortWeak;
56-
WeakReference longWeak;
57-
{
58-
MakeAGarbage(out shortWeak, out longWeak);
59-
}
55+
IntPtr pyObj = MakeAGarbage(out var shortWeak, out var longWeak);
6056
FullGCCollect();
6157
// The object has been resurrected
6258
Warn.If(
@@ -74,7 +70,7 @@ public void CollectBasicObject()
7470
var garbage = Finalizer.Instance.GetCollectedObjects();
7571
Assert.NotZero(garbage.Count, "There should still be garbage around");
7672
Warn.Unless(
77-
garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)),
73+
garbage.Contains(pyObj),
7874
$"The {nameof(longWeak)} reference doesn't show up in the garbage list",
7975
garbage
8076
);
@@ -98,9 +94,9 @@ public void CollectOnShutdown()
9894
IntPtr op = MakeAGarbage(out var shortWeak, out var longWeak);
9995
FullGCCollect();
10096
Assert.IsFalse(shortWeak.IsAlive);
101-
List<WeakReference> garbage = Finalizer.Instance.GetCollectedObjects();
97+
List<IntPtr> garbage = Finalizer.Instance.GetCollectedObjects();
10298
Assert.IsNotEmpty(garbage, "The garbage object should be collected");
103-
Assert.IsTrue(garbage.Any(r => ReferenceEquals(r.Target, longWeak.Target)),
99+
Assert.IsTrue(garbage.Contains(op),
104100
"Garbage should contains the collected object");
105101

106102
PythonEngine.Shutdown();
@@ -191,63 +187,6 @@ public void SimpleTestMemory()
191187
}
192188
}
193189

194-
class MyPyObject : PyObject
195-
{
196-
public MyPyObject(IntPtr op) : base(op)
197-
{
198-
}
199-
200-
protected override void Dispose(bool disposing)
201-
{
202-
base.Dispose(disposing);
203-
GC.SuppressFinalize(this);
204-
throw new Exception("MyPyObject");
205-
}
206-
internal static void CreateMyPyObject(IntPtr op)
207-
{
208-
Runtime.Runtime.XIncref(op);
209-
new MyPyObject(op);
210-
}
211-
}
212-
213-
[Test]
214-
[Obsolete("GC tests are not guaranteed")]
215-
public void ErrorHandling()
216-
{
217-
bool called = false;
218-
var errorMessage = "";
219-
EventHandler<Finalizer.ErrorArgs> handleFunc = (sender, args) =>
220-
{
221-
called = true;
222-
errorMessage = args.Error.Message;
223-
};
224-
Finalizer.Instance.Threshold = 1;
225-
Finalizer.Instance.ErrorHandler += handleFunc;
226-
try
227-
{
228-
WeakReference shortWeak;
229-
WeakReference longWeak;
230-
{
231-
MakeAGarbage(out shortWeak, out longWeak);
232-
var obj = (PyObject)longWeak.Target;
233-
IntPtr handle = obj.Handle;
234-
shortWeak = null;
235-
longWeak = null;
236-
MyPyObject.CreateMyPyObject(handle);
237-
obj.Dispose();
238-
obj = null;
239-
}
240-
FullGCCollect();
241-
Finalizer.Instance.Collect();
242-
Assert.IsTrue(called);
243-
}
244-
finally
245-
{
246-
Finalizer.Instance.ErrorHandler -= handleFunc;
247-
}
248-
Assert.AreEqual(errorMessage, "MyPyObject");
249-
}
250-
251190
[Test]
252191
public void ValidateRefCount()
253192
{

‎src/runtime/debughelper.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,5 +137,12 @@ public static void PrintHexBytes(byte[] bytes)
137137
Console.WriteLine();
138138
}
139139
}
140+
141+
[Conditional("DEBUG")]
142+
public static void AssertHasReferences(IntPtr obj)
143+
{
144+
long refcount = Runtime.Refcount(obj);
145+
Debug.Assert(refcount > 0, "Object refcount is 0 or less");
146+
}
140147
}
141148
}

‎src/runtime/delegatemanager.cs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ A possible alternate strategy would be to create custom subclasses
181181
too "special" for this to work. It would be more work, so for now
182182
the 80/20 rule applies :) */
183183

184-
public class Dispatcher : IPyDisposable
184+
public class Dispatcher
185185
{
186186
public IntPtr target;
187187
public Type dtype;
@@ -202,7 +202,7 @@ public Dispatcher(IntPtr target, Type dtype)
202202
return;
203203
}
204204
_finalized = true;
205-
Finalizer.Instance.AddFinalizedObject(this);
205+
Finalizer.Instance.AddFinalizedObject(ref target);
206206
}
207207

208208
public void Dispose()
@@ -276,11 +276,6 @@ public object TrueDispatch(ArrayList args)
276276
Runtime.XDecref(op);
277277
return result;
278278
}
279-
280-
public IntPtr[] GetTrackedHandles()
281-
{
282-
return new IntPtr[] { target };
283-
}
284279
}
285280

286281

‎src/runtime/finalizer.cs

Lines changed: 58 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public class ErrorArgs : EventArgs
2727
public int Threshold { get; set; }
2828
public bool Enable { get; set; }
2929

30-
private ConcurrentQueue<IPyDisposable> _objQueue = new ConcurrentQueue<IPyDisposable>();
30+
private ConcurrentQueue<IntPtr> _objQueue = new ConcurrentQueue<IntPtr>();
3131
private int _throttled;
3232

3333
#region FINALIZER_CHECK
@@ -42,7 +42,7 @@ public class ErrorArgs : EventArgs
4242
public class IncorrectFinalizeArgs : EventArgs
4343
{
4444
public IntPtr Handle { get; internal set; }
45-
public ICollection<IPyDisposable> ImpactedObjects { get; internal set; }
45+
public ICollection<IntPtr> ImpactedObjects { get; internal set; }
4646
}
4747

4848
public class IncorrectRefCountException : Exception
@@ -73,8 +73,6 @@ private Finalizer()
7373
Threshold = 200;
7474
}
7575

76-
[Obsolete("forceDispose parameter is unused. All objects are disposed regardless.")]
77-
public void Collect(bool forceDispose) => this.DisposeAll();
7876
public void Collect() => this.DisposeAll();
7977

8078
internal void ThrottledCollect()
@@ -85,14 +83,14 @@ internal void ThrottledCollect()
8583
this.Collect();
8684
}
8785

88-
public List<WeakReference> GetCollectedObjects()
86+
internal List<IntPtr> GetCollectedObjects()
8987
{
90-
return _objQueue.Select(T => new WeakReference(T)).ToList();
88+
return _objQueue.ToList();
9189
}
9290

93-
internal void AddFinalizedObject(IPyDisposable obj)
91+
internal void AddFinalizedObject(ref IntPtr obj)
9492
{
95-
if (!Enable)
93+
if (!Enable || obj == IntPtr.Zero)
9694
{
9795
return;
9896
}
@@ -103,6 +101,7 @@ internal void AddFinalizedObject(IPyDisposable obj)
103101
{
104102
this._objQueue.Enqueue(obj);
105103
}
104+
obj = IntPtr.Zero;
106105
}
107106

108107
internal static void Shutdown()
@@ -123,29 +122,44 @@ private void DisposeAll()
123122
#if FINALIZER_CHECK
124123
ValidateRefCount();
125124
#endif
126-
IPyDisposable obj;
127-
while (_objQueue.TryDequeue(out obj))
125+
IntPtr obj;
126+
Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback);
127+
128+
try
128129
{
129-
try
130-
{
131-
obj.Dispose();
132-
}
133-
catch (Exception e)
130+
while (!_objQueue.IsEmpty)
134131
{
135-
var handler = ErrorHandler;
136-
if (handler is null)
132+
if (!_objQueue.TryDequeue(out obj))
133+
continue;
134+
135+
Runtime.XDecref(obj);
136+
try
137137
{
138-
throw new FinalizationException(
139-
"Python object finalization failed",
140-
disposable: obj, innerException: e);
138+
Runtime.CheckExceptionOccurred();
141139
}
142-
143-
handler.Invoke(this, new ErrorArgs()
140+
catch (Exception e)
144141
{
145-
Error = e
146-
});
142+
var handler = ErrorHandler;
143+
if (handler is null)
144+
{
145+
throw new FinalizationException(
146+
"Python object finalization failed",
147+
disposable: obj, innerException: e);
148+
}
149+
150+
handler.Invoke(this, new ErrorArgs()
151+
{
152+
Error = e
153+
});
154+
}
147155
}
148156
}
157+
finally
158+
{
159+
// Python requires finalizers to preserve exception:
160+
// https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation
161+
Runtime.PyErr_Restore(errType, errVal, traceback);
162+
}
149163
}
150164
}
151165

@@ -158,33 +172,26 @@ private void ValidateRefCount()
158172
}
159173
var counter = new Dictionary<IntPtr, long>();
160174
var holdRefs = new Dictionary<IntPtr, long>();
161-
var indexer = new Dictionary<IntPtr, List<IPyDisposable>>();
175+
var indexer = new Dictionary<IntPtr, List<IntPtr>>();
162176
foreach (var obj in _objQueue)
163177
{
164-
IntPtr[] handles = obj.GetTrackedHandles();
165-
foreach (var handle in handles)
178+
var handle = obj;
179+
if (!counter.ContainsKey(handle))
166180
{
167-
if (handle == IntPtr.Zero)
168-
{
169-
continue;
170-
}
171-
if (!counter.ContainsKey(handle))
172-
{
173-
counter[handle] = 0;
174-
}
175-
counter[handle]++;
176-
if (!holdRefs.ContainsKey(handle))
177-
{
178-
holdRefs[handle] = Runtime.Refcount(handle);
179-
}
180-
List<IPyDisposable> objs;
181-
if (!indexer.TryGetValue(handle, out objs))
182-
{
183-
objs = new List<IPyDisposable>();
184-
indexer.Add(handle, objs);
185-
}
186-
objs.Add(obj);
181+
counter[handle] = 0;
182+
}
183+
counter[handle]++;
184+
if (!holdRefs.ContainsKey(handle))
185+
{
186+
holdRefs[handle] = Runtime.Refcount(handle);
187+
}
188+
List<IntPtr> objs;
189+
if (!indexer.TryGetValue(handle, out objs))
190+
{
191+
objs = new List<IntPtr>();
192+
indexer.Add(handle, objs);
187193
}
194+
objs.Add(obj);
188195
}
189196
foreach (var pair in counter)
190197
{
@@ -227,12 +234,13 @@ private void ValidateRefCount()
227234

228235
public class FinalizationException : Exception
229236
{
230-
public IPyDisposable Disposable { get; }
237+
public IntPtr PythonObject { get; }
231238

232-
public FinalizationException(string message, IPyDisposable disposable, Exception innerException)
239+
public FinalizationException(string message, IntPtr disposable, Exception innerException)
233240
: base(message, innerException)
234241
{
235-
this.Disposable = disposable ?? throw new ArgumentNullException(nameof(disposable));
242+
if (disposable == IntPtr.Zero) throw new ArgumentNullException(nameof(disposable));
243+
this.PythonObject = disposable;
236244
}
237245
}
238246
}

‎src/runtime/pybuffer.cs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
namespace Python.Runtime
99
{
10-
public sealed class PyBuffer : IPyDisposable
10+
public sealed class PyBuffer : IDisposable
1111
{
1212
private PyObject _exporter;
1313
private Py_buffer _view;
@@ -236,7 +236,7 @@ private void Dispose(bool disposing)
236236
{
237237
return;
238238
}
239-
Finalizer.Instance.AddFinalizedObject(this);
239+
Finalizer.Instance.AddFinalizedObject(ref _view.obj);
240240
}
241241

242242
/// <summary>
@@ -248,10 +248,5 @@ public void Dispose()
248248
Dispose(true);
249249
GC.SuppressFinalize(this);
250250
}
251-
252-
public IntPtr[] GetTrackedHandles()
253-
{
254-
return new IntPtr[] { _view.obj };
255-
}
256251
}
257252
}

0 commit comments

Comments
 (0)