Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 88850f5

Browse files
lostmsulostmsu
committed
cleanup PyBuffer a bit:
- better finalizer, that actually calls PyBuffer_Release - improved input parameter handling for common routines - added support for copying data to/from large Python buffers fixes #1556
1 parent efad01c commit 88850f5

File tree

6 files changed

+160
-63
lines changed

6 files changed

+160
-63
lines changed

‎src/embed_tests/TestPyBuffer.cs

Lines changed: 74 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Runtime.CompilerServices;
23
using System.Text;
34
using NUnit.Framework;
45
using Python.Runtime;
@@ -24,48 +25,40 @@ public void Dispose()
2425
public void TestBufferWrite()
2526
{
2627
string bufferTestString = "hello world! !$%&/()=?";
28+
string bufferTestString2 = "h llo world! !$%&/()=?";
2729

28-
using (Py.GIL())
30+
using var _ = Py.GIL();
31+
32+
using var pythonArray = ByteArrayFromAsciiString(bufferTestString);
33+
34+
using (PyBuffer buf = pythonArray.GetBuffer(PyBUF.WRITABLE))
2935
{
30-
using (var scope = Py.CreateScope())
31-
{
32-
scope.Exec($"arr = bytearray({bufferTestString.Length})");
33-
PyObject pythonArray = scope.Get("arr");
34-
byte[] managedArray = new UTF8Encoding().GetBytes(bufferTestString);
35-
36-
using (PyBuffer buf = pythonArray.GetBuffer())
37-
{
38-
buf.Write(managedArray, 0, managedArray.Length);
39-
}
40-
41-
string result = scope.Eval("arr.decode('utf-8')").ToString();
42-
Assert.IsTrue(result == bufferTestString);
43-
}
36+
byte[] managedArray = { (byte)' ' };
37+
buf.Write(managedArray, 0, managedArray.Length, 1);
4438
}
39+
40+
string result = pythonArray.InvokeMethod("decode", "utf-8".ToPython()).As<string>();
41+
Assert.IsTrue(result == bufferTestString2);
4542
}
4643

4744
[Test]
4845
public void TestBufferRead()
4946
{
5047
string bufferTestString = "hello world! !$%&/()=?";
5148

52-
using (Py.GIL())
49+
using var _ = Py.GIL();
50+
51+
using var pythonArray = ByteArrayFromAsciiString(bufferTestString);
52+
byte[] managedArray = new byte[bufferTestString.Length];
53+
54+
using (PyBuffer buf = pythonArray.GetBuffer())
5355
{
54-
using (var scope = Py.CreateScope())
55-
{
56-
scope.Exec($"arr = b'{bufferTestString}'");
57-
PyObject pythonArray = scope.Get("arr");
58-
byte[] managedArray = new byte[bufferTestString.Length];
59-
60-
using (PyBuffer buf = pythonArray.GetBuffer())
61-
{
62-
buf.Read(managedArray, 0, managedArray.Length);
63-
}
64-
65-
string result = new UTF8Encoding().GetString(managedArray);
66-
Assert.IsTrue(result == bufferTestString);
67-
}
56+
managedArray[0] = (byte)' ';
57+
buf.Read(managedArray, 1, managedArray.Length - 1, 1);
6858
}
59+
60+
string result = new UTF8Encoding().GetString(managedArray);
61+
Assert.IsTrue(result == " " + bufferTestString.Substring(1));
6962
}
7063

7164
[Test]
@@ -77,5 +70,56 @@ public void ArrayHasBuffer()
7770
Assert.AreEqual(1, mem[(0, 0).ToPython()].As<int>());
7871
Assert.AreEqual(array[1,0], mem[(1, 0).ToPython()].As<int>());
7972
}
73+
74+
[Test]
75+
public void RefCount()
76+
{
77+
using var _ = Py.GIL();
78+
using var arr = ByteArrayFromAsciiString("hello world! !$%&/()=?");
79+
80+
Assert.AreEqual(1, arr.Refcount);
81+
82+
using (PyBuffer buf = arr.GetBuffer())
83+
{
84+
Assert.AreEqual(2, arr.Refcount);
85+
}
86+
87+
Assert.AreEqual(1, arr.Refcount);
88+
}
89+
90+
[Test]
91+
public void Finalization()
92+
{
93+
if (Type.GetType("Mono.Runtime") is not null)
94+
{
95+
Assert.Inconclusive("test unreliable in Mono");
96+
return;
97+
}
98+
99+
using var _ = Py.GIL();
100+
using var arr = ByteArrayFromAsciiString("hello world! !$%&/()=?");
101+
102+
Assert.AreEqual(1, arr.Refcount);
103+
104+
MakeBufAndLeak(arr);
105+
106+
GC.Collect();
107+
GC.WaitForPendingFinalizers();
108+
Finalizer.Instance.Collect();
109+
110+
Assert.AreEqual(1, arr.Refcount);
111+
}
112+
113+
[MethodImpl(MethodImplOptions.NoInlining)]
114+
static void MakeBufAndLeak(PyObject bufProvider)
115+
{
116+
PyBuffer buf = bufProvider.GetBuffer();
117+
}
118+
119+
static PyObject ByteArrayFromAsciiString(string str)
120+
{
121+
using var scope = Py.CreateScope();
122+
return Runtime.Runtime.PyByteArray_FromStringAndSize(str).MoveToPyObject();
123+
}
80124
}
81125
}

‎src/runtime/bufferinterface.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,11 @@ namespace Python.Runtime
88
internal struct Py_buffer {
99
public IntPtr buf;
1010
public IntPtr obj; /* owned reference */
11+
/// <summary>Buffer size in bytes</summary>
1112
[MarshalAs(UnmanagedType.SysInt)]
12-
public IntPtr len;
13+
public nint len;
1314
[MarshalAs(UnmanagedType.SysInt)]
14-
public IntPtr itemsize; /* This is Py_ssize_t so it can be
15+
public nint itemsize; /* This is Py_ssize_t so it can be
1516
pointed to by strides in simple case.*/
1617
[MarshalAs(UnmanagedType.Bool)]
1718
public bool _readonly;

‎src/runtime/finalizer.cs

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public ErrorArgs(Exception error)
4343

4444
private ConcurrentQueue<PendingFinalization> _objQueue = new();
4545
private readonly ConcurrentQueue<PendingFinalization> _derivedQueue = new();
46+
private readonly ConcurrentQueue<Py_buffer> _bufferQueue = new();
4647
private int _throttled;
4748

4849
#region FINALIZER_CHECK
@@ -165,6 +166,19 @@ internal void AddDerivedFinalizedObject(ref IntPtr derived, int run)
165166
_derivedQueue.Enqueue(pending);
166167
}
167168

169+
internal void AddFinalizedBuffer(ref Py_buffer buffer)
170+
{
171+
if (buffer.obj == IntPtr.Zero)
172+
throw new ArgumentNullException(nameof(buffer));
173+
174+
if (!Enable)
175+
return;
176+
177+
var pending = buffer;
178+
buffer = default;
179+
_bufferQueue.Enqueue(pending);
180+
}
181+
168182
internal static void Initialize()
169183
{
170184
Instance.started = true;
@@ -178,7 +192,7 @@ internal static void Shutdown()
178192

179193
internal nint DisposeAll()
180194
{
181-
if (_objQueue.IsEmpty && _derivedQueue.IsEmpty)
195+
if (_objQueue.IsEmpty && _derivedQueue.IsEmpty && _bufferQueue.IsEmpty)
182196
return 0;
183197

184198
nint collected = 0;
@@ -242,6 +256,15 @@ internal nint DisposeAll()
242256

243257
collected++;
244258
}
259+
260+
while (!_bufferQueue.IsEmpty)
261+
{
262+
if (!_bufferQueue.TryDequeue(out var buffer))
263+
continue;
264+
265+
Runtime.PyBuffer_Release(ref buffer);
266+
collected++;
267+
}
245268
}
246269
finally
247270
{

‎src/runtime/pybuffer.cs

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,9 @@ public static long SizeFromFormat(string format)
8989
{
9090
if (Runtime.PyVersion < new Version(3,9))
9191
throw new NotSupportedException("SizeFromFormat requires at least Python 3.9");
92-
return (long)Runtime.PyBuffer_SizeFromFormat(format);
92+
nint result = Runtime.PyBuffer_SizeFromFormat(format);
93+
if (result == -1) throw PythonException.ThrowLastAsClrException();
94+
return result;
9395
}
9496

9597
/// <summary>
@@ -113,7 +115,7 @@ public IntPtr GetPointer(long[] indices)
113115
throw new ObjectDisposedException(nameof(PyBuffer));
114116
if (Runtime.PyVersion < new Version(3, 7))
115117
throw new NotSupportedException("GetPointer requires at least Python 3.7");
116-
return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => (IntPtr)x).ToArray());
118+
return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => checked((nint)x)).ToArray());
117119
}
118120

119121
/// <summary>
@@ -126,7 +128,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort)
126128
if (Runtime.PyVersion < new Version(3, 7))
127129
throw new NotSupportedException("FromContiguous requires at least Python 3.7");
128130

129-
if (Runtime.PyBuffer_FromContiguous(ref _view, buf, (IntPtr)len, OrderStyleToChar(fort, false)) < 0)
131+
if (Runtime.PyBuffer_FromContiguous(ref _view, buf, checked((nint)len), OrderStyleToChar(fort, false)) < 0)
130132
throw PythonException.ThrowLastAsClrException();
131133
}
132134

@@ -173,44 +175,60 @@ internal void FillInfo(BorrowedReference exporter, IntPtr buf, long len, bool _r
173175
/// <summary>
174176
/// Writes a managed byte array into the buffer of a python object. This can be used to pass data like images from managed to python.
175177
/// </summary>
176-
public void Write(byte[] buffer, int offset, int count)
178+
public void Write(byte[] buffer, int sourceOffset, int count, nint destinationOffset)
177179
{
178180
if (disposedValue)
179181
throw new ObjectDisposedException(nameof(PyBuffer));
182+
if (_view.ndim != 1)
183+
throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
184+
if (!this.IsContiguous(BufferOrderStyle.C))
185+
throw new NotImplementedException("Only continuous buffers are supported");
180186
if (ReadOnly)
181187
throw new InvalidOperationException("Buffer is read-only");
182-
if ((long)_view.len > int.MaxValue)
183-
throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported.");
184-
if (count > buffer.Length)
188+
if (buffer is null)
189+
throw new ArgumentNullException(nameof(buffer));
190+
191+
if (sourceOffset < 0)
192+
throw new IndexOutOfRangeException($"{nameof(sourceOffset)} is negative");
193+
if (destinationOffset < 0)
194+
throw new IndexOutOfRangeException($"{nameof(destinationOffset)} is negative");
195+
if (count < 0)
196+
throw new ArgumentOutOfRangeException(nameof(count), count, "Value must be >= 0");
197+
198+
if (checked(count + sourceOffset) > buffer.Length)
185199
throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer.");
186-
if (count > (int)_view.len)
200+
if (checked(count + destinationOffset) > _view.len)
187201
throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer.");
188-
if (_view.ndim != 1)
189-
throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
190-
if (!this.IsContiguous(BufferOrderStyle.C))
191-
throw new NotImplementedException("Only continuous buffers are supported");
192202

193-
Marshal.Copy(buffer, offset, _view.buf, count);
203+
Marshal.Copy(buffer, sourceOffset, _view.buf + destinationOffset, count);
194204
}
195205

196206
/// <summary>
197207
/// Reads the buffer of a python object into a managed byte array. This can be used to pass data like images from python to managed.
198208
/// </summary>
199-
public int Read(byte[] buffer, int offset, int count) {
209+
public void Read(byte[] buffer, int destinationOffset, int count, nint sourceOffset) {
200210
if (disposedValue)
201211
throw new ObjectDisposedException(nameof(PyBuffer));
202-
if (count > buffer.Length)
203-
throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer.");
204212
if (_view.ndim != 1)
205-
throw new NotSupportedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
206-
if (_view.len.ToInt64() > int.MaxValue)
207-
throw new NotSupportedException("Python buffers bigger than int.MaxValue are not supported.");
213+
throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported.");
208214
if (!this.IsContiguous(BufferOrderStyle.C))
209215
throw new NotImplementedException("Only continuous buffers are supported");
216+
if (buffer is null)
217+
throw new ArgumentNullException(nameof(buffer));
218+
219+
if (sourceOffset < 0)
220+
throw new IndexOutOfRangeException($"{nameof(sourceOffset)} is negative");
221+
if (destinationOffset < 0)
222+
throw new IndexOutOfRangeException($"{nameof(destinationOffset)} is negative");
223+
if (count < 0)
224+
throw new ArgumentOutOfRangeException(nameof(count), count, "Value must be >= 0");
225+
226+
if (checked(count + destinationOffset) > buffer.Length)
227+
throw new ArgumentOutOfRangeException("count", "Count is bigger than the buffer.");
228+
if (checked(count + sourceOffset) > _view.len)
229+
throw new ArgumentOutOfRangeException("count", "Count is bigger than the python buffer.");
210230

211-
int copylen = count < (int)_view.len ? count : (int)_view.len;
212-
Marshal.Copy(_view.buf, buffer, offset, copylen);
213-
return copylen;
231+
Marshal.Copy(_view.buf + sourceOffset, buffer, destinationOffset, count);
214232
}
215233

216234
private bool disposedValue = false; // To detect redundant calls
@@ -240,11 +258,7 @@ private void Dispose(bool disposing)
240258

241259
if (_view.obj != IntPtr.Zero)
242260
{
243-
Finalizer.Instance.AddFinalizedObject(ref _view.obj, _exporter.run
244-
#if TRACE_ALLOC
245-
, _exporter.Traceback
246-
#endif
247-
);
261+
Finalizer.Instance.AddFinalizedBuffer(ref _view);
248262
}
249263

250264
Dispose(false);

‎src/runtime/pyobject.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,11 @@ public static PyObject FromManagedObject(object ob)
179179

180180
internal bool IsDisposed => rawPtr == IntPtr.Zero;
181181

182+
void CheckDisposed()
183+
{
184+
if (IsDisposed) throw new ObjectDisposedException(nameof(PyObject));
185+
}
186+
182187
protected virtual void Dispose(bool disposing)
183188
{
184189
if (IsDisposed)
@@ -1114,6 +1119,7 @@ public override int GetHashCode()
11141119
/// </remarks>
11151120
public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE)
11161121
{
1122+
CheckDisposed();
11171123
return new PyBuffer(this, flags);
11181124
}
11191125

‎src/runtime/runtime.cs

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1107,7 +1107,7 @@ internal static nint PyBuffer_SizeFromFormat(string format)
11071107
internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order);
11081108

11091109

1110-
internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, IntPtr[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices);
1110+
internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, nint[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices);
11111111

11121112

11131113
internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort);
@@ -1362,6 +1362,13 @@ internal static NewReference EmptyPyBytes()
13621362
return Delegates.PyBytes_FromString((IntPtr)bytes);
13631363
}
13641364

1365+
internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len);
1366+
internal static NewReference PyByteArray_FromStringAndSize(string s)
1367+
{
1368+
using var ptr = new StrPtr(s, Encoding.UTF8);
1369+
return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount));
1370+
}
1371+
13651372
internal static IntPtr PyBytes_AsString(BorrowedReference ob)
13661373
{
13671374
Debug.Assert(ob != null);
@@ -1977,7 +1984,7 @@ static Delegates()
19771984
// only in 3.9+
19781985
}
19791986
PyBuffer_IsContiguous = (delegate* unmanaged[Cdecl]<ref Py_buffer, char, int>)GetFunctionByName(nameof(PyBuffer_IsContiguous), GetUnmanagedDll(_PythonDll));
1980-
PyBuffer_GetPointer = (delegate* unmanaged[Cdecl]<ref Py_buffer, IntPtr[], IntPtr>)GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll));
1987+
PyBuffer_GetPointer = (delegate* unmanaged[Cdecl]<ref Py_buffer, nint[], IntPtr>)GetFunctionByName(nameof(PyBuffer_GetPointer), GetUnmanagedDll(_PythonDll));
19811988
PyBuffer_FromContiguous = (delegate* unmanaged[Cdecl]<ref Py_buffer, IntPtr, IntPtr, char, int>)GetFunctionByName(nameof(PyBuffer_FromContiguous), GetUnmanagedDll(_PythonDll));
19821989
PyBuffer_ToContiguous = (delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, IntPtr, char, int>)GetFunctionByName(nameof(PyBuffer_ToContiguous), GetUnmanagedDll(_PythonDll));
19831990
PyBuffer_FillContiguousStrides = (delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, int, char, void>)GetFunctionByName(nameof(PyBuffer_FillContiguousStrides), GetUnmanagedDll(_PythonDll));
@@ -2037,6 +2044,7 @@ static Delegates()
20372044
PySequence_List = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PySequence_List), GetUnmanagedDll(_PythonDll));
20382045
PyBytes_AsString = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr>)GetFunctionByName(nameof(PyBytes_AsString), GetUnmanagedDll(_PythonDll));
20392046
PyBytes_FromString = (delegate* unmanaged[Cdecl]<IntPtr, NewReference>)GetFunctionByName(nameof(PyBytes_FromString), GetUnmanagedDll(_PythonDll));
2047+
PyByteArray_FromStringAndSize = (delegate* unmanaged[Cdecl]<IntPtr, nint, NewReference>)GetFunctionByName(nameof(PyByteArray_FromStringAndSize), GetUnmanagedDll(_PythonDll));
20402048
PyBytes_Size = (delegate* unmanaged[Cdecl]<BorrowedReference, nint>)GetFunctionByName(nameof(PyBytes_Size), GetUnmanagedDll(_PythonDll));
20412049
PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr>)GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll));
20422050
PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl]<IntPtr, nint, IntPtr, IntPtr, NewReference>)GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll));
@@ -2250,7 +2258,7 @@ static Delegates()
22502258
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, void> PyBuffer_Release { get; }
22512259
internal static delegate* unmanaged[Cdecl]<StrPtr, nint> PyBuffer_SizeFromFormat { get; }
22522260
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, char, int> PyBuffer_IsContiguous { get; }
2253-
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, IntPtr[], IntPtr> PyBuffer_GetPointer { get; }
2261+
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, nint[], IntPtr> PyBuffer_GetPointer { get; }
22542262
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, IntPtr, IntPtr, char, int> PyBuffer_FromContiguous { get; }
22552263
internal static delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, IntPtr, char, int> PyBuffer_ToContiguous { get; }
22562264
internal static delegate* unmanaged[Cdecl]<int, IntPtr, IntPtr, int, char, void> PyBuffer_FillContiguousStrides { get; }
@@ -2310,6 +2318,7 @@ static Delegates()
23102318
internal static delegate* unmanaged[Cdecl]<BorrowedReference, NewReference> PySequence_List { get; }
23112319
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr> PyBytes_AsString { get; }
23122320
internal static delegate* unmanaged[Cdecl]<IntPtr, NewReference> PyBytes_FromString { get; }
2321+
internal static delegate* unmanaged[Cdecl]<IntPtr, nint, NewReference> PyByteArray_FromStringAndSize { get; }
23132322
internal static delegate* unmanaged[Cdecl]<BorrowedReference, nint> PyBytes_Size { get; }
23142323
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr> PyUnicode_AsUTF8 { get; }
23152324
internal static delegate* unmanaged[Cdecl]<IntPtr, nint, IntPtr, IntPtr, NewReference> PyUnicode_DecodeUTF16 { get; }

0 commit comments

Comments
 (0)