From 9c3431b0ad489b1a28a25f979c3bd39447b6ea06 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 3 Sep 2017 13:35:56 +0200 Subject: [PATCH 01/23] Add queue.SimpleQueue type --- Lib/queue.py | 18 +++++ Lib/test/test_queue.py | 149 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 163 insertions(+), 4 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 572425e844c52e..1a46b4d627f6b3 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -244,3 +244,21 @@ def _put(self, item): def _get(self): return self.queue.pop() + + +class SimpleQueue: + + def __init__(self): + self._queue = deque() + self._count = threading.Semaphore(0) + + def put(self, item): + self._queue.append(item) + self._count.release() + + def get(self, block=True, timeout=None): + if timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + if not self._count.acquire(block, timeout): + raise Empty + return self._queue.popleft() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 4ccaa39adff69f..b4673e2f3a629a 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -1,6 +1,8 @@ # Some simple queue module tests, plus some failure conditions # to ensure the Queue locks remain stable. +import collections import queue +import random import time import unittest from test import support @@ -90,7 +92,7 @@ def setUp(self): self.cum = 0 self.cumlock = threading.Lock() - def simple_queue_test(self, q): + def basic_queue_test(self, q): if q.qsize(): raise RuntimeError("Call this function with an empty queue") self.assertTrue(q.empty()) @@ -193,12 +195,12 @@ def test_queue_join(self): else: self.fail("Did not detect task count going negative") - def test_simple_queue(self): + def test_basic(self): # Do it a couple of times on the same queue. # Done twice to make sure works with same instance reused. q = self.type2test(QUEUE_SIZE) - self.simple_queue_test(q) - self.simple_queue_test(q) + self.basic_queue_test(q) + self.basic_queue_test(q) def test_negative_timeout_raises_exception(self): q = self.type2test(QUEUE_SIZE) @@ -354,5 +356,144 @@ def test_failing_queue(self): self.failing_queue_test(q) +class SimpleQueueTest(unittest.TestCase): + type2test = queue.SimpleQueue + + def setUp(self): + self.q = self.type2test() + + def feed(self, q, seq, rnd): + while True: + try: + val = seq.pop() + except IndexError: + return + q.put(val) + if rnd.random() > 0.5: + time.sleep(rnd.random() * 1e-3) + + def consume(self, q, results, sentinel): + while True: + val = q.get() + if val == sentinel: + return + results.append(val) + + def consume_nonblock(self, q, results, sentinel): + while True: + while True: + try: + val = q.get(block=False) + except queue.Empty: + time.sleep(1e-5) + else: + break + if val == sentinel: + return + results.append(val) + + def consume_timeout(self, q, results, sentinel): + while True: + while True: + try: + val = q.get(timeout=1e-5) + except queue.Empty: + pass + else: + break + if val == sentinel: + return + results.append(val) + + def run_threads(self, n_feeders, n_consumers, q, inputs, + feed_func, consume_func): + results = [] + sentinel = None + seq = inputs + [sentinel] * n_consumers + seq.reverse() + rnd = random.Random(42) + + exceptions = [] + def log_exceptions(f): + def wrapper(*args, **kwargs): + try: + f(*args, **kwargs) + except BaseException as e: + exceptions.append(e) + return wrapper + + feeders = [threading.Thread(target=log_exceptions(feed_func), + args=(q, seq, rnd)) + for i in range(n_feeders)] + consumers = [threading.Thread(target=log_exceptions(consume_func), + args=(q, results, sentinel)) + for i in range(n_consumers)] + + with support.start_threads(feeders + consumers): + pass + + self.assertFalse(exceptions) + + return results + + def test_simple(self): + q = self.q + q.put(1) + q.put(2) + q.put(3) + q.put(4) + self.assertEqual(q.get(), 1) + self.assertEqual(q.get(), 2) + self.assertEqual(q.get(block=False), 3) + self.assertEqual(q.get(timeout=0.1), 4) + with self.assertRaises(queue.Empty): + q.get(block=False) + with self.assertRaises(queue.Empty): + q.get(timeout=1e-3) + + def test_negative_timeout_raises_exception(self): + q = self.q + q.put(1) + with self.assertRaises(ValueError): + q.get(timeout=-1) + + def test_order(self): + N = 3 + q = self.q + inputs = list(range(100)) + results = self.run_threads(N, 1, q, inputs, self.feed, self.consume) + + # One consumer => results appended in well-defined order + self.assertEqual(results, inputs) + + def test_many_threads(self): + N = 50 + q = self.q + inputs = list(range(10000)) + results = self.run_threads(N, N, q, inputs, self.feed, self.consume) + + # Multiple consumers without synchronization append the + # results in random order + self.assertEqual(sorted(results), inputs) + + def test_many_threads_nonblock(self): + N = 50 + q = self.q + inputs = list(range(10000)) + results = self.run_threads(N, N, q, inputs, + self.feed, self.consume_nonblock) + + self.assertEqual(sorted(results), inputs) + + def test_many_threads_timeout(self): + N = 50 + q = self.q + inputs = list(range(10000)) + results = self.run_threads(N, N, q, inputs, + self.feed, self.consume_timeout) + + self.assertEqual(sorted(results), inputs) + + if __name__ == "__main__": unittest.main() From 5d2f5ccf09f8fb0b0db44208bb478594f2771bd4 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 4 Sep 2017 12:28:25 +0200 Subject: [PATCH 02/23] Add C version of SimpleQueue --- Lib/queue.py | 24 +- Lib/test/test_queue.py | 25 ++- Modules/_queuemodule.c | 379 ++++++++++++++++++++++++++++++++ Modules/clinic/_queuemodule.c.h | 94 ++++++++ setup.py | 2 + 5 files changed, 516 insertions(+), 8 deletions(-) create mode 100644 Modules/_queuemodule.c create mode 100644 Modules/clinic/_queuemodule.c.h diff --git a/Lib/queue.py b/Lib/queue.py index 1a46b4d627f6b3..038179c0b1d133 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -8,16 +8,26 @@ from heapq import heappush, heappop from time import monotonic as time +try: + import _queue +except ImportError: + _queue = None + __all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue'] -class Empty(Exception): - 'Exception raised by Queue.get(block=0)/get_nowait().' - pass + +try: + Empty = _queue.Empty +except AttributeError: + class Empty(Exception): + 'Exception raised by Queue.get(block=0)/get_nowait().' + pass class Full(Exception): 'Exception raised by Queue.put(block=0)/put_nowait().' pass + class Queue: '''Create a queue object with a given maximum size. @@ -246,7 +256,7 @@ def _get(self): return self.queue.pop() -class SimpleQueue: +class _PySimpleQueue: def __init__(self): self._queue = deque() @@ -262,3 +272,9 @@ def get(self, block=True, timeout=None): if not self._count.acquire(block, timeout): raise Empty return self._queue.popleft() + + +try: + SimpleQueue = _queue.SimpleQueue +except AttributeError: + SimpleQueue = _PySimpleQueue diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index b4673e2f3a629a..f9e8d123999d1f 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -8,6 +8,11 @@ from test import support threading = support.import_module('threading') +try: + import _queue +except ImportError: + _queue = None + QUEUE_SIZE = 5 def qfull(q): @@ -356,8 +361,7 @@ def test_failing_queue(self): self.failing_queue_test(q) -class SimpleQueueTest(unittest.TestCase): - type2test = queue.SimpleQueue +class BaseSimpleQueueTest: def setUp(self): self.q = self.type2test() @@ -436,7 +440,7 @@ def wrapper(*args, **kwargs): return results - def test_simple(self): + def test_basic(self): q = self.q q.put(1) q.put(2) @@ -488,12 +492,25 @@ def test_many_threads_nonblock(self): def test_many_threads_timeout(self): N = 50 q = self.q - inputs = list(range(10000)) + inputs = list(range(1000)) results = self.run_threads(N, N, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) +class PySimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): + type2test = queue._PySimpleQueue + + +if _queue is not None: + + class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): + type2test = _queue.SimpleQueue + + def test_is_default(self): + self.assertIs(self.type2test, queue.SimpleQueue) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c new file mode 100644 index 00000000000000..1746e1aa7290af --- /dev/null +++ b/Modules/_queuemodule.c @@ -0,0 +1,379 @@ + +/* Thread module */ +/* Interface to Sjoerd's portable C thread library */ + +#include "Python.h" +#include "structmember.h" /* offsetof */ + +#ifndef WITH_THREAD +#error "Error! The rest of Python is not compiled with thread support." +#error "Rerun configure, adding a --with-threads option." +#error "Then run `make clean' followed by `make'." +#endif + +#include "pythread.h" + +/*[clinic input] +module _queue +class _queue.SimpleQueue "simplequeueobject *" "&PySimpleQueueType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=cf49af81bcbbbea6]*/ + +extern PyTypeObject PySimpleQueueType; /* forward decl */ + +static PyObject *EmptyError; + +static _PyTime_t infinite_timeout_val; + + +typedef struct { + PyObject_HEAD + PyThread_type_lock lock; + int locked; + PyObject *lst; + Py_ssize_t lst_pos; + PyObject *weakreflist; +} simplequeueobject; + + +static void +simplequeue_dealloc(simplequeueobject *self) +{ + _PyObject_GC_UNTRACK(self); + if (self->lock != NULL) { + /* Unlock the lock so it's safe to free it */ + if (self->locked > 0) + PyThread_release_lock(self->lock); + PyThread_free_lock(self->lock); + } + Py_XDECREF(self->lst); + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_TYPE(self)->tp_free(self); +} + +static int +simplequeue_traverse(simplequeueobject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->lst); + return 0; +} + +static PyObject * +simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + simplequeueobject *self; + + self = (simplequeueobject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->weakreflist = NULL; + self->lst = PyList_New(0); + self->lock = PyThread_allocate_lock(); + self->lst_pos = 0; + if (self->lock == NULL) { + Py_DECREF(self); + PyErr_SetString(PyExc_MemoryError, "can't allocate lock"); + return NULL; + } + if (self->lst == NULL) { + Py_DECREF(self); + return NULL; + } + } + + return (PyObject *) self; +} + +/*[clinic input] +_queue.SimpleQueue.__init__ + +Simple reentrant queue. +[clinic start generated code]*/ + +static int +_queue_SimpleQueue___init___impl(simplequeueobject *self) +/*[clinic end generated code: output=111d7b9a0c26aa23 input=fb09ddd81b016cea]*/ +{ + return 0; +} + +/*[clinic input] +_queue.SimpleQueue.put + item: object + +Put the item on the queue, without blocking. + +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item) +/*[clinic end generated code: output=5b502db806a97f16 input=fbf18e55e43edea9]*/ +{ +// fprintf(stderr, +// "PUT [%lu] (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + /* BEGIN GIL-protected critical section */ + if (PyList_Append(self->lst, item) < 0) + return NULL; + if (self->locked) { + /* A get() may be waiting, wake it up */ + self->locked = 0; + PyThread_release_lock(self->lock); +// fprintf(stderr, +// "PUT [%lu] release lock (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + } + /* END GIL-protected critical section */ +// fprintf(stderr, +// "/PUT [%lu]\n", +// PyThread_get_thread_ident()); + Py_RETURN_NONE; +} + +static PyObject * +simplequeue_pop_item(simplequeueobject *self) +{ + Py_ssize_t count, n; + PyObject *item; + + n = PyList_GET_SIZE(self->lst); + assert(self->lst_pos < n); + + item = PyList_GET_ITEM(self->lst, self->lst_pos); + Py_INCREF(Py_None); + PyList_SET_ITEM(self->lst, self->lst_pos, Py_None); + self->lst_pos += 1; + count = n - self->lst_pos; + if (self->lst_pos > count) { + /* reclaim space at beginning of list */ + if (PyList_SetSlice(self->lst, 0, self->lst_pos, NULL)) { + self->lst_pos -= 1; + PyList_SET_ITEM(self->lst, self->lst_pos, item); + return NULL; + } + self->lst_pos = 0; + } + return item; +} + +/*[clinic input] +_queue.SimpleQueue.get + block: bool = True + timeout: object = None + +XXX + +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_get_impl(simplequeueobject *self, int block, + PyObject *timeout) +/*[clinic end generated code: output=ec82a7157dcccd1a input=08e407fb75711d10]*/ +{ + _PyTime_t endtime = 0; + _PyTime_t timeout_val; + PyObject *item; + PyLockStatus r; + PY_TIMEOUT_T microseconds; + + if (block == 0) { + /* Non-blocking */ + microseconds = 0; + } + else if (timeout != Py_None) { + /* With timeout */ + if (_PyTime_FromSecondsObject(&timeout_val, + timeout, _PyTime_ROUND_CEILING) < 0) + return NULL; + if (timeout_val < 0) { + PyErr_SetString(PyExc_ValueError, + "'timeout' must be a non-negative number"); + return NULL; + } + microseconds = _PyTime_AsMicroseconds(timeout_val, + _PyTime_ROUND_CEILING); + if (microseconds >= PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "timeout value is too large"); + return NULL; + } + endtime = _PyTime_GetMonotonicClock() + timeout_val; + } + else { + /* Infinitely blocking */ + microseconds = -1; + } + +// fprintf(stderr, +// "GET [%lu] (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + if (self->lst_pos < PyList_GET_SIZE(self->lst)) { + /* Fast path */ + assert(!self->locked); + /* BEGIN GIL-protected critical section */ + item = simplequeue_pop_item(self); + if (item == NULL) { + return NULL; + } + /* END GIL-protected critical section */ + assert(!self->locked); +// fprintf(stderr, +// "/GET [%lu] fast (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + return item; + } + + /* put() signals the queue to be non-empty by releasing the lock. + * So we simply try to acquire the lock in a loop, until the condition + * (queue non-empty) becomes true. + */ + while (self->lst_pos == PyList_GET_SIZE(self->lst)) { +// fprintf(stderr, +// "GET [%lu] acquire lock 2 (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + + /* First a simple non-blocking try without releasing the GIL */ + r = PyThread_acquire_lock_timed(self->lock, 0, 0); + if (r == PY_LOCK_FAILURE && microseconds != 0) { + Py_BEGIN_ALLOW_THREADS + r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); + Py_END_ALLOW_THREADS + } + if (r == PY_LOCK_INTR) { + return NULL; + } + if (r == PY_LOCK_FAILURE) { + /* Timed out */ + PyErr_SetNone(EmptyError); + return NULL; + } + self->locked = 1; + /* Adjust timeout for next iteration (if any) */ + if (endtime > 0) { + timeout_val = endtime - _PyTime_GetMonotonicClock(); + microseconds = _PyTime_AsMicroseconds(timeout_val, _PyTime_ROUND_CEILING); + } + } + /* BEGIN GIL-protected critical section */ + assert(self->lst_pos < PyList_GET_SIZE(self->lst)); + item = simplequeue_pop_item(self); + if (self->locked) { +// fprintf(stderr, +// "GET [%lu] release lock (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + PyThread_release_lock(self->lock); + self->locked = 0; + } + /* END GIL-protected critical section */ +// fprintf(stderr, +// "/GET [%lu] slow (lst_pos = %zd, count = %zd, locked = %d)\n", +// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); + + return item; +} + +#include "clinic/_queuemodule.c.h" + + +static PyMethodDef simplequeue_methods[] = { + _QUEUE_SIMPLEQUEUE_GET_METHODDEF + _QUEUE_SIMPLEQUEUE_PUT_METHODDEF + /* XXX implement __sizeof__ */ + {NULL, NULL} /* sentinel */ +}; + + +PyTypeObject PySimpleQueueType = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "_queue.SimpleQueue", /*tp_name*/ + sizeof(simplequeueobject), /*tp_size*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)simplequeue_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + _queue_SimpleQueue___init____doc__, /*tp_doc*/ + (traverseproc)simplequeue_traverse, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + offsetof(simplequeueobject, weakreflist), /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + simplequeue_methods, /*tp_methods*/ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + _queue_SimpleQueue___init__, /* tp_init */ + 0, /* tp_alloc */ + simplequeue_new /* tp_new */ +}; + + +/* Initialization function */ + +PyDoc_STRVAR(queue_module_doc, +"XXX \n\ +This module provides primitive operations to write multi-threaded programs.\n\ +The 'threading' module provides a more convenient interface."); + +static struct PyModuleDef queuemodule = { + PyModuleDef_HEAD_INIT, + "_queue", + queue_module_doc, + -1, + NULL, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__queue(void) +{ + PyObject *m; + + /* Create the module */ + m = PyModule_Create(&queuemodule); + if (m == NULL) + return NULL; + + EmptyError = PyErr_NewExceptionWithDoc( + "_queue.Empty", + "Exception raised by Queue.get(block=0)/get_nowait().", + NULL, NULL); + if (EmptyError == NULL) + return NULL; + + Py_INCREF(EmptyError); + if (PyModule_AddObject(m, "Empty", EmptyError) < 0) + return NULL; + + if (PyType_Ready(&PySimpleQueueType) < 0) + return NULL; + Py_INCREF(&PySimpleQueueType); + if (PyModule_AddObject(m, "SimpleQueue", (PyObject *)&PySimpleQueueType) < 0) + return NULL; + + infinite_timeout_val = _PyTime_FromSeconds(-1); + return m; +} diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h new file mode 100644 index 00000000000000..5cbe7255eb0988 --- /dev/null +++ b/Modules/clinic/_queuemodule.c.h @@ -0,0 +1,94 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_queue_SimpleQueue___init____doc__, +"SimpleQueue()\n" +"--\n" +"\n" +"Simple reentrant queue."); + +static int +_queue_SimpleQueue___init___impl(simplequeueobject *self); + +static int +_queue_SimpleQueue___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + + if ((Py_TYPE(self) == &PySimpleQueueType) && + !_PyArg_NoPositional("SimpleQueue", args)) { + goto exit; + } + if ((Py_TYPE(self) == &PySimpleQueueType) && + !_PyArg_NoKeywords("SimpleQueue", kwargs)) { + goto exit; + } + return_value = _queue_SimpleQueue___init___impl((simplequeueobject *)self); + +exit: + return return_value; +} + +PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, +"put($self, /, item)\n" +"--\n" +"\n" +"Put the item on the queue, without blocking."); + +#define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ + {"put", (PyCFunction)_queue_SimpleQueue_put, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, + +static PyObject * +_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item); + +static PyObject * +_queue_SimpleQueue_put(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"item", NULL}; + static _PyArg_Parser _parser = {"O:put", _keywords, 0}; + PyObject *item; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &item)) { + goto exit; + } + return_value = _queue_SimpleQueue_put_impl(self, item); + +exit: + return return_value; +} + +PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, +"get($self, /, block=True, timeout=None)\n" +"--\n" +"\n" +"XXX"); + +#define _QUEUE_SIMPLEQUEUE_GET_METHODDEF \ + {"get", (PyCFunction)_queue_SimpleQueue_get, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_get__doc__}, + +static PyObject * +_queue_SimpleQueue_get_impl(simplequeueobject *self, int block, + PyObject *timeout); + +static PyObject * +_queue_SimpleQueue_get(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"block", "timeout", NULL}; + static _PyArg_Parser _parser = {"|pO:get", _keywords, 0}; + int block = 1; + PyObject *timeout = Py_None; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &block, &timeout)) { + goto exit; + } + return_value = _queue_SimpleQueue_get_impl(self, block, timeout); + +exit: + return return_value; +} +/*[clinic end generated code: output=1203bbd791fee07c input=a9049054013a1b77]*/ diff --git a/setup.py b/setup.py index 36a52bc6aaceac..297dee0461b2d2 100644 --- a/setup.py +++ b/setup.py @@ -681,6 +681,8 @@ def detect_modules(self): # asyncio speedups exts.append( Extension("_asyncio", ["_asynciomodule.c"]) ) + exts.append( Extension("_queue", ["_queuemodule.c"]) ) + # Modules with some UNIX dependencies -- on by default: # (If you have a really backward UNIX, select and socket may not be # supported...) From d3e890cd716c9d3b3c84bdb51fbc0f2f3e3f9ef3 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 17:55:21 +0200 Subject: [PATCH 03/23] Docstrings and comments --- Modules/_queuemodule.c | 19 ++++++++++++++----- Modules/_threadmodule.c | 4 ---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 1746e1aa7290af..e537e9c1a16031 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -97,6 +97,8 @@ _queue_SimpleQueue___init___impl(simplequeueobject *self) return 0; } +/* XXX should we support dummy "block" and "timeout" args for compatibility? */ + /*[clinic input] _queue.SimpleQueue.put item: object @@ -161,7 +163,15 @@ _queue.SimpleQueue.get block: bool = True timeout: object = None -XXX +Remove and return an item from the queue. + +If optional args 'block' is true and 'timeout' is None (the default), +block if necessary until an item is available. If 'timeout' is +a non-negative number, it blocks at most 'timeout' seconds and raises +the Empty exception if no item was available within that time. +Otherwise ('block' is false), return an item if one is immediately +available, else raise the Empty exception ('timeout' is ignored +in that case). [clinic start generated code]*/ @@ -278,7 +288,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_GET_METHODDEF _QUEUE_SIMPLEQUEUE_PUT_METHODDEF - /* XXX implement __sizeof__ */ + /* XXX implement __sizeof__, empty and qsize? */ {NULL, NULL} /* sentinel */ }; @@ -330,9 +340,8 @@ PyTypeObject PySimpleQueueType = { /* Initialization function */ PyDoc_STRVAR(queue_module_doc, -"XXX \n\ -This module provides primitive operations to write multi-threaded programs.\n\ -The 'threading' module provides a more convenient interface."); +"C implementation of the Python queue module.\n\ +This module is an implementation detail, please do not use it directly."); static struct PyModuleDef queuemodule = { PyModuleDef_HEAD_INIT, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index da750c01cd9598..0c44f883325b59 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1,7 +1,3 @@ - -/* Thread module */ -/* Interface to Sjoerd's portable C thread library */ - #include "Python.h" #include "structmember.h" /* offsetof */ From c0a087a4a830ea82699db27e4dc52a5c8e091240 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 18:01:53 +0200 Subject: [PATCH 04/23] Simplify some code away. --- Modules/_queuemodule.c | 31 +++---------------------------- Modules/_threadmodule.c | 4 ++++ 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index e537e9c1a16031..b9c0751117a799 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -1,7 +1,3 @@ - -/* Thread module */ -/* Interface to Sjoerd's portable C thread library */ - #include "Python.h" #include "structmember.h" /* offsetof */ @@ -23,8 +19,6 @@ extern PyTypeObject PySimpleQueueType; /* forward decl */ static PyObject *EmptyError; -static _PyTime_t infinite_timeout_val; - typedef struct { PyObject_HEAD @@ -103,7 +97,7 @@ _queue_SimpleQueue___init___impl(simplequeueobject *self) _queue.SimpleQueue.put item: object -Put the item on the queue, without blocking. +Put the item on the queue. This method never blocks. [clinic start generated code]*/ @@ -147,8 +141,9 @@ simplequeue_pop_item(simplequeueobject *self) self->lst_pos += 1; count = n - self->lst_pos; if (self->lst_pos > count) { - /* reclaim space at beginning of list */ + /* The list is more than 50% empty, reclaim space at the beginning */ if (PyList_SetSlice(self->lst, 0, self->lst_pos, NULL)) { + /* Undo pop */ self->lst_pos -= 1; PyList_SET_ITEM(self->lst, self->lst_pos, item); return NULL; @@ -214,25 +209,6 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, microseconds = -1; } -// fprintf(stderr, -// "GET [%lu] (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); - if (self->lst_pos < PyList_GET_SIZE(self->lst)) { - /* Fast path */ - assert(!self->locked); - /* BEGIN GIL-protected critical section */ - item = simplequeue_pop_item(self); - if (item == NULL) { - return NULL; - } - /* END GIL-protected critical section */ - assert(!self->locked); -// fprintf(stderr, -// "/GET [%lu] fast (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); - return item; - } - /* put() signals the queue to be non-empty by releasing the lock. * So we simply try to acquire the lock in a loop, until the condition * (queue non-empty) becomes true. @@ -383,6 +359,5 @@ PyInit__queue(void) if (PyModule_AddObject(m, "SimpleQueue", (PyObject *)&PySimpleQueueType) < 0) return NULL; - infinite_timeout_val = _PyTime_FromSeconds(-1); return m; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0c44f883325b59..da750c01cd9598 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1,3 +1,7 @@ + +/* Thread module */ +/* Interface to Sjoerd's portable C thread library */ + #include "Python.h" #include "structmember.h" /* offsetof */ From fa67c79c48fe21629e99022d1c8c1fec06fe669d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 18:05:19 +0200 Subject: [PATCH 05/23] Fix expectation in test --- Lib/test/test_queue.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index f9e8d123999d1f..f07c6ce970a3c3 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -462,12 +462,11 @@ def test_negative_timeout_raises_exception(self): q.get(timeout=-1) def test_order(self): - N = 3 q = self.q inputs = list(range(100)) - results = self.run_threads(N, 1, q, inputs, self.feed, self.consume) + results = self.run_threads(1, 1, q, inputs, self.feed, self.consume) - # One consumer => results appended in well-defined order + # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) def test_many_threads(self): From a58acf0a13a9ce7c4e7074100f5e5f38d3fe1eeb Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 18:10:02 +0200 Subject: [PATCH 06/23] Updated generated files --- Modules/_queuemodule.c | 4 ++-- Modules/clinic/_queuemodule.c.h | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index b9c0751117a799..ca51da8412f86b 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -103,7 +103,7 @@ Put the item on the queue. This method never blocks. static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item) -/*[clinic end generated code: output=5b502db806a97f16 input=fbf18e55e43edea9]*/ +/*[clinic end generated code: output=5b502db806a97f16 input=6d5e2cb86ae2246a]*/ { // fprintf(stderr, // "PUT [%lu] (lst_pos = %zd, count = %zd, locked = %d)\n", @@ -173,7 +173,7 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, PyObject *timeout) -/*[clinic end generated code: output=ec82a7157dcccd1a input=08e407fb75711d10]*/ +/*[clinic end generated code: output=ec82a7157dcccd1a input=4bf691f9f01fa297]*/ { _PyTime_t endtime = 0; _PyTime_t timeout_val; diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 5cbe7255eb0988..5632306fa9244f 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -34,7 +34,7 @@ PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, "put($self, /, item)\n" "--\n" "\n" -"Put the item on the queue, without blocking."); +"Put the item on the queue. This method never blocks."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", (PyCFunction)_queue_SimpleQueue_put, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, @@ -64,7 +64,15 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "get($self, /, block=True, timeout=None)\n" "--\n" "\n" -"XXX"); +"Remove and return an item from the queue.\n" +"\n" +"If optional args \'block\' is true and \'timeout\' is None (the default),\n" +"block if necessary until an item is available. If \'timeout\' is\n" +"a non-negative number, it blocks at most \'timeout\' seconds and raises\n" +"the Empty exception if no item was available within that time.\n" +"Otherwise (\'block\' is false), return an item if one is immediately\n" +"available, else raise the Empty exception (\'timeout\' is ignored\n" +"in that case)."); #define _QUEUE_SIMPLEQUEUE_GET_METHODDEF \ {"get", (PyCFunction)_queue_SimpleQueue_get, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_get__doc__}, @@ -91,4 +99,4 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyObject **args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=1203bbd791fee07c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=79f2a6160c9ce4be input=a9049054013a1b77]*/ From 0f86f5cd501aaebf3d4cef3cc34a52de8a6f16a4 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 19:01:57 +0200 Subject: [PATCH 07/23] Try to support --without-threads --- Modules/_queuemodule.c | 4 +--- setup.py | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index ca51da8412f86b..8bec9041839bef 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -2,9 +2,7 @@ #include "structmember.h" /* offsetof */ #ifndef WITH_THREAD -#error "Error! The rest of Python is not compiled with thread support." -#error "Rerun configure, adding a --with-threads option." -#error "Then run `make clean' followed by `make'." +#error "This module can only be compiled with threads support." #endif #include "pythread.h" diff --git a/setup.py b/setup.py index 297dee0461b2d2..ee90ba680915dc 100644 --- a/setup.py +++ b/setup.py @@ -681,7 +681,10 @@ def detect_modules(self): # asyncio speedups exts.append( Extension("_asyncio", ["_asynciomodule.c"]) ) - exts.append( Extension("_queue", ["_queuemodule.c"]) ) + if sysconfig.get_config_var('WITH_THREAD'): + exts.append( Extension("_queue", ["_queuemodule.c"]) ) + else: + missing.append('queue') # Modules with some UNIX dependencies -- on by default: # (If you have a really backward UNIX, select and socket may not be From 3d9c7094bdeca34b3fa47ba5dbace4e99d549e68 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 19:17:01 +0200 Subject: [PATCH 08/23] Fix EINTR-handling logic --- Modules/_queuemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 8bec9041839bef..db02b8d1e82683 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -223,7 +223,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, r = PyThread_acquire_lock_timed(self->lock, microseconds, 1); Py_END_ALLOW_THREADS } - if (r == PY_LOCK_INTR) { + if (r == PY_LOCK_INTR && Py_MakePendingCalls() < 0) { return NULL; } if (r == PY_LOCK_FAILURE) { From a76c04159b2376aa9cd2ece5b70680e2f9a15855 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 5 Sep 2017 23:53:54 +0200 Subject: [PATCH 09/23] Add qsize(), empty() and __sizeof__() --- Lib/queue.py | 20 ++++++++++ Lib/test/test_queue.py | 36 ++++++++++++++++++ Modules/_queuemodule.c | 66 +++++++++++++++++++++++---------- Modules/clinic/_queuemodule.c.h | 38 ++++++++++++++++++- 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 038179c0b1d133..e3002596980815 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -263,16 +263,36 @@ def __init__(self): self._count = threading.Semaphore(0) def put(self, item): + '''Put an item into the queue. This method never blocks. + ''' self._queue.append(item) self._count.release() def get(self, block=True, timeout=None): + '''Remove and return an item from the queue. + + If optional args 'block' is true and 'timeout' is None (the default), + block if necessary until an item is available. If 'timeout' is + a non-negative number, it blocks at most 'timeout' seconds and raises + the Empty exception if no item was available within that time. + Otherwise ('block' is false), return an item if one is immediately + available, else raise the Empty exception ('timeout' is ignored + in that case). + ''' if timeout is not None and timeout < 0: raise ValueError("'timeout' must be a non-negative number") if not self._count.acquire(block, timeout): raise Empty return self._queue.popleft() + def empty(self): + '''Return True if the queue is empty, False otherwise (not reliable!).''' + return len(self._queue) == 0 + + def qsize(self): + '''Return the approximate size of the queue (not reliable!).''' + return len(self._queue) + try: SimpleQueue = _queue.SimpleQueue diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index f07c6ce970a3c3..d8d1dbef4a65d9 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -3,6 +3,7 @@ import collections import queue import random +import sys import time import unittest from test import support @@ -437,23 +438,44 @@ def wrapper(*args, **kwargs): pass self.assertFalse(exceptions) + self.assertTrue(q.empty()) + self.assertEqual(q.qsize(), 0) return results def test_basic(self): q = self.q + self.assertTrue(q.empty()) + self.assertEqual(q.qsize(), 0) q.put(1) + self.assertFalse(q.empty()) + self.assertEqual(q.qsize(), 1) q.put(2) q.put(3) q.put(4) + self.assertFalse(q.empty()) + self.assertEqual(q.qsize(), 4) + self.assertEqual(q.get(), 1) + self.assertEqual(q.qsize(), 3) + self.assertEqual(q.get(), 2) + self.assertEqual(q.qsize(), 2) + self.assertEqual(q.get(block=False), 3) + self.assertFalse(q.empty()) + self.assertEqual(q.qsize(), 1) + self.assertEqual(q.get(timeout=0.1), 4) + self.assertTrue(q.empty()) + self.assertEqual(q.qsize(), 0) + with self.assertRaises(queue.Empty): q.get(block=False) with self.assertRaises(queue.Empty): q.get(timeout=1e-3) + self.assertTrue(q.empty()) + self.assertEqual(q.qsize(), 0) def test_negative_timeout_raises_exception(self): q = self.q @@ -510,6 +532,20 @@ class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): def test_is_default(self): self.assertIs(self.type2test, queue.SimpleQueue) + def test_sizeof(self): + N = 10 + q = self.q + # getsizeof() takes into account the internal queue size + cur = sys.getsizeof(q) + for item in range(N): + q.put(item) + new = sys.getsizeof(q) + self.assertGreater(new, cur) + # popping items eventually frees up some internal space + for i in range(N - 1): + q.get() + self.assertLess(sys.getsizeof(q), new) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index db02b8d1e82683..3f6d6af83ab706 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -103,9 +103,6 @@ static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item) /*[clinic end generated code: output=5b502db806a97f16 input=6d5e2cb86ae2246a]*/ { -// fprintf(stderr, -// "PUT [%lu] (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); /* BEGIN GIL-protected critical section */ if (PyList_Append(self->lst, item) < 0) return NULL; @@ -113,14 +110,8 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item) /* A get() may be waiting, wake it up */ self->locked = 0; PyThread_release_lock(self->lock); -// fprintf(stderr, -// "PUT [%lu] release lock (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); } /* END GIL-protected critical section */ -// fprintf(stderr, -// "/PUT [%lu]\n", -// PyThread_get_thread_ident()); Py_RETURN_NONE; } @@ -212,10 +203,6 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, * (queue non-empty) becomes true. */ while (self->lst_pos == PyList_GET_SIZE(self->lst)) { -// fprintf(stderr, -// "GET [%lu] acquire lock 2 (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); - /* First a simple non-blocking try without releasing the GIL */ r = PyThread_acquire_lock_timed(self->lock, 0, 0); if (r == PY_LOCK_FAILURE && microseconds != 0) { @@ -242,27 +229,66 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, assert(self->lst_pos < PyList_GET_SIZE(self->lst)); item = simplequeue_pop_item(self); if (self->locked) { -// fprintf(stderr, -// "GET [%lu] release lock (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); PyThread_release_lock(self->lock); self->locked = 0; } /* END GIL-protected critical section */ -// fprintf(stderr, -// "/GET [%lu] slow (lst_pos = %zd, count = %zd, locked = %d)\n", -// PyThread_get_thread_ident(), self->lst_pos, PyList_GET_SIZE(self->lst) - self->lst_pos, self->locked); return item; } +/*[clinic input] +_queue.SimpleQueue.empty + +Return True if the queue is empty, False otherwise (not reliable!). +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_empty_impl(simplequeueobject *self) +/*[clinic end generated code: output=25164355586aaad0 input=990412cc070aa6a2]*/ +{ + PyObject *r; + if (self->lst_pos == PyList_GET_SIZE(self->lst)) + r = Py_True; + else + r = Py_False; + Py_INCREF(r); + return r; +} + +/*[clinic input] +_queue.SimpleQueue.qsize + +Return the approximate size of the queue (not reliable!). +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_qsize_impl(simplequeueobject *self) +/*[clinic end generated code: output=4c6f52e316a53f60 input=76444ac3249e90c5]*/ +{ + return PyLong_FromSsize_t(PyList_GET_SIZE(self->lst) - self->lst_pos); +} + +static PyObject * +simplequeue_sizeof(simplequeueobject *self, void *unused) +{ + Py_ssize_t res; + + res = _PyObject_SIZE(Py_TYPE(self)); + res += _PySys_GetSizeOf(self->lst); + return PyLong_FromSsize_t(res); +} + + #include "clinic/_queuemodule.c.h" static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_GET_METHODDEF _QUEUE_SIMPLEQUEUE_PUT_METHODDEF - /* XXX implement __sizeof__, empty and qsize? */ + _QUEUE_SIMPLEQUEUE_EMPTY_METHODDEF + _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF + {"__sizeof__", (PyCFunction) simplequeue_sizeof, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 5632306fa9244f..ad68c8dabcf78d 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -99,4 +99,40 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyObject **args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=79f2a6160c9ce4be input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, +"empty($self, /)\n" +"--\n" +"\n" +"Return True if the queue is empty, False otherwise (not reliable!)."); + +#define _QUEUE_SIMPLEQUEUE_EMPTY_METHODDEF \ + {"empty", (PyCFunction)_queue_SimpleQueue_empty, METH_NOARGS, _queue_SimpleQueue_empty__doc__}, + +static PyObject * +_queue_SimpleQueue_empty_impl(simplequeueobject *self); + +static PyObject * +_queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +{ + return _queue_SimpleQueue_empty_impl(self); +} + +PyDoc_STRVAR(_queue_SimpleQueue_qsize__doc__, +"qsize($self, /)\n" +"--\n" +"\n" +"Return the approximate size of the queue (not reliable!)."); + +#define _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF \ + {"qsize", (PyCFunction)_queue_SimpleQueue_qsize, METH_NOARGS, _queue_SimpleQueue_qsize__doc__}, + +static PyObject * +_queue_SimpleQueue_qsize_impl(simplequeueobject *self); + +static PyObject * +_queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +{ + return _queue_SimpleQueue_qsize_impl(self); +} +/*[clinic end generated code: output=9dbbdf1f531051c2 input=a9049054013a1b77]*/ From bc07e44b1e9292ccea8e7395b5d9b6d4566a0f8f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 00:00:14 +0200 Subject: [PATCH 10/23] Add optional block and timeout args to put(), for compatibility. --- Lib/queue.py | 11 +++++++++-- Modules/_queuemodule.c | 16 ++++++++++------ Modules/clinic/_queuemodule.c.h | 22 ++++++++++++++-------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index e3002596980815..8887a7725a1cab 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -257,13 +257,20 @@ def _get(self): class _PySimpleQueue: + '''Simple, unbounded FIFO queue. + + This pure Python implementation is not reentrant. + ''' def __init__(self): self._queue = deque() self._count = threading.Semaphore(0) - def put(self, item): - '''Put an item into the queue. This method never blocks. + def put(self, item, block=True, timeout=None): + '''Put the item on the queue. + + The optional 'block' and 'timeout' arguments are ignored, as this method + never blocks. They are provided for compatibility with the Queue class. ''' self._queue.append(item) self._count.release() diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 3f6d6af83ab706..4d84516c63f5ef 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -79,7 +79,7 @@ simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /*[clinic input] _queue.SimpleQueue.__init__ -Simple reentrant queue. +Simple, unbounded, reentrant FIFO queue. [clinic start generated code]*/ static int @@ -89,19 +89,23 @@ _queue_SimpleQueue___init___impl(simplequeueobject *self) return 0; } -/* XXX should we support dummy "block" and "timeout" args for compatibility? */ - /*[clinic input] _queue.SimpleQueue.put item: object + block: bool = True + timeout: object = None + +Put the item on the queue. -Put the item on the queue. This method never blocks. +The optional 'block' and 'timeout' arguments are ignored, as this method +never blocks. They are provided for compatibility with the Queue class. [clinic start generated code]*/ static PyObject * -_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item) -/*[clinic end generated code: output=5b502db806a97f16 input=6d5e2cb86ae2246a]*/ +_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, + int block, PyObject *timeout) +/*[clinic end generated code: output=4333136e88f90d8b input=6e601fa707a782d5]*/ { /* BEGIN GIL-protected critical section */ if (PyList_Append(self->lst, item) < 0) diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index ad68c8dabcf78d..29867726454209 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -31,30 +31,36 @@ _queue_SimpleQueue___init__(PyObject *self, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, -"put($self, /, item)\n" +"put($self, /, item, block=True, timeout=None)\n" "--\n" "\n" -"Put the item on the queue. This method never blocks."); +"Put the item on the queue.\n" +"\n" +"The optional \'block\' and \'timeout\' arguments are ignored, as this method\n" +"never blocks. They are provided for compatibility with the Queue class."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", (PyCFunction)_queue_SimpleQueue_put, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, static PyObject * -_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item); +_queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, + int block, PyObject *timeout); static PyObject * _queue_SimpleQueue_put(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"item", NULL}; - static _PyArg_Parser _parser = {"O:put", _keywords, 0}; + static const char * const _keywords[] = {"item", "block", "timeout", NULL}; + static _PyArg_Parser _parser = {"O|pO:put", _keywords, 0}; PyObject *item; + int block = 1; + PyObject *timeout = Py_None; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &item)) { + &item, &block, &timeout)) { goto exit; } - return_value = _queue_SimpleQueue_put_impl(self, item); + return_value = _queue_SimpleQueue_put_impl(self, item, block, timeout); exit: return return_value; @@ -135,4 +141,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) { return _queue_SimpleQueue_qsize_impl(self); } -/*[clinic end generated code: output=9dbbdf1f531051c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b2b51cb20569489b input=a9049054013a1b77]*/ From f8fe7137566e612f1ab8549087a4477236aec57f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 00:28:43 +0200 Subject: [PATCH 11/23] Add put_nowait() and get_nowait() compatibility methods --- Lib/queue.py | 20 ++++++++++++ Lib/test/test_queue.py | 6 ++-- Modules/_queuemodule.c | 40 +++++++++++++++++++++-- Modules/clinic/_queuemodule.c.h | 58 +++++++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 6 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 8887a7725a1cab..4cdc6b6711edc4 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -261,6 +261,10 @@ class _PySimpleQueue: This pure Python implementation is not reentrant. ''' + # Note: while this pure Python version provides fairness + # (by using a threading.Semaphore which is itself fair, being based + # on threading.Condition), fairness is not part of the API contract. + # This allows the C version to use a different implementation. def __init__(self): self._queue = deque() @@ -292,6 +296,22 @@ def get(self, block=True, timeout=None): raise Empty return self._queue.popleft() + def put_nowait(self, item): + '''Put an item into the queue without blocking. + + This is exactly equivalent to `put(item)` and is only provided + for compatibility with the Queue class. + ''' + return self.put(item, block=False) + + def get_nowait(self): + '''Remove and return an item from the queue without blocking. + + Only get an item if one is immediately available. Otherwise + raise the Empty exception. + ''' + return self.get(block=False) + def empty(self): '''Return True if the queue is empty, False otherwise (not reliable!).''' return len(self._queue) == 0 diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index d8d1dbef4a65d9..709934c07bc989 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -451,7 +451,7 @@ def test_basic(self): self.assertFalse(q.empty()) self.assertEqual(q.qsize(), 1) q.put(2) - q.put(3) + q.put_nowait(3) q.put(4) self.assertFalse(q.empty()) self.assertEqual(q.qsize(), 4) @@ -459,7 +459,7 @@ def test_basic(self): self.assertEqual(q.get(), 1) self.assertEqual(q.qsize(), 3) - self.assertEqual(q.get(), 2) + self.assertEqual(q.get_nowait(), 2) self.assertEqual(q.qsize(), 2) self.assertEqual(q.get(block=False), 3) @@ -474,6 +474,8 @@ def test_basic(self): q.get(block=False) with self.assertRaises(queue.Empty): q.get(timeout=1e-3) + with self.assertRaises(queue.Empty): + q.get_nowait() self.assertTrue(q.empty()) self.assertEqual(q.qsize(), 0) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 4d84516c63f5ef..02af98fb853e52 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -84,7 +84,7 @@ Simple, unbounded, reentrant FIFO queue. static int _queue_SimpleQueue___init___impl(simplequeueobject *self) -/*[clinic end generated code: output=111d7b9a0c26aa23 input=fb09ddd81b016cea]*/ +/*[clinic end generated code: output=111d7b9a0c26aa23 input=21461b956918fcfb]*/ { return 0; } @@ -119,6 +119,24 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, Py_RETURN_NONE; } +/*[clinic input] +_queue.SimpleQueue.put_nowait + item: object + +Put an item into the queue without blocking. + +This is exactly equivalent to `put(item)` and is only provided +for compatibility with the Queue class. + +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item) +/*[clinic end generated code: output=0990536715efb1f1 input=36b1ea96756b2ece]*/ +{ + return _queue_SimpleQueue_put_impl(self, item, 0, Py_None); +} + static PyObject * simplequeue_pop_item(simplequeueobject *self) { @@ -241,6 +259,22 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, return item; } +/*[clinic input] +_queue.SimpleQueue.get_nowait + +Remove and return an item from the queue without blocking. + +Only get an item if one is immediately available. Otherwise +raise the Empty exception. +[clinic start generated code]*/ + +static PyObject * +_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self) +/*[clinic end generated code: output=a89731a75dbe4937 input=6fe5102db540a1b9]*/ +{ + return _queue_SimpleQueue_get_impl(self, 0, Py_None); +} + /*[clinic input] _queue.SimpleQueue.empty @@ -288,9 +322,11 @@ simplequeue_sizeof(simplequeueobject *self, void *unused) static PyMethodDef simplequeue_methods[] = { + _QUEUE_SIMPLEQUEUE_EMPTY_METHODDEF _QUEUE_SIMPLEQUEUE_GET_METHODDEF + _QUEUE_SIMPLEQUEUE_GET_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_PUT_METHODDEF - _QUEUE_SIMPLEQUEUE_EMPTY_METHODDEF + _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF {"__sizeof__", (PyCFunction) simplequeue_sizeof, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 29867726454209..3b6788b7213f2f 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -6,7 +6,7 @@ PyDoc_STRVAR(_queue_SimpleQueue___init____doc__, "SimpleQueue()\n" "--\n" "\n" -"Simple reentrant queue."); +"Simple, unbounded, reentrant FIFO queue."); static int _queue_SimpleQueue___init___impl(simplequeueobject *self); @@ -66,6 +66,39 @@ _queue_SimpleQueue_put(simplequeueobject *self, PyObject **args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(_queue_SimpleQueue_put_nowait__doc__, +"put_nowait($self, /, item)\n" +"--\n" +"\n" +"Put an item into the queue without blocking.\n" +"\n" +"This is exactly equivalent to `put(item)` and is only provided\n" +"for compatibility with the Queue class."); + +#define _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF \ + {"put_nowait", (PyCFunction)_queue_SimpleQueue_put_nowait, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put_nowait__doc__}, + +static PyObject * +_queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item); + +static PyObject * +_queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"item", NULL}; + static _PyArg_Parser _parser = {"O:put_nowait", _keywords, 0}; + PyObject *item; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &item)) { + goto exit; + } + return_value = _queue_SimpleQueue_put_nowait_impl(self, item); + +exit: + return return_value; +} + PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "get($self, /, block=True, timeout=None)\n" "--\n" @@ -106,6 +139,27 @@ _queue_SimpleQueue_get(simplequeueobject *self, PyObject **args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(_queue_SimpleQueue_get_nowait__doc__, +"get_nowait($self, /)\n" +"--\n" +"\n" +"Remove and return an item from the queue without blocking.\n" +"\n" +"Only get an item if one is immediately available. Otherwise\n" +"raise the Empty exception."); + +#define _QUEUE_SIMPLEQUEUE_GET_NOWAIT_METHODDEF \ + {"get_nowait", (PyCFunction)_queue_SimpleQueue_get_nowait, METH_NOARGS, _queue_SimpleQueue_get_nowait__doc__}, + +static PyObject * +_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self); + +static PyObject * +_queue_SimpleQueue_get_nowait(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +{ + return _queue_SimpleQueue_get_nowait_impl(self); +} + PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, "empty($self, /)\n" "--\n" @@ -141,4 +195,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) { return _queue_SimpleQueue_qsize_impl(self); } -/*[clinic end generated code: output=b2b51cb20569489b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d05b0a7744934b91 input=a9049054013a1b77]*/ From 3e1b27dfc4f5c85bbe38b99e34032e8e2cac6f35 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 13:03:25 +0200 Subject: [PATCH 12/23] Add reentrancy tests --- Lib/test/test_queue.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 709934c07bc989..a100ef6a84e1e7 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -1,6 +1,7 @@ # Some simple queue module tests, plus some failure conditions # to ensure the Queue locks remain stable. import collections +import itertools import queue import random import sys @@ -444,6 +445,7 @@ def wrapper(*args, **kwargs): return results def test_basic(self): + # Basic tests for get(), put() etc. q = self.q self.assertTrue(q.empty()) self.assertEqual(q.qsize(), 0) @@ -486,6 +488,7 @@ def test_negative_timeout_raises_exception(self): q.get(timeout=-1) def test_order(self): + # Test a pair of concurrent put() and get() q = self.q inputs = list(range(100)) results = self.run_threads(1, 1, q, inputs, self.feed, self.consume) @@ -494,6 +497,7 @@ def test_order(self): self.assertEqual(results, inputs) def test_many_threads(self): + # Test multiple concurrent put() and get() N = 50 q = self.q inputs = list(range(10000)) @@ -504,6 +508,7 @@ def test_many_threads(self): self.assertEqual(sorted(results), inputs) def test_many_threads_nonblock(self): + # Test multiple concurrent put() and get(block=False) N = 50 q = self.q inputs = list(range(10000)) @@ -513,6 +518,7 @@ def test_many_threads_nonblock(self): self.assertEqual(sorted(results), inputs) def test_many_threads_timeout(self): + # Test multiple concurrent put() and get(timeout=...) N = 50 q = self.q inputs = list(range(1000)) @@ -548,6 +554,34 @@ def test_sizeof(self): q.get() self.assertLess(sys.getsizeof(q), new) + def test_reentrancy(self): + # bpo-14976: put() may be called reentrantly in an asynchronous + # callback. + q = self.q + gen = itertools.count() + N = 10000 + results = [] + + # This test exploits the fact that __del__ in a reference cycle + # can be called any time the GC may run. + + class Circular(object): + def __init__(self): + self.circular = self + + def __del__(self): + q.put(next(gen)) + + while True: + o = Circular() + q.put(next(gen)) + del o + results.append(q.get()) + if results[-1] >= N: + break + + self.assertEqual(results, list(range(N + 1))) + if __name__ == "__main__": unittest.main() From fdb53799d85d8f32bb38c6f60f2d98fc647ed7d7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 19:29:24 +0200 Subject: [PATCH 13/23] Try to add _queue module to Visual Studio build files --- PCbuild/_queue.vcxproj | 77 ++++++++++++++++++++++++++++++ PCbuild/_queue.vcxproj.filters | 16 +++++++ PCbuild/pcbuild.proj | 2 +- PCbuild/pcbuild.sln | 18 +++++++ PCbuild/pythoncore.vcxproj.filters | 3 ++ 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 PCbuild/_queue.vcxproj create mode 100644 PCbuild/_queue.vcxproj.filters diff --git a/PCbuild/_queue.vcxproj b/PCbuild/_queue.vcxproj new file mode 100644 index 00000000000000..81df2dfe514e1e --- /dev/null +++ b/PCbuild/_queue.vcxproj @@ -0,0 +1,77 @@ + + + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + Win32 + + + Release + x64 + + + + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} + _queue + Win32Proj + + + + + DynamicLibrary + NotSet + + + + .pyd + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + + + + + + + {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} + false + + + + + + diff --git a/PCbuild/_queue.vcxproj.filters b/PCbuild/_queue.vcxproj.filters new file mode 100644 index 00000000000000..88b80826adfeec --- /dev/null +++ b/PCbuild/_queue.vcxproj.filters @@ -0,0 +1,16 @@ + + + + + + + + {c56a5dd3-7838-48e9-a781-855d8be7370f} + + + + + Source Files + + + \ No newline at end of file diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 8d30e0895cf33c..5b6b1c9be2666f 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -49,7 +49,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 82cfaf249d739a..5ff86a49c7c9ab 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -89,6 +89,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsol EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_queue", "_queue.vcxproj", "{78D80A15-BD8C-44E2-B49E-1F05B0A0A687}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "liblzma", "liblzma.vcxproj", "{12728250-16EC-4DC6-94D7-E21DD88947F8}" EndProject Global @@ -659,6 +661,22 @@ Global {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.ActiveCfg = Release|x64 {384C224A-7474-476E-A01B-750EA7DE918C}.Release|x64.Build.0 = Release|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.ActiveCfg = Debug|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.Build.0 = Debug|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|x64.ActiveCfg = Debug|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|x64.Build.0 = Debug|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.ActiveCfg = Release|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.Build.0 = Release|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|x64.ActiveCfg = Release|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|x64.Build.0 = Release|x64 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.ActiveCfg = Debug|Win32 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|Win32.Build.0 = Debug|Win32 {12728250-16EC-4DC6-94D7-E21DD88947F8}.Debug|x64.ActiveCfg = Debug|x64 diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index cbe1a3943ff19c..982cf8f12aad64 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -491,6 +491,9 @@ Modules + + Modules + Modules From f1d731df419c239083319cf5c25859fc3e69e212 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 20:01:54 +0200 Subject: [PATCH 14/23] Try to fix compilation on MSVC --- Modules/_queuemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 02af98fb853e52..713b6a6c20bbfa 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -334,7 +334,7 @@ static PyMethodDef simplequeue_methods[] = { PyTypeObject PySimpleQueueType = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + PyVarObject_HEAD_INIT(NULL, 0) "_queue.SimpleQueue", /*tp_name*/ sizeof(simplequeueobject), /*tp_size*/ 0, /*tp_itemsize*/ From 6f66a52278295fded8746283370a318bb1bdec8b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 6 Sep 2017 21:30:58 +0200 Subject: [PATCH 15/23] Change import idiom (and fix __all__) --- Lib/queue.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Lib/queue.py b/Lib/queue.py index 4cdc6b6711edc4..f3043f493782d4 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -7,17 +7,16 @@ from collections import deque from heapq import heappush, heappop from time import monotonic as time - try: - import _queue + from _queue import SimpleQueue except ImportError: - _queue = None + SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue'] +__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] try: - Empty = _queue.Empty + from _queue import Empty except AttributeError: class Empty(Exception): 'Exception raised by Queue.get(block=0)/get_nowait().' @@ -321,7 +320,5 @@ def qsize(self): return len(self._queue) -try: - SimpleQueue = _queue.SimpleQueue -except AttributeError: +if SimpleQueue is None: SimpleQueue = _PySimpleQueue From 8847e03303b206696c43a8cd647efe94a814eb0a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 7 Sep 2017 18:43:55 +0200 Subject: [PATCH 16/23] Add a test that references to items are not kept longer than they should --- Lib/test/test_queue.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index a100ef6a84e1e7..e9de6be88eed87 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -7,6 +7,7 @@ import sys import time import unittest +import weakref from test import support threading = support.import_module('threading') @@ -527,6 +528,20 @@ def test_many_threads_timeout(self): self.assertEqual(sorted(results), inputs) + def test_references(self): + # The queue should lose references to each item as soon as + # it leaves the queue. + class C: + pass + + N = 20 + q = self.q + for i in range(N): + q.put(C()) + for i in range(N): + wr = weakref.ref(q.get()) + self.assertIsNone(wr()) + class PySimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): type2test = queue._PySimpleQueue From 1773affbd193eebbd5c6e5cb13900e8c5e65f31f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 7 Sep 2017 19:01:25 +0200 Subject: [PATCH 17/23] Add docs --- Doc/library/queue.rst | 73 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index bd0fc2d8f3c735..ba85a89ee0d273 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -23,8 +23,14 @@ the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the lowest valued entry is retrieved first. -Internally, the module uses locks to temporarily block competing threads; -however, it is not designed to handle reentrancy within a thread. +Internally, those three types of queues use locks to temporarily block +competing threads; however, they are not designed to handle reentrancy +within a thread. + +In addition, the module implements a "simple" +:abbr:`FIFO (first-in, first-out)` queue type where +specific implementations can provide additional guarantees +in exchange for the smaller functionality. The :mod:`queue` module defines the following classes and exceptions: @@ -57,6 +63,14 @@ The :mod:`queue` module defines the following classes and exceptions: is a tuple in the form: ``(priority_number, data)``. +.. class:: SimpleQueue() + + Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. + Simple queues lack advanced functionality such as task tracking. + + .. versionadded:: 3.7 + + .. exception:: Empty Exception raised when non-blocking :meth:`~Queue.get` (or @@ -191,6 +205,60 @@ Example of how to wait for enqueued tasks to be completed:: t.join() +SimpleQueue Objects +------------------- + +:class:`SimpleQueue` objects provide the public methods described below. + +.. method:: SimpleQueue.qsize() + + Return the approximate size of the queue. Note, qsize() > 0 doesn't + guarantee that a subsequent get() will not block. + + +.. method:: SimpleQueue.empty() + + Return ``True`` if the queue is empty, ``False`` otherwise. If empty() + returns ``False`` it doesn't guarantee that a subsequent call to get() + will not block. + + +.. method:: SimpleQueue.put(item, block=True, timeout=None) + + Put *item* into the queue. The method never blocks and always succeeds + (except for potential low-level errors such as failure to allocate memory). + The optional args *block* and *timeout* are ignored and only provided + for compatibility with :meth:`Queue.put`. + + .. impl-detail:: + This method has a C implementation which is reentrant. That is, a + ``put()`` or ``get()`` call can be interrupted by another ``put()`` + call in the same thread without deadlocking or corrupting internal + state inside the queue. This makes it appropriate for use in + destructors such as ``__del__`` methods or :mod:`weakref` callbacks. + + +.. method:: SimpleQueue.put_nowait(item) + + Equivalent to ``put(item)``, provided for compatibility with + :meth:`Queue.put_nowait`. + + +.. method:: SimpleQueue.get(block=True, timeout=None) + + Remove and return an item from the queue. If optional args *block* is true and + *timeout* is ``None`` (the default), block if necessary until an item is available. + If *timeout* is a positive number, it blocks at most *timeout* seconds and + raises the :exc:`Empty` exception if no item was available within that time. + Otherwise (*block* is false), return an item if one is immediately available, + else raise the :exc:`Empty` exception (*timeout* is ignored in that case). + + +.. method:: SimpleQueue.get_nowait() + + Equivalent to ``get(False)``. + + .. seealso:: Class :class:`multiprocessing.Queue` @@ -200,4 +268,3 @@ Example of how to wait for enqueued tasks to be completed:: :class:`collections.deque` is an alternative implementation of unbounded queues with fast atomic :meth:`~collections.deque.append` and :meth:`~collections.deque.popleft` operations that do not require locking. - From 404184703469dab0d606f258e5dce78627cae712 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 7 Sep 2017 19:13:12 +0200 Subject: [PATCH 18/23] Add NEWS blurb --- .../next/Library/2017-09-07-19-12-47.bpo-14976.dx0Zxb.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-09-07-19-12-47.bpo-14976.dx0Zxb.rst diff --git a/Misc/NEWS.d/next/Library/2017-09-07-19-12-47.bpo-14976.dx0Zxb.rst b/Misc/NEWS.d/next/Library/2017-09-07-19-12-47.bpo-14976.dx0Zxb.rst new file mode 100644 index 00000000000000..a4087551322317 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-09-07-19-12-47.bpo-14976.dx0Zxb.rst @@ -0,0 +1,2 @@ +Add a queue.SimpleQueue class, an unbounded FIFO queue with a reentrant C +implementation of put(). From c75ccf6a6b1a42a1986f6967b0f85484178e8b85 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 19 Dec 2017 16:10:13 +0100 Subject: [PATCH 19/23] Remove obsolete WITH_THREAD guard --- Modules/_queuemodule.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 713b6a6c20bbfa..d26330836829bf 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -1,10 +1,5 @@ #include "Python.h" #include "structmember.h" /* offsetof */ - -#ifndef WITH_THREAD -#error "This module can only be compiled with threads support." -#endif - #include "pythread.h" /*[clinic input] From c963121fa8133168fc9229eea5cfcfca8c5aa053 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Jan 2018 20:43:51 +0100 Subject: [PATCH 20/23] Update clinic file --- Modules/clinic/_queuemodule.c.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 3b6788b7213f2f..e8e3d4ff971777 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -47,7 +47,7 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout); static PyObject * -_queue_SimpleQueue_put(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_put(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"item", "block", "timeout", NULL}; @@ -82,7 +82,7 @@ static PyObject * _queue_SimpleQueue_put_nowait_impl(simplequeueobject *self, PyObject *item); static PyObject * -_queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_put_nowait(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"item", NULL}; @@ -121,7 +121,7 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, PyObject *timeout); static PyObject * -_queue_SimpleQueue_get(simplequeueobject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_get(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"block", "timeout", NULL}; @@ -195,4 +195,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) { return _queue_SimpleQueue_qsize_impl(self); } -/*[clinic end generated code: output=d05b0a7744934b91 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47d790833e9d5fbb input=a9049054013a1b77]*/ From 3337e2496a9819ebca268e47afc3bd48af8e2f80 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Jan 2018 22:44:20 +0100 Subject: [PATCH 21/23] Address review comments --- Lib/test/test_queue.py | 94 +++++++++++++++++---------------- Modules/_queuemodule.c | 22 +++----- Modules/clinic/_queuemodule.c.h | 30 +++++++++-- 3 files changed, 81 insertions(+), 65 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index cf0161cc3d65bd..3fab0654ea51c6 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -546,55 +546,57 @@ class PySimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): type2test = queue._PySimpleQueue -if _queue is not None: - - class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): - type2test = _queue.SimpleQueue - - def test_is_default(self): - self.assertIs(self.type2test, queue.SimpleQueue) - - def test_sizeof(self): - N = 10 - q = self.q - # getsizeof() takes into account the internal queue size - cur = sys.getsizeof(q) - for item in range(N): - q.put(item) - new = sys.getsizeof(q) - self.assertGreater(new, cur) - # popping items eventually frees up some internal space - for i in range(N - 1): - q.get() - self.assertLess(sys.getsizeof(q), new) - - def test_reentrancy(self): - # bpo-14976: put() may be called reentrantly in an asynchronous - # callback. - q = self.q - gen = itertools.count() - N = 10000 - results = [] - - # This test exploits the fact that __del__ in a reference cycle - # can be called any time the GC may run. - - class Circular(object): - def __init__(self): - self.circular = self - - def __del__(self): - q.put(next(gen)) +@unittest.skipIf(_queue is None, "No _queue module found") +class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): - while True: - o = Circular() + def setUp(self): + self.type2test = _queue.SimpleQueue + super().setUp() + + def test_is_default(self): + self.assertIs(self.type2test, queue.SimpleQueue) + + def test_sizeof(self): + N = 10 + q = self.q + # getsizeof() takes into account the internal queue size + cur = sys.getsizeof(q) + for item in range(N): + q.put(item) + new = sys.getsizeof(q) + self.assertGreater(new, cur) + # popping items eventually frees up some internal space + for i in range(N - 1): + q.get() + self.assertLess(sys.getsizeof(q), new) + + def test_reentrancy(self): + # bpo-14976: put() may be called reentrantly in an asynchronous + # callback. + q = self.q + gen = itertools.count() + N = 10000 + results = [] + + # This test exploits the fact that __del__ in a reference cycle + # can be called any time the GC may run. + + class Circular(object): + def __init__(self): + self.circular = self + + def __del__(self): q.put(next(gen)) - del o - results.append(q.get()) - if results[-1] >= N: - break - self.assertEqual(results, list(range(N + 1))) + while True: + o = Circular() + q.put(next(gen)) + del o + results.append(q.get()) + if results[-1] >= N: + break + + self.assertEqual(results, list(range(N + 1))) if __name__ == "__main__": diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index d26330836829bf..b979a7dd93d9a5 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -271,35 +271,29 @@ _queue_SimpleQueue_get_nowait_impl(simplequeueobject *self) } /*[clinic input] -_queue.SimpleQueue.empty +_queue.SimpleQueue.empty -> bool Return True if the queue is empty, False otherwise (not reliable!). [clinic start generated code]*/ -static PyObject * +static int _queue_SimpleQueue_empty_impl(simplequeueobject *self) -/*[clinic end generated code: output=25164355586aaad0 input=990412cc070aa6a2]*/ +/*[clinic end generated code: output=1a02a1b87c0ef838 input=1a98431c45fd66f9]*/ { - PyObject *r; - if (self->lst_pos == PyList_GET_SIZE(self->lst)) - r = Py_True; - else - r = Py_False; - Py_INCREF(r); - return r; + return self->lst_pos == PyList_GET_SIZE(self->lst); } /*[clinic input] -_queue.SimpleQueue.qsize +_queue.SimpleQueue.qsize -> Py_ssize_t Return the approximate size of the queue (not reliable!). [clinic start generated code]*/ -static PyObject * +static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self) -/*[clinic end generated code: output=4c6f52e316a53f60 input=76444ac3249e90c5]*/ +/*[clinic end generated code: output=f9dcd9d0a90e121e input=7a74852b407868a1]*/ { - return PyLong_FromSsize_t(PyList_GET_SIZE(self->lst) - self->lst_pos); + return PyList_GET_SIZE(self->lst) - self->lst_pos; } static PyObject * diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index e8e3d4ff971777..e4f993224bd522 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -169,13 +169,23 @@ PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, #define _QUEUE_SIMPLEQUEUE_EMPTY_METHODDEF \ {"empty", (PyCFunction)_queue_SimpleQueue_empty, METH_NOARGS, _queue_SimpleQueue_empty__doc__}, -static PyObject * +static int _queue_SimpleQueue_empty_impl(simplequeueobject *self); static PyObject * _queue_SimpleQueue_empty(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) { - return _queue_SimpleQueue_empty_impl(self); + PyObject *return_value = NULL; + int _return_value; + + _return_value = _queue_SimpleQueue_empty_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; } PyDoc_STRVAR(_queue_SimpleQueue_qsize__doc__, @@ -187,12 +197,22 @@ PyDoc_STRVAR(_queue_SimpleQueue_qsize__doc__, #define _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF \ {"qsize", (PyCFunction)_queue_SimpleQueue_qsize, METH_NOARGS, _queue_SimpleQueue_qsize__doc__}, -static PyObject * +static Py_ssize_t _queue_SimpleQueue_qsize_impl(simplequeueobject *self); static PyObject * _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) { - return _queue_SimpleQueue_qsize_impl(self); + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + _return_value = _queue_SimpleQueue_qsize_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; } -/*[clinic end generated code: output=47d790833e9d5fbb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2c48491738047b09 input=a9049054013a1b77]*/ From 3646e990789ccd9120819b000d4b8bd97bb5c570 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Jan 2018 23:14:10 +0100 Subject: [PATCH 22/23] Use Argument Clinic for __new__ --- Modules/_queuemodule.c | 29 ++++++++++++----------------- Modules/clinic/_queuemodule.c.h | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index b979a7dd93d9a5..c37744da9746bc 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -46,8 +46,16 @@ simplequeue_traverse(simplequeueobject *self, visitproc visit, void *arg) return 0; } +/*[clinic input] +@classmethod +_queue.SimpleQueue.__new__ as simplequeue_new + +Simple, unbounded, reentrant FIFO queue. +[clinic start generated code]*/ + static PyObject * -simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +simplequeue_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=ba97740608ba31cd input=a0674a1643e3e2fb]*/ { simplequeueobject *self; @@ -71,19 +79,6 @@ simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *) self; } -/*[clinic input] -_queue.SimpleQueue.__init__ - -Simple, unbounded, reentrant FIFO queue. -[clinic start generated code]*/ - -static int -_queue_SimpleQueue___init___impl(simplequeueobject *self) -/*[clinic end generated code: output=111d7b9a0c26aa23 input=21461b956918fcfb]*/ -{ - return 0; -} - /*[clinic input] _queue.SimpleQueue.put item: object @@ -328,7 +323,7 @@ PyTypeObject PySimpleQueueType = { sizeof(simplequeueobject), /*tp_size*/ 0, /*tp_itemsize*/ /* methods */ - (destructor)simplequeue_dealloc, /*tp_dealloc*/ + (destructor)simplequeue_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -345,7 +340,7 @@ PyTypeObject PySimpleQueueType = { 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - _queue_SimpleQueue___init____doc__, /*tp_doc*/ + simplequeue_new__doc__, /*tp_doc*/ (traverseproc)simplequeue_traverse, /*tp_traverse*/ 0, /*tp_clear*/ 0, /*tp_richcompare*/ @@ -360,7 +355,7 @@ PyTypeObject PySimpleQueueType = { 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - _queue_SimpleQueue___init__, /* tp_init */ + 0, /* tp_init */ 0, /* tp_alloc */ simplequeue_new /* tp_new */ }; diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index e4f993224bd522..97247fd8a129e6 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -2,29 +2,29 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_queue_SimpleQueue___init____doc__, +PyDoc_STRVAR(simplequeue_new__doc__, "SimpleQueue()\n" "--\n" "\n" "Simple, unbounded, reentrant FIFO queue."); -static int -_queue_SimpleQueue___init___impl(simplequeueobject *self); +static PyObject * +simplequeue_new_impl(PyTypeObject *type); -static int -_queue_SimpleQueue___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; - if ((Py_TYPE(self) == &PySimpleQueueType) && + if ((type == &PySimpleQueueType) && !_PyArg_NoPositional("SimpleQueue", args)) { goto exit; } - if ((Py_TYPE(self) == &PySimpleQueueType) && + if ((type == &PySimpleQueueType) && !_PyArg_NoKeywords("SimpleQueue", kwargs)) { goto exit; } - return_value = _queue_SimpleQueue___init___impl((simplequeueobject *)self); + return_value = simplequeue_new_impl(type); exit: return return_value; @@ -215,4 +215,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=2c48491738047b09 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8badc3bb85263689 input=a9049054013a1b77]*/ From 0d03ff5d06bf22ab380ac8f1a14247de8c8609e6 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 15 Jan 2018 23:19:54 +0100 Subject: [PATCH 23/23] Remove __sizeof__ implementation as suggested by Serhiy --- Lib/test/test_queue.py | 14 -------------- Modules/_queuemodule.c | 11 ----------- 2 files changed, 25 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 3fab0654ea51c6..1a8d5f8856c5e8 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -556,20 +556,6 @@ def setUp(self): def test_is_default(self): self.assertIs(self.type2test, queue.SimpleQueue) - def test_sizeof(self): - N = 10 - q = self.q - # getsizeof() takes into account the internal queue size - cur = sys.getsizeof(q) - for item in range(N): - q.put(item) - new = sys.getsizeof(q) - self.assertGreater(new, cur) - # popping items eventually frees up some internal space - for i in range(N - 1): - q.get() - self.assertLess(sys.getsizeof(q), new) - def test_reentrancy(self): # bpo-14976: put() may be called reentrantly in an asynchronous # callback. diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index c37744da9746bc..8715337fb51179 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -291,16 +291,6 @@ _queue_SimpleQueue_qsize_impl(simplequeueobject *self) return PyList_GET_SIZE(self->lst) - self->lst_pos; } -static PyObject * -simplequeue_sizeof(simplequeueobject *self, void *unused) -{ - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)); - res += _PySys_GetSizeOf(self->lst); - return PyLong_FromSsize_t(res); -} - #include "clinic/_queuemodule.c.h" @@ -312,7 +302,6 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_PUT_METHODDEF _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF - {"__sizeof__", (PyCFunction) simplequeue_sizeof, METH_NOARGS, NULL}, {NULL, NULL} /* sentinel */ };