Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dc113a8

Browse files
gpsheadgpshead
committed
* Fix the singlethreaded deadlocks occurring in the simple bsddb interface.
* Add support for multiple iterator/generator objects at once on the simple bsddb _DBWithCursor interface.
1 parent e276717 commit dc113a8

3 files changed

Lines changed: 155 additions & 9 deletions

File tree

‎Lib/bsddb/__init__.py‎

Lines changed: 102 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -70,20 +70,74 @@
7070
class _iter_mixin(UserDict.DictMixin):
7171
def __iter__(self):
7272
try:
73-
yield self.first()[0]
74-
next = self.next
73+
cur = self.db.cursor()
74+
self._iter_cursors[str(cur)] = cur
75+
76+
# since we're only returning keys, we call the cursor
77+
# methods with flags=0, dlen=0, dofs=0
78+
curkey = cur.first(0,0,0)[0]
79+
yield curkey
80+
81+
next = cur.next
7582
while 1:
76-
yield next()[0]
83+
try:
84+
curkey = next(0,0,0)[0]
85+
yield curkey
86+
except _bsddb.DBCursorClosedError:
87+
# our cursor object was closed since we last yielded
88+
# create a new one and attempt to reposition to the
89+
# right place
90+
cur = self.db.cursor()
91+
self._iter_cursors[str(cur)] = cur
92+
# FIXME-20031101-greg: race condition. cursor could
93+
# be closed by another thread before this set call.
94+
try:
95+
cur.set(curkey,0,0,0)
96+
except _bsddb.DBCursorClosedError:
97+
# halt iteration on race condition...
98+
raise _bsddb.DBNotFoundError
99+
next = cur.next
77100
except _bsddb.DBNotFoundError:
101+
try:
102+
del self._iter_cursors[str(cur)]
103+
except KeyError:
104+
pass
78105
return
79106
80107
def iteritems(self):
81108
try:
82-
yield self.first()
83-
next = self.next
109+
cur = self.db.cursor()
110+
self._iter_cursors[str(cur)] = cur
111+
112+
kv = cur.first()
113+
curkey = kv[0]
114+
yield kv
115+
116+
next = cur.next
84117
while 1:
85-
yield next()
118+
try:
119+
kv = next()
120+
curkey = kv[0]
121+
yield kv
122+
except _bsddb.DBCursorClosedError:
123+
# our cursor object was closed since we last yielded
124+
# create a new one and attempt to reposition to the
125+
# right place
126+
cur = self.db.cursor()
127+
self._iter_cursors[str(cur)] = cur
128+
# FIXME-20031101-greg: race condition. cursor could
129+
# be closed by another thread before this set call.
130+
try:
131+
cur.set(curkey,0,0,0)
132+
except _bsddb.DBCursorClosedError:
133+
# halt iteration on race condition...
134+
raise _bsddb.DBNotFoundError
135+
next = cur.next
86136
except _bsddb.DBNotFoundError:
137+
try:
138+
del self._iter_cursors[str(cur)]
139+
except KeyError:
140+
pass
87141
return
88142
"""
89143
else:
@@ -97,15 +151,53 @@ class _DBWithCursor(_iter_mixin):
97151
"""
98152
def __init__(self, db):
99153
self.db = db
100-
self.dbc = None
101154
self.db.set_get_returns_none(0)
102155

156+
# FIXME-20031101-greg: I believe there is still the potential
157+
# for deadlocks in a multithreaded environment if someone
158+
# attempts to use the any of the cursor interfaces in one
159+
# thread while doing a put or delete in another thread. The
160+
# reason is that _checkCursor and _closeCursors are not atomic
161+
# operations. Doing our own locking around self.dbc,
162+
# self.saved_dbc_key and self._iter_cursors could prevent this.
163+
# TODO: A test case demonstrating the problem needs to be written.
164+
165+
# self.dbc is a DBCursor object used to implement the
166+
# first/next/previous/last/set_location methods.
167+
self.dbc = None
168+
self.saved_dbc_key = None
169+
170+
# a collection of all DBCursor objects currently allocated
171+
# by the _iter_mixin interface.
172+
self._iter_cursors = {}
173+
174+
103175
def __del__(self):
104176
self.close()
105177

178+
def _get_dbc(self):
179+
return self.dbc
180+
106181
def _checkCursor(self):
107182
if self.dbc is None:
108183
self.dbc = self.db.cursor()
184+
if self.saved_dbc_key is not None:
185+
self.dbc.set(self.saved_dbc_key)
186+
self.saved_dbc_key = None
187+
188+
# This method is needed for all non-cursor DB calls to avoid
189+
# BerkeleyDB deadlocks (due to being opened with DB_INIT_LOCK
190+
# and DB_THREAD to be thread safe) when intermixing database
191+
# operations that use the cursor internally with those that don't.
192+
def _closeCursors(self, save=True):
193+
if self.dbc:
194+
c = self.dbc
195+
self.dbc = None
196+
if save:
197+
self.saved_dbc_key = c.current(0,0,0)[0]
198+
c.close()
199+
del c
200+
map(lambda c: c.close(), self._iter_cursors.values())
109201

110202
def _checkOpen(self):
111203
if self.db is None:
@@ -124,13 +216,16 @@ def __getitem__(self, key):
124216

125217
def __setitem__(self, key, value):
126218
self._checkOpen()
219+
self._closeCursors()
127220
self.db[key] = value
128221

129222
def __delitem__(self, key):
130223
self._checkOpen()
224+
self._closeCursors()
131225
del self.db[key]
132226

133227
def close(self):
228+
self._closeCursors(save=False)
134229
if self.dbc is not None:
135230
self.dbc.close()
136231
v = 0

‎Lib/test/test_bsddb.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"""Test script for the bsddb C module by Roger E. Masse
33
Adapted to unittest format and expanded scope by Raymond Hettinger
44
"""
5-
import os
5+
import os, sys
66
import bsddb
77
import dbhash # Just so we know it's imported
88
import unittest
@@ -93,6 +93,57 @@ def test_clear(self):
9393
self.f.clear()
9494
self.assertEqual(len(self.f), 0)
9595

96+
def test__no_deadlock_first(self, debug=0):
97+
# do this so that testers can see what function we're in in
98+
# verbose mode when we deadlock.
99+
sys.stdout.flush()
100+
101+
# in pybsddb's _DBWithCursor this causes an internal DBCursor
102+
# object is created. Other test_ methods in this class could
103+
# inadvertently cause the deadlock but an explicit test is needed.
104+
if debug: print "A"
105+
k,v = self.f.first()
106+
if debug: print "B", k
107+
self.f[k] = "deadlock. do not pass go. do not collect $200."
108+
if debug: print "C"
109+
# if the bsddb implementation leaves the DBCursor open during
110+
# the database write and locking+threading support is enabled
111+
# the cursor's read lock will deadlock the write lock request..
112+
113+
# test the iterator interface (if present)
114+
if hasattr(self, 'iteritems'):
115+
if debug: print "D"
116+
k,v = self.f.iteritems()
117+
if debug: print "E"
118+
self.f[k] = "please don't deadlock"
119+
if debug: print "F"
120+
while 1:
121+
try:
122+
k,v = self.f.iteritems()
123+
except StopIteration:
124+
break
125+
if debug: print "F2"
126+
127+
i = iter(self.f)
128+
if debug: print "G"
129+
while i:
130+
try:
131+
if debug: print "H"
132+
k = i.next()
133+
if debug: print "I"
134+
self.f[k] = "deadlocks-r-us"
135+
if debug: print "J"
136+
except StopIteration:
137+
i = None
138+
if debug: print "K"
139+
140+
# test the legacy cursor interface mixed with writes
141+
self.assert_(self.f.first()[0] in self.d)
142+
k = self.f.next()[0]
143+
self.assert_(k in self.d)
144+
self.f[k] = "be gone with ye deadlocks"
145+
self.assert_(self.f[k], "be gone with ye deadlocks")
146+
96147
def test_popitem(self):
97148
k, v = self.f.popitem()
98149
self.assert_(k in self.d)

‎Modules/_bsddb.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@
9393
/* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */
9494
#define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR)
9595

96-
#define PY_BSDDB_VERSION "4.2.2"
96+
#define PY_BSDDB_VERSION "4.2.3"
9797
static char *rcs_id = "$Id$";
9898

9999

0 commit comments

Comments
 (0)