Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b2c7aff

Browse files
loewisloewis
committed
Merge with bsddb3 2002.11.23.10.42.36
1 parent a797d81 commit b2c7aff

6 files changed

Lines changed: 100 additions & 47 deletions

File tree

‎Lib/bsddb/dbtables.py‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#-----------------------------------------------------------------------
22
#
33
# Copyright (C) 2000, 2001 by Autonomous Zone Industries
4+
# Copyright (C) 2002 Gregory P. Smith
45
#
56
# License: This is free software. You may use this software for any
67
# purpose including modification/redistribution, so long as
@@ -54,6 +55,13 @@ def __init__(self, prefix):
5455
def __call__(self, s):
5556
return s[:len(self.prefix)] == self.prefix
5657

58+
class PostfixCond(Cond):
59+
"""Acts as a condition function for matching a string postfix"""
60+
def __init__(self, postfix):
61+
self.postfix = postfix
62+
def __call__(self, s):
63+
return s[-len(self.postfix):] == self.postfix
64+
5765
class LikeCond(Cond):
5866
"""
5967
Acts as a function that will match using an SQL 'LIKE' style
@@ -523,17 +531,10 @@ def cmp_conditions(atuple, btuple):
523531
# if no condition was specified or the condition
524532
# succeeds, add row to our match list.
525533
if not condition or condition(data) :
526-
# only create new entries in matcing_rowids on
527-
# the first pass, otherwise reject the
528-
# rowid as it must not have matched
529-
# the previous passes
530-
if column_num == 0 :
531-
if not matching_rowids.has_key(rowid) :
532-
matching_rowids[rowid] = {}
533-
if savethiscolumndata :
534-
matching_rowids[rowid][column] = data
535-
else :
536-
rejected_rowids[rowid] = rowid
534+
if not matching_rowids.has_key(rowid) :
535+
matching_rowids[rowid] = {}
536+
if savethiscolumndata :
537+
matching_rowids[rowid][column] = data
537538
else :
538539
if matching_rowids.has_key(rowid) :
539540
del matching_rowids[rowid]

‎Lib/bsddb/dbutils.py‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
#------------------------------------------------------------------------
22
#
3-
# In my performance tests, using this (as in dbtest.py test4) is
4-
# slightly slower than simply compiling _db.c with MYDB_THREAD
5-
# undefined to prevent multithreading support in the C module.
6-
# Using NoDeadlockDb also prevent deadlocks from mutliple processes
7-
# accessing the same database.
8-
#
93
# Copyright (C) 2000 Autonomous Zone Industries
104
#
115
# License: This is free software. You may use this software for any
@@ -18,7 +12,7 @@
1812
# Author: Gregory P. Smith <[email protected]>
1913
#
2014
# Note: I don't know how useful this is in reality since when a
21-
# DBDeadlockError happens the current transaction is supposed to be
15+
# DBLockDeadlockError happens the current transaction is supposed to be
2216
# aborted. If it doesn't then when the operation is attempted again
2317
# the deadlock is still happening...
2418
# --Robin
@@ -34,35 +28,47 @@
3428
_sleep = sleep
3529
del sleep
3630

37-
import _db
31+
import _bsddb
3832

39-
_deadlock_MinSleepTime = 1.0/64 # always sleep at least N seconds between retrys
40-
_deadlock_MaxSleepTime = 1.0 # never sleep more than N seconds between retrys
33+
_deadlock_MinSleepTime = 1.0/64 # always sleep at least N seconds between retrys
34+
_deadlock_MaxSleepTime = 3.14159 # never sleep more than N seconds between retrys
4135

36+
_deadlock_VerboseFile = None # Assign a file object to this for a "sleeping"
37+
# message to be written to it each retry
4238

4339
def DeadlockWrap(function, *_args, **_kwargs):
4440
"""DeadlockWrap(function, *_args, **_kwargs) - automatically retries
4541
function in case of a database deadlock.
4642
47-
This is a DeadlockWrapper method which DB calls can be made using to
48-
preform infinite retrys with sleeps in between when a DBLockDeadlockError
49-
exception is raised in a database call:
43+
This is a function intended to be used to wrap database calls such
44+
that they perform retrys with exponentially backing off sleeps in
45+
between when a DBLockDeadlockError exception is raised.
46+
47+
A 'max_retries' parameter may optionally be passed to prevent it
48+
from retrying forever (in which case the exception will be reraised).
5049
5150
d = DB(...)
5251
d.open(...)
5352
DeadlockWrap(d.put, "foo", data="bar") # set key "foo" to "bar"
5453
"""
5554
sleeptime = _deadlock_MinSleepTime
56-
while (1) :
55+
max_retries = _kwargs.get('max_retries', -1)
56+
if _kwargs.has_key('max_retries'):
57+
del _kwargs['max_retries']
58+
while 1:
5759
try:
5860
return apply(function, _args, _kwargs)
5961
except _db.DBLockDeadlockError:
60-
print 'DeadlockWrap sleeping ', sleeptime
62+
if _deadlock_VerboseFile:
63+
_deadlock_VerboseFile.write('dbutils.DeadlockWrap: sleeping %1.3f\n' % sleeptime)
6164
_sleep(sleeptime)
6265
# exponential backoff in the sleep time
6366
sleeptime = sleeptime * 2
6467
if sleeptime > _deadlock_MaxSleepTime :
6568
sleeptime = _deadlock_MaxSleepTime
69+
max_retries = max_retries - 1
70+
if max_retries == -1:
71+
raise
6672

6773

6874
#------------------------------------------------------------------------

‎Lib/bsddb/test/test_dbshelve.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ class BTreeShelveTestCase(BasicShelveTestCase):
210210

211211

212212
class HashShelveTestCase(BasicShelveTestCase):
213-
dbtype = db.DB_BTREE
213+
dbtype = db.DB_HASH
214214
dbflags = db.DB_CREATE
215215

216216

@@ -220,7 +220,7 @@ class ThreadBTreeShelveTestCase(BasicShelveTestCase):
220220

221221

222222
class ThreadHashShelveTestCase(BasicShelveTestCase):
223-
dbtype = db.DB_BTREE
223+
dbtype = db.DB_HASH
224224
dbflags = db.DB_CREATE | db.DB_THREAD
225225

226226

@@ -261,7 +261,7 @@ class EnvBTreeShelveTestCase(BasicEnvShelveTestCase):
261261

262262
class EnvHashShelveTestCase(BasicEnvShelveTestCase):
263263
envflags = 0
264-
dbtype = db.DB_BTREE
264+
dbtype = db.DB_HASH
265265
dbflags = db.DB_CREATE
266266

267267

@@ -273,7 +273,7 @@ class EnvThreadBTreeShelveTestCase(BasicEnvShelveTestCase):
273273

274274
class EnvThreadHashShelveTestCase(BasicEnvShelveTestCase):
275275
envflags = db.DB_THREAD
276-
dbtype = db.DB_BTREE
276+
dbtype = db.DB_HASH
277277
dbflags = db.DB_CREATE | db.DB_THREAD
278278

279279

‎Lib/bsddb/test/test_dbtables.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#-----------------------------------------------------------------------
66
#
77
# Copyright (C) 2000, 2001 by Autonomous Zone Industries
8+
# Copyright (C) 2002 Gregory P. Smith
89
#
910
# March 20, 2000
1011
#
@@ -159,6 +160,40 @@ def test03(self):
159160
assert values[0]['b'] == "bad"
160161

161162

163+
def test04_MultiCondSelect(self):
164+
tabname = "test04_MultiCondSelect"
165+
try:
166+
self.tdb.Drop(tabname)
167+
except dbtables.TableDBError:
168+
pass
169+
self.tdb.CreateTable(tabname, ['a', 'b', 'c', 'd', 'e'])
170+
171+
try:
172+
self.tdb.Insert(tabname, {'a': "", 'e': pickle.dumps([{4:5, 6:7}, 'foo'], 1), 'f': "Zero"})
173+
assert 0
174+
except dbtables.TableDBError:
175+
pass
176+
177+
self.tdb.Insert(tabname, {'a': "A", 'b': "B", 'c': "C", 'd': "D", 'e': "E"})
178+
self.tdb.Insert(tabname, {'a': "-A", 'b': "-B", 'c': "-C", 'd': "-D", 'e': "-E"})
179+
self.tdb.Insert(tabname, {'a': "A-", 'b': "B-", 'c': "C-", 'd': "D-", 'e': "E-"})
180+
181+
if verbose:
182+
self.tdb._db_print()
183+
184+
# This select should return 0 rows. it is designed to test
185+
# the bug identified and fixed in sourceforge bug # 590449
186+
# (Big Thanks to "Rob Tillotson (n9mtb)" for tracking this down
187+
# and supplying a fix!! This one caused many headaches to say
188+
# the least...)
189+
values = self.tdb.Select(tabname, ['b', 'a', 'd'],
190+
conditions={'e': dbtables.ExactCond('E'),
191+
'a': dbtables.ExactCond('A'),
192+
'd': dbtables.PrefixCond('-')
193+
} )
194+
assert len(values) == 0, values
195+
196+
162197
def test_CreateOrExtend(self):
163198
tabname = "test_CreateOrExtend"
164199

‎Lib/bsddb/test/test_thread.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import unittest
1919
from test.test_support import verbose
2020

21-
from bsddb import db
21+
from bsddb import db, dbutils
2222

2323

2424
#----------------------------------------------------------------------
@@ -31,6 +31,9 @@ class BaseThreadedTestCase(unittest.TestCase):
3131

3232

3333
def setUp(self):
34+
if verbose:
35+
dbutils._deadlock_VerboseFile = sys.stdout
36+
3437
homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home')
3538
self.homeDir = homeDir
3639
try: os.mkdir(homeDir)
@@ -109,7 +112,7 @@ def writerThread(self, d, howMany, writerNum):
109112

110113
for x in range(start, stop):
111114
key = '%04d' % x
112-
d.put(key, self.makeData(key))
115+
dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12)
113116
if verbose and x % 100 == 0:
114117
print "%s: records %d - %d finished" % (name, start, x)
115118

@@ -212,7 +215,7 @@ def writerThread(self, d, howMany, writerNum):
212215
# create a bunch of records
213216
for x in xrange(start, stop):
214217
key = '%04d' % x
215-
d.put(key, self.makeData(key))
218+
dbutils.DeadlockWrap(d.put, key, self.makeData(key), max_retries=12)
216219

217220
if verbose and x % 100 == 0:
218221
print "%s: records %d - %d finished" % (name, start, x)
@@ -221,25 +224,25 @@ def writerThread(self, d, howMany, writerNum):
221224
if random() <= 0.05:
222225
for y in xrange(start, x):
223226
key = '%04d' % x
224-
data = d.get(key)
227+
data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
225228
assert data == self.makeData(key)
226229

227230
# flush them
228231
try:
229-
d.sync()
232+
dbutils.DeadlockWrap(d.sync, max_retries=12)
230233
except db.DBIncompleteError, val:
231234
if verbose:
232235
print "could not complete sync()..."
233236

234237
# read them back, deleting a few
235238
for x in xrange(start, stop):
236239
key = '%04d' % x
237-
data = d.get(key)
240+
data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
238241
if verbose and x % 100 == 0:
239242
print "%s: fetched record (%s, %s)" % (name, key, data)
240-
assert data == self.makeData(key)
243+
assert data == self.makeData(key), (key, data, self.makeData(key))
241244
if random() <= 0.10:
242-
d.delete(key)
245+
dbutils.DeadlockWrap(d.delete, key, max_retries=12)
243246
if verbose:
244247
print "%s: deleted record %s" % (name, key)
245248

@@ -273,7 +276,7 @@ class BTreeSimpleThreaded(SimpleThreadedBase):
273276

274277

275278
class HashSimpleThreaded(SimpleThreadedBase):
276-
dbtype = db.DB_BTREE
279+
dbtype = db.DB_HASH
277280

278281

279282
#----------------------------------------------------------------------

‎Modules/_bsddb.c‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,12 @@
7575
/* --------------------------------------------------------------------- */
7676
/* Various macro definitions */
7777

78-
#define PY_BSDDB_VERSION "3.4.0"
78+
#define PY_BSDDB_VERSION "3.4.2"
7979

8080
/* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */
8181
#define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR)
8282

83-
static char *orig_rcs_id = "/Id: _db.c,v 1.44 2002/06/07 18:24:00 greg Exp /";
83+
static char *orig_rcs_id = "/Id: _db.c,v 1.48 2002/11/21 19:11:19 greg Exp /";
8484
static char *rcs_id = "$Id$";
8585

8686

@@ -1012,7 +1012,17 @@ DB_associate(DBObject* self, PyObject* args, PyObject* kwargs)
10121012
secondaryDB->associateCallback = callback;
10131013
secondaryDB->primaryDBType = _DB_get_type(self);
10141014

1015-
1015+
/* PyEval_InitThreads is called here due to a quirk in python 1.5
1016+
* - 2.2.1 (at least) according to Russell Williamson <[email protected]>:
1017+
* The global interepreter lock is not initialized until the first
1018+
* thread is created using thread.start_new_thread() or fork() is
1019+
* called. that would cause the ALLOW_THREADS here to segfault due
1020+
* to a null pointer reference if no threads or child processes
1021+
* have been created. This works around that and is a no-op if
1022+
* threads have already been initialized.
1023+
* (see pybsddb-users mailing list post on 2002-08-07)
1024+
*/
1025+
PyEval_InitThreads();
10161026
MYDB_BEGIN_ALLOW_THREADS;
10171027
err = self->db->associate(self->db,
10181028
secondaryDB->db,
@@ -2323,8 +2333,6 @@ DBC_close(DBCursorObject* self, PyObject* args)
23232333
if (!PyArg_ParseTuple(args, ":close"))
23242334
return NULL;
23252335

2326-
CHECK_CURSOR_NOT_CLOSED(self);
2327-
23282336
if (self->dbc != NULL) {
23292337
MYDB_BEGIN_ALLOW_THREADS;
23302338
err = self->dbc->c_close(self->dbc);
@@ -2413,7 +2421,7 @@ DBC_first(DBCursorObject* self, PyObject* args, PyObject* kwargs)
24132421
static PyObject*
24142422
DBC_get(DBCursorObject* self, PyObject* args, PyObject *kwargs)
24152423
{
2416-
int err, flags;
2424+
int err, flags=0;
24172425
PyObject* keyobj = NULL;
24182426
PyObject* dataobj = NULL;
24192427
PyObject* retval = NULL;
@@ -3298,7 +3306,7 @@ DBEnv_lock_stat(DBEnvObject* self, PyObject* args)
32983306
int err;
32993307
DB_LOCK_STAT* sp;
33003308
PyObject* d = NULL;
3301-
u_int32_t flags;
3309+
u_int32_t flags = 0;
33023310

33033311
if (!PyArg_ParseTuple(args, "|i:lock_stat", &flags))
33043312
return NULL;
@@ -3410,7 +3418,7 @@ DBEnv_txn_stat(DBEnvObject* self, PyObject* args)
34103418
int err;
34113419
DB_TXN_STAT* sp;
34123420
PyObject* d = NULL;
3413-
u_int32_t flags;
3421+
u_int32_t flags=0;
34143422

34153423
if (!PyArg_ParseTuple(args, "|i:txn_stat", &flags))
34163424
return NULL;

0 commit comments

Comments
 (0)