Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4f199ea

Browse files
gvanrossumgvanrossum
committed
Threading speedup patches by Christopher Lindblad <[email protected]>.
1 parent b5903ac commit 4f199ea

2 files changed

Lines changed: 192 additions & 24 deletions

File tree

‎Modules/bsddbmodule.c‎

Lines changed: 148 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ PERFORMANCE OF THIS SOFTWARE.
4141
*/
4242

4343
#include "Python.h"
44+
#ifdef WITH_THREAD
45+
#include "thread.h"
46+
#endif
4447

4548
#include <sys/types.h>
4649
#include <sys/stat.h>
@@ -53,6 +56,9 @@ typedef struct {
5356
PyObject_HEAD
5457
DB *di_bsddb;
5558
int di_size; /* -1 means recompute */
59+
#ifdef WITH_THREAD
60+
type_lock di_lock;
61+
#endif
5662
} bsddbobject;
5763

5864
staticforward PyTypeObject Bsddbtype;
@@ -93,14 +99,27 @@ newdbhashobject(file, flags, mode,
9399
#ifdef O_BINARY
94100
flags |= O_BINARY;
95101
#endif
96-
if ((dp->di_bsddb = dbopen(file, flags,
97-
mode, DB_HASH, &info)) == NULL) {
102+
Py_BEGIN_ALLOW_THREADS
103+
dp->di_bsddb = dbopen(file, flags, mode, DB_HASH, &info);
104+
Py_END_ALLOW_THREADS
105+
if (dp->di_bsddb == NULL) {
98106
PyErr_SetFromErrno(BsddbError);
107+
#ifdef WITH_THREAD
108+
dp->di_lock = NULL;
109+
#endif
99110
Py_DECREF(dp);
100111
return NULL;
101112
}
102113

103114
dp->di_size = -1;
115+
#ifdef WITH_THREAD
116+
dp->di_lock = allocate_lock();
117+
if (dp->di_lock == NULL) {
118+
PyErr_SetString(BsddbError, "can't allocate lock");
119+
Py_DECREF(dp);
120+
return NULL;
121+
}
122+
#endif
104123

105124
return (PyObject *)dp;
106125
}
@@ -136,14 +155,27 @@ newdbbtobject(file, flags, mode,
136155
#ifdef O_BINARY
137156
flags |= O_BINARY;
138157
#endif
139-
if ((dp->di_bsddb = dbopen(file, flags,
140-
mode, DB_BTREE, &info)) == NULL) {
158+
Py_BEGIN_ALLOW_THREADS
159+
dp->di_bsddb = dbopen(file, flags, mode, DB_BTREE, &info);
160+
Py_END_ALLOW_THREADS
161+
if (dp->di_bsddb == NULL) {
141162
PyErr_SetFromErrno(BsddbError);
163+
#ifdef WITH_THREAD
164+
dp->di_lock = NULL;
165+
#endif
142166
Py_DECREF(dp);
143167
return NULL;
144168
}
145169

146170
dp->di_size = -1;
171+
#ifdef WITH_THREAD
172+
dp->di_lock = allocate_lock();
173+
if (dp->di_lock == NULL) {
174+
PyErr_SetString(BsddbError, "can't allocate lock");
175+
Py_DECREF(dp);
176+
return NULL;
177+
}
178+
#endif
147179

148180
return (PyObject *)dp;
149181
}
@@ -179,32 +211,64 @@ newdbrnobject(file, flags, mode,
179211
#ifdef O_BINARY
180212
flags |= O_BINARY;
181213
#endif
182-
if ((dp->di_bsddb = dbopen(file, flags, mode,
183-
DB_RECNO, &info)) == NULL) {
214+
Py_BEGIN_ALLOW_THREADS
215+
dp->di_bsddb = dbopen(file, flags, mode, DB_RECNO, &info);
216+
Py_END_ALLOW_THREADS
217+
if (dp->di_bsddb == NULL) {
184218
PyErr_SetFromErrno(BsddbError);
219+
#ifdef WITH_THREAD
220+
dp->di_lock = NULL;
221+
#endif
185222
Py_DECREF(dp);
186223
return NULL;
187224
}
188225

189226
dp->di_size = -1;
227+
#ifdef WITH_THREAD
228+
dp->di_lock = allocate_lock();
229+
if (dp->di_lock == NULL) {
230+
PyErr_SetString(BsddbError, "can't allocate lock");
231+
Py_DECREF(dp);
232+
return NULL;
233+
}
234+
#endif
190235

191236
return (PyObject *)dp;
192237
}
193238

194-
195239
static void
196240
bsddb_dealloc(dp)
197241
bsddbobject *dp;
198242
{
243+
#ifdef WITH_THREAD
244+
if (dp->di_lock) {
245+
acquire_lock(dp->di_lock, 0);
246+
release_lock(dp->di_lock);
247+
free_lock(dp->di_lock);
248+
dp->di_lock = NULL;
249+
}
250+
#endif
199251
if (dp->di_bsddb != NULL) {
200-
if ((dp->di_bsddb->close)(dp->di_bsddb) != 0)
252+
int status;
253+
Py_BEGIN_ALLOW_THREADS
254+
status = (dp->di_bsddb->close)(dp->di_bsddb);
255+
Py_END_ALLOW_THREADS
256+
if (status != 0)
201257
fprintf(stderr,
202258
"Python bsddb: close errno %d in dealloc\n",
203259
errno);
204260
}
205261
PyMem_DEL(dp);
206262
}
207263

264+
#ifdef WITH_THREAD
265+
#define BSDDB_BGN_SAVE(_dp) Py_BEGIN_ALLOW_THREADS acquire_lock(_dp->di_lock,1);
266+
#define BSDDB_END_SAVE(_dp) release_lock(_dp->di_lock); Py_END_ALLOW_THREADS
267+
#else
268+
#define BSDDB_BGN_SAVE(_dp) Py_BEGIN_ALLOW_THREADS
269+
#define BSDDB_END_SAVE(_dp) Py_END_ALLOW_THREADS
270+
#endif
271+
208272
static int
209273
bsddb_length(dp)
210274
bsddbobject *dp;
@@ -217,12 +281,14 @@ bsddb_length(dp)
217281
DBT krec, drec;
218282
int status;
219283
int size = 0;
284+
BSDDB_BGN_SAVE(dp)
220285
for (status = (dp->di_bsddb->seq)(dp->di_bsddb,
221286
&krec, &drec,R_FIRST);
222287
status == 0;
223288
status = (dp->di_bsddb->seq)(dp->di_bsddb,
224289
&krec, &drec, R_NEXT))
225290
size++;
291+
BSDDB_END_SAVE(dp)
226292
if (status < 0) {
227293
PyErr_SetFromErrno(BsddbError);
228294
return -1;
@@ -239,8 +305,9 @@ bsddb_subscript(dp, key)
239305
{
240306
int status;
241307
DBT krec, drec;
242-
char *data;
308+
char *data,buf[4096];
243309
int size;
310+
PyObject *result;
244311

245312
if (!PyArg_Parse(key, "s#", &data, &size))
246313
return NULL;
@@ -249,7 +316,14 @@ bsddb_subscript(dp, key)
249316
krec.data = data;
250317
krec.size = size;
251318

319+
BSDDB_BGN_SAVE(dp)
252320
status = (dp->di_bsddb->get)(dp->di_bsddb, &krec, &drec, 0);
321+
if (status == 0) {
322+
if (drec.size > sizeof(buf)) data = malloc(drec.size);
323+
else data = buf;
324+
memcpy(data,drec.data,drec.size);
325+
}
326+
BSDDB_END_SAVE(dp)
253327
if (status != 0) {
254328
if (status < 0)
255329
PyErr_SetFromErrno(BsddbError);
@@ -258,7 +332,9 @@ bsddb_subscript(dp, key)
258332
return NULL;
259333
}
260334

261-
return PyString_FromStringAndSize((char *)drec.data, (int)drec.size);
335+
result = PyString_FromStringAndSize(data, (int)drec.size);
336+
if (data != buf) free(data);
337+
return result;
262338
}
263339

264340
static int
@@ -284,7 +360,9 @@ bsddb_ass_sub(dp, key, value)
284360
krec.size = size;
285361
dp->di_size = -1;
286362
if (value == NULL) {
363+
BSDDB_BGN_SAVE(dp)
287364
status = (dp->di_bsddb->del)(dp->di_bsddb, &krec, 0);
365+
BSDDB_END_SAVE(dp)
288366
}
289367
else {
290368
if (!PyArg_Parse(value, "s#", &data, &size)) {
@@ -305,7 +383,9 @@ bsddb_ass_sub(dp, key, value)
305383
printf("before put key= '%s', size= %d\n",
306384
krec.data, krec.size);
307385
#endif
386+
BSDDB_BGN_SAVE(dp)
308387
status = (dp->di_bsddb->put)(dp->di_bsddb, &krec, &drec, 0);
388+
BSDDB_END_SAVE(dp)
309389
}
310390
if (status != 0) {
311391
if (status < 0)
@@ -331,7 +411,11 @@ bsddb_close(dp, args)
331411
if (!PyArg_NoArgs(args))
332412
return NULL;
333413
if (dp->di_bsddb != NULL) {
334-
if ((dp->di_bsddb->close)(dp->di_bsddb) != 0) {
414+
int status;
415+
BSDDB_BGN_SAVE(dp)
416+
status = (dp->di_bsddb->close)(dp->di_bsddb);
417+
BSDDB_END_SAVE(dp)
418+
if (status != 0) {
335419
dp->di_bsddb = NULL;
336420
PyErr_SetFromErrno(BsddbError);
337421
return NULL;
@@ -349,6 +433,7 @@ bsddb_keys(dp, args)
349433
{
350434
PyObject *list, *item;
351435
DBT krec, drec;
436+
char *data,buf[4096];
352437
int status;
353438
int err;
354439

@@ -358,12 +443,17 @@ bsddb_keys(dp, args)
358443
list = PyList_New(0);
359444
if (list == NULL)
360445
return NULL;
361-
for (status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_FIRST);
362-
status == 0;
363-
status = (dp->di_bsddb->seq)(dp->di_bsddb,
364-
&krec, &drec, R_NEXT)) {
365-
item = PyString_FromStringAndSize((char *)krec.data,
366-
(int)krec.size);
446+
BSDDB_BGN_SAVE(dp)
447+
status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_FIRST);
448+
if (status == 0) {
449+
if (krec.size > sizeof(buf)) data = malloc(krec.size);
450+
else data = buf;
451+
memcpy(data,krec.data,krec.size);
452+
}
453+
BSDDB_END_SAVE(dp)
454+
while (status == 0) {
455+
item = PyString_FromStringAndSize(data, (int)krec.size);
456+
if (data != buf) free(data);
367457
if (item == NULL) {
368458
Py_DECREF(list);
369459
return NULL;
@@ -374,6 +464,14 @@ bsddb_keys(dp, args)
374464
Py_DECREF(list);
375465
return NULL;
376466
}
467+
BSDDB_BGN_SAVE(dp)
468+
status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_NEXT);
469+
if (status == 0) {
470+
if (krec.size > sizeof(buf)) data = malloc(krec.size);
471+
else data = buf;
472+
memcpy(data,krec.data,krec.size);
473+
}
474+
BSDDB_END_SAVE(dp)
377475
}
378476
if (status < 0) {
379477
PyErr_SetFromErrno(BsddbError);
@@ -401,7 +499,9 @@ bsddb_has_key(dp, args)
401499
krec.data = data;
402500
krec.size = size;
403501

502+
BSDDB_BGN_SAVE(dp)
404503
status = (dp->di_bsddb->get)(dp->di_bsddb, &krec, &drec, 0);
504+
BSDDB_END_SAVE(dp)
405505
if (status < 0) {
406506
PyErr_SetFromErrno(BsddbError);
407507
return NULL;
@@ -417,16 +517,24 @@ bsddb_set_location(dp, key)
417517
{
418518
int status;
419519
DBT krec, drec;
420-
char *data;
520+
char *data,buf[4096];
421521
int size;
522+
PyObject *result;
422523

423524
if (!PyArg_Parse(key, "s#", &data, &size))
424525
return NULL;
425526
check_bsddbobject_open(dp);
426527
krec.data = data;
427528
krec.size = size;
428529

530+
BSDDB_BGN_SAVE(dp)
429531
status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_CURSOR);
532+
if (status == 0) {
533+
if (drec.size > sizeof(buf)) data = malloc(drec.size);
534+
else data = buf;
535+
memcpy(data,drec.data,drec.size);
536+
}
537+
BSDDB_END_SAVE(dp)
430538
if (status != 0) {
431539
if (status < 0)
432540
PyErr_SetFromErrno(BsddbError);
@@ -435,8 +543,9 @@ bsddb_set_location(dp, key)
435543
return NULL;
436544
}
437545

438-
return Py_BuildValue("s#s#", krec.data, krec.size,
439-
drec.data, drec.size);
546+
result = Py_BuildValue("s#s#", krec.data, krec.size, data, drec.size);
547+
if (data != buf) free(data);
548+
return result;
440549
}
441550

442551
static PyObject *
@@ -447,6 +556,9 @@ bsddb_seq(dp, args, sequence_request)
447556
{
448557
int status;
449558
DBT krec, drec;
559+
char *kdata,kbuf[4096];
560+
char *ddata,dbuf[4096];
561+
PyObject *result;
450562

451563
if (!PyArg_NoArgs(args))
452564
return NULL;
@@ -455,8 +567,18 @@ bsddb_seq(dp, args, sequence_request)
455567
krec.data = 0;
456568
krec.size = 0;
457569

570+
BSDDB_BGN_SAVE(dp)
458571
status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec,
459572
&drec, sequence_request);
573+
if (status == 0) {
574+
if (krec.size > sizeof(kbuf)) kdata = malloc(krec.size);
575+
else kdata = kbuf;
576+
memcpy(kdata,krec.data,krec.size);
577+
if (drec.size > sizeof(dbuf)) ddata = malloc(drec.size);
578+
else ddata = dbuf;
579+
memcpy(ddata,drec.data,drec.size);
580+
}
581+
BSDDB_END_SAVE(dp)
460582
if (status != 0) {
461583
if (status < 0)
462584
PyErr_SetFromErrno(BsddbError);
@@ -465,8 +587,10 @@ bsddb_seq(dp, args, sequence_request)
465587
return NULL;
466588
}
467589

468-
return Py_BuildValue("s#s#", krec.data, krec.size,
469-
drec.data, drec.size);
590+
result = Py_BuildValue("s#s#", kdata, krec.size, ddata, drec.size);
591+
if (kdata != kbuf) free(kdata);
592+
if (ddata != dbuf) free(ddata);
593+
return result;
470594
}
471595

472596
static PyObject *
@@ -507,7 +631,9 @@ bsddb_sync(dp, args)
507631
if (!PyArg_NoArgs(args))
508632
return NULL;
509633
check_bsddbobject_open(dp);
634+
BSDDB_BGN_SAVE(dp)
510635
status = (dp->di_bsddb->sync)(dp->di_bsddb, 0);
636+
BSDDB_END_SAVE(dp)
511637
if (status != 0) {
512638
PyErr_SetFromErrno(BsddbError);
513639
return NULL;

0 commit comments

Comments
 (0)