Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a60343e

Browse files
serhiy-storchakaclaude
serhiy-storchaka
and
claude
authored
gh-156955: Speed up csv.writer by caching the set of special characters (GH-157298)
Cache in the dialect a 128-bit set of ASCII characters which need quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n' and characters of lineterminator) and a flag whether any of them is non-ASCII. Testing a character is now one bit test instead of five comparisons and a call to PyUnicode_FindChar(). Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent f802d2d commit a60343e

3 files changed

Lines changed: 80 additions & 11 deletions

File tree

‎Lib/test/test_csv.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,18 @@ def test_write_quoting(self):
227227
quoting = csv.QUOTE_STRINGS)
228228
self._write_test(['a','',None,1], '"a","",,"1"',
229229
quoting = csv.QUOTE_NOTNULL)
230+
# FULLWIDTH QUOTATION MARK
231+
self._write_test(['a', 1, 'p,q', 'r＂s', 'x！y'],
232+
'a,1,＂p,q＂,＂r＂＂s＂,x！y',
233+
quotechar='＂')
234+
235+
def test_write_delimiter(self):
236+
self._write_test(['a', 1, 'p,q', 'x;y'], 'a,1,"p,q",x;y')
237+
self._write_test(['a', 1, 'p;q', 'x,y'], 'a;1;"p;q";x,y', delimiter=';')
238+
self._write_test(['a', 1, 'p\0q', 'x,y'], 'a\x001\0"p\0q"\0x,y',
239+
delimiter='\0')
240+
self._write_test(['a', 1, 'p🍌q', 'x🍍y'], 'a🍌1🍌"p🍌q"🍌x🍍y',
241+
delimiter='🍌')
230242

231243
def test_write_escape(self):
232244
self._write_test(['a',1,'p,q'], 'a,1,"p,q"',
@@ -258,19 +270,26 @@ def test_write_escape(self):
258270
escapechar='\\', quoting=csv.QUOTE_MINIMAL)
259271
self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""',
260272
escapechar='\\', quoting=csv.QUOTE_MINIMAL)
273+
# SYMBOL FOR ESCAPE
274+
self._write_test(['a', 1, 'p,q', 'r\u241bs', 'x\u241ay'],
275+
'a,1,p\u241b,q,r\u241b\u241bs,x\u241ay',
276+
escapechar='\u241b', quoting=csv.QUOTE_NONE)
261277

262278
def test_write_lineterminator(self):
263-
for lineterminator in '\r\n', '\n', '\r', '!@#', '\0':
279+
for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0',
280+
'\x85', '\u2028', '\U0001f600'):
264281
with self.subTest(lineterminator=lineterminator):
265282
with StringIO() as sio:
266283
writer = csv.writer(sio, lineterminator=lineterminator)
267284
writer.writerow(['a', 'b'])
268285
writer.writerow([1, 2])
269286
writer.writerow(['\r', '\n'])
287+
writer.writerow([f'a{lineterminator[-1]}b', 'c'])
270288
self.assertEqual(sio.getvalue(),
271289
f'a,b{lineterminator}'
272290
f'1,2{lineterminator}'
273-
f'"\r","\n"{lineterminator}')
291+
f'"\r","\n"{lineterminator}'
292+
f'"a{lineterminator[-1]}b",c{lineterminator}')
274293

275294
def test_write_iterable(self):
276295
self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"')

‎Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Speed up :func:`csv.writer` by caching the set of characters that need
2+
quoting or escaping in the dialect. Writing long fields is now up to 5 times
3+
faster.

‎Modules/_csv.c‎

Lines changed: 56 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,12 @@ typedef struct {
117117
Py_UCS4 quotechar; /* quote character */
118118
Py_UCS4 escapechar; /* escape character */
119119
PyObject *lineterminator; /* string to write between records */
120-
120+
/* Cache for the writer: bit c is set if the ASCII character c needs
121+
quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n'
122+
and the characters of lineterminator). */
123+
uint64_t special_chars[2];
124+
/* Whether any of the special characters is non-ASCII. */
125+
bool nonascii_special;
121126
} DialectObj;
122127

123128
typedef struct {
@@ -332,6 +337,54 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt)
332337
return 0;
333338
}
334339

340+
static void
341+
dialect_add_special_char(DialectObj *self, Py_UCS4 c)
342+
{
343+
if (c == NOT_SET) {
344+
return;
345+
}
346+
if (c < 128) {
347+
self->special_chars[c / 64] |= (uint64_t)1 << (c % 64);
348+
}
349+
else {
350+
self->nonascii_special = true;
351+
}
352+
}
353+
354+
static void
355+
dialect_init_special_chars_cache(DialectObj *self)
356+
{
357+
self->special_chars[0] = self->special_chars[1] = 0;
358+
self->nonascii_special = false;
359+
dialect_add_special_char(self, self->delimiter);
360+
dialect_add_special_char(self, self->quotechar);
361+
dialect_add_special_char(self, self->escapechar);
362+
dialect_add_special_char(self, '\r');
363+
dialect_add_special_char(self, '\n');
364+
PyObject *lt = self->lineterminator;
365+
for (Py_ssize_t i = 0; i < PyUnicode_GET_LENGTH(lt); i++) {
366+
dialect_add_special_char(self, PyUnicode_READ_CHAR(lt, i));
367+
}
368+
}
369+
370+
/* Whether the character needs quoting or escaping by the writer. */
371+
static inline bool
372+
dialect_is_special_char(DialectObj *self, Py_UCS4 c)
373+
{
374+
if (c < 128) {
375+
return (self->special_chars[c / 64] >> (c % 64)) & 1;
376+
}
377+
if (!self->nonascii_special) {
378+
return false;
379+
}
380+
return (c == self->delimiter ||
381+
c == self->quotechar ||
382+
c == self->escapechar ||
383+
PyUnicode_FindChar(self->lineterminator, c, 0,
384+
PyUnicode_GET_LENGTH(self->lineterminator),
385+
1) >= 0);
386+
}
387+
335388
static int
336389
dialect_check_quoting(int quoting)
337390
{
@@ -558,6 +611,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
558611
{
559612
goto err;
560613
}
614+
dialect_init_special_chars_cache(self);
561615

562616
ret = Py_NewRef(self);
563617
err:
@@ -1208,14 +1262,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data,
12081262
Py_UCS4 c = PyUnicode_READ(field_kind, field_data, i);
12091263
int want_escape = 0;
12101264

1211-
if (c == dialect->delimiter ||
1212-
c == dialect->escapechar ||
1213-
c == dialect->quotechar ||
1214-
c == '\n' ||
1215-
c == '\r' ||
1216-
PyUnicode_FindChar(
1217-
dialect->lineterminator, c, 0,
1218-
PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) {
1265+
if (dialect_is_special_char(dialect, c)) {
12191266
if (dialect->quoting == QUOTE_NONE)
12201267
want_escape = 1;
12211268
else {

0 commit comments

Comments
 (0)