/* csv module */

/*

This module provides the low-level underpinnings of a CSV reading/writing

module. Users should not use this module directly, but import the csv.py

module instead.

*/

// clinic/_csv.c.h uses internal pycore_modsupport.h API

#ifndef Py_BUILD_CORE_BUILTIN

# define Py_BUILD_CORE_MODULE 1

#endif

#include "Python.h"

#include "pycore_pyatomic_ft_wrappers.h"

#include <stddef.h> // offsetof()

#include <stdbool.h>

/*[clinic input]

module _csv

[clinic start generated code]*/

/*[clinic end generated code: output=da39a3ee5e6b4b0d input=385118b71aa43706]*/

#include "clinic/_csv.c.h"

#define NOT_SET ((Py_UCS4)-1)

#define EOL ((Py_UCS4)-2)

typedef struct {

PyObject *error_obj; /* CSV exception */

PyObject *dialects; /* Dialect registry */

PyTypeObject *dialect_type;

PyTypeObject *reader_type;

PyTypeObject *writer_type;

Py_ssize_t field_limit; /* max parsed field size */

PyObject *str_write;

} _csvstate;

static struct PyModuleDef _csvmodule;

static inline _csvstate*

get_csv_state(PyObject *module)

{

void *state = PyModule_GetState(module);

assert(state != NULL);

return (_csvstate *)state;

}

static int

_csv_clear(PyObject *module)

{

_csvstate *module_state = PyModule_GetState(module);

Py_CLEAR(module_state->error_obj);

Py_CLEAR(module_state->dialects);

Py_CLEAR(module_state->dialect_type);

Py_CLEAR(module_state->reader_type);

Py_CLEAR(module_state->writer_type);

Py_CLEAR(module_state->str_write);

return 0;

}

static int

_csv_traverse(PyObject *module, visitproc visit, void *arg)

{

_csvstate *module_state = PyModule_GetState(module);

Py_VISIT(module_state->error_obj);

Py_VISIT(module_state->dialects);

Py_VISIT(module_state->dialect_type);

Py_VISIT(module_state->reader_type);

Py_VISIT(module_state->writer_type);

return 0;

}

static void

_csv_free(void *module)

{

(void)_csv_clear((PyObject *)module);

}

typedef enum {

START_RECORD, START_FIELD, ESCAPED_CHAR, IN_FIELD,

IN_QUOTED_FIELD, ESCAPE_IN_QUOTED_FIELD, QUOTE_IN_QUOTED_FIELD,

EAT_CRNL,AFTER_ESCAPED_CRNL

} ParserState;

typedef enum {

QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE,

QUOTE_STRINGS, QUOTE_NOTNULL

} QuoteStyle;

typedef struct {

QuoteStyle style;

const char *name;

} StyleDesc;

static const StyleDesc quote_styles[] = {

{ QUOTE_MINIMAL, "QUOTE_MINIMAL" },

{ QUOTE_ALL, "QUOTE_ALL" },

{ QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" },

{ QUOTE_NONE, "QUOTE_NONE" },

{ QUOTE_STRINGS, "QUOTE_STRINGS" },

{ QUOTE_NOTNULL, "QUOTE_NOTNULL" },

{ 0 }

};

typedef struct {

PyObject_HEAD

char doublequote; /* is " represented by ""? */

char skipinitialspace; /* ignore spaces following delimiter? */

char strict; /* raise exception on bad CSV */

int quoting; /* style of quoting to write */

Py_UCS4 delimiter; /* field separator */

Py_UCS4 quotechar; /* quote character */

Py_UCS4 escapechar; /* escape character */

PyObject *lineterminator; /* string to write between records */

} DialectObj;

typedef struct {

PyObject_HEAD

PyObject *input_iter; /* iterate over this for input lines */

DialectObj *dialect; /* parsing dialect */

PyObject *fields; /* field list for current record */

ParserState state; /* current CSV parse state */

Py_UCS4 *field; /* temporary buffer */

Py_ssize_t field_size; /* size of allocated buffer */

Py_ssize_t field_len; /* length of current field */

bool unquoted_field; /* true if no quotes around the current field */

unsigned long line_num; /* Source-file line number */

} ReaderObj;

typedef struct {

PyObject_HEAD

PyObject *write; /* write output lines to this file */

DialectObj *dialect; /* parsing dialect */

Py_UCS4 *rec; /* buffer for parser.join */

Py_ssize_t rec_size; /* size of allocated record */

Py_ssize_t rec_len; /* length of record */

int num_fields; /* number of fields in record */

PyObject *error_obj; /* cached error object */

} WriterObj;

#define _DialectObj_CAST(op) ((DialectObj *)(op))

#define _ReaderObj_CAST(op) ((ReaderObj *)(op))

#define _WriterObj_CAST(op) ((WriterObj *)(op))

/*

* DIALECT class

*/

static PyObject *

get_dialect_from_registry(PyObject *name_obj, _csvstate *module_state)

{

PyObject *dialect_obj;

if (PyDict_GetItemRef(module_state->dialects, name_obj, &dialect_obj) == 0) {

PyErr_SetString(module_state->error_obj, "unknown dialect");

}

return dialect_obj;

}

static PyObject *

get_char_or_None(Py_UCS4 c)

{

if (c == NOT_SET) {

Py_RETURN_NONE;

}

else

return PyUnicode_FromOrdinal(c);

}

static PyObject *

Dialect_get_lineterminator(PyObject *op, void *Py_UNUSED(ignored))

{

DialectObj *self = _DialectObj_CAST(op);

return Py_XNewRef(self->lineterminator);

}

static PyObject *

Dialect_get_delimiter(PyObject *op, void *Py_UNUSED(ignored))

{

DialectObj *self = _DialectObj_CAST(op);

return get_char_or_None(self->delimiter);

}

static PyObject *

Dialect_get_escapechar(PyObject *op, void *Py_UNUSED(ignored))

{

DialectObj *self = _DialectObj_CAST(op);

return get_char_or_None(self->escapechar);

}

static PyObject *

Dialect_get_quotechar(PyObject *op, void *Py_UNUSED(ignored))

{

DialectObj *self = _DialectObj_CAST(op);

return get_char_or_None(self->quotechar);

}

static PyObject *

Dialect_get_quoting(PyObject *op, void *Py_UNUSED(ignored))

{

DialectObj *self = _DialectObj_CAST(op);

return PyLong_FromLong(self->quoting);

}

static int

_set_bool(const char *name, char *target, PyObject *src, bool dflt)

{

if (src == NULL)

*target = dflt;

else {

int b = PyObject_IsTrue(src);

if (b < 0)

return -1;

*target = (char)b;

}

return 0;

}

static int

_set_int(const char *name, int *target, PyObject *src, int dflt)

{

if (src == NULL)

*target = dflt;

else {

int value;

if (!PyLong_CheckExact(src)) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be an integer, not %T", name, src);

return -1;

}

value = PyLong_AsInt(src);

if (value == -1 && PyErr_Occurred()) {

return -1;

}

*target = value;

}

return 0;

}

static int

_set_char_or_none(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt)

{

if (src == NULL) {

*target = dflt;

}

else if (src == Py_None) {

*target = NOT_SET;

}

else {

// similar to PyArg_Parse("C?")

if (!PyUnicode_Check(src)) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be a unicode character or None, not %T",

name, src);

return -1;

}

Py_ssize_t len = PyUnicode_GetLength(src);

if (len < 0) {

return -1;

}

if (len != 1) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be a unicode character or None, "

"not a string of length %zd",

name, len);

return -1;

}

*target = PyUnicode_READ_CHAR(src, 0);

}

return 0;

}

static int

_set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt)

{

if (src == NULL) {

*target = dflt;

}

else {

// similar to PyArg_Parse("C")

if (!PyUnicode_Check(src)) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be a unicode character, not %T",

name, src);

return -1;

}

Py_ssize_t len = PyUnicode_GetLength(src);

if (len < 0) {

return -1;

}

if (len != 1) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be a unicode character, "

"not a string of length %zd",

name, len);

return -1;

}

*target = PyUnicode_READ_CHAR(src, 0);

}

return 0;

}

static int

_set_str(const char *name, PyObject **target, PyObject *src, const char *dflt)

{

if (src == NULL) {

*target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL);

if (*target == NULL) {

return -1;

}

}

else {

if (!PyUnicode_Check(src)) {

PyErr_Format(PyExc_TypeError,

"\"%s\" must be a string, not %T", name, src);

return -1;

}

Py_XSETREF(*target, Py_NewRef(src));

}

return 0;

}

static int

dialect_check_quoting(int quoting)

{

const StyleDesc *qs;

for (qs = quote_styles; qs->name; qs++) {

if ((int)qs->style == quoting)

return 0;

}

PyErr_Format(PyExc_TypeError, "bad \"quoting\" value");

return -1;

}

static int

dialect_check_char(const char *name, Py_UCS4 c, DialectObj *dialect, bool allowspace)

{

if (c == '\r' || c == '\n' || (c == ' ' && !allowspace)) {

PyErr_Format(PyExc_ValueError, "bad %s value", name);

return -1;

}

if (PyUnicode_FindChar(

dialect->lineterminator, c, 0,

PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0)

{

PyErr_Format(PyExc_ValueError, "bad %s or lineterminator value", name);

return -1;

}

return 0;

}

static int

dialect_check_chars(const char *name1, const char *name2, Py_UCS4 c1, Py_UCS4 c2)

{

if (c1 == c2 && c1 != NOT_SET) {

PyErr_Format(PyExc_ValueError, "bad %s or %s value", name1, name2);

return -1;

}

return 0;

}

#define D_OFF(x) offsetof(DialectObj, x)

static struct PyMemberDef Dialect_memberlist[] = {

{ "skipinitialspace", Py_T_BOOL, D_OFF(skipinitialspace), Py_READONLY },

{ "doublequote", Py_T_BOOL, D_OFF(doublequote), Py_READONLY },

{ "strict", Py_T_BOOL, D_OFF(strict), Py_READONLY },

{ NULL }

};

#undef D_OFF

static PyGetSetDef Dialect_getsetlist[] = {

{"delimiter", Dialect_get_delimiter},

{"escapechar", Dialect_get_escapechar},

{"lineterminator", Dialect_get_lineterminator},

{"quotechar", Dialect_get_quotechar},

{"quoting", Dialect_get_quoting},

{NULL},

};

static void

Dialect_dealloc(PyObject *self)

{

PyTypeObject *tp = Py_TYPE(self);

PyObject_GC_UnTrack(self);

tp->tp_clear((PyObject *)self);

PyObject_GC_Del(self);

Py_DECREF(tp);

}

static char *dialect_kws[] = {

"dialect",

"delimiter",

"doublequote",

"escapechar",

"lineterminator",

"quotechar",

"quoting",

"skipinitialspace",

"strict",

NULL

};

static _csvstate *

_csv_state_from_type(PyTypeObject *type, const char *name)

{

PyObject *module = PyType_GetModuleByDef(type, &_csvmodule);

if (module == NULL) {

return NULL;

}

_csvstate *module_state = PyModule_GetState(module);

if (module_state == NULL) {

PyErr_Format(PyExc_SystemError,

"%s: No _csv module state found", name);

return NULL;

}

return module_state;

}

static PyObject *

dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)

{

DialectObj *self;

PyObject *ret = NULL;

PyObject *dialect = NULL;

PyObject *delimiter = NULL;

PyObject *doublequote = NULL;

PyObject *escapechar = NULL;

PyObject *lineterminator = NULL;

PyObject *quotechar = NULL;

PyObject *quoting = NULL;

PyObject *skipinitialspace = NULL;

PyObject *strict = NULL;

if (!PyArg_ParseTupleAndKeywords(args, kwargs,

"|OOOOOOOOO", dialect_kws,

&dialect,

&delimiter,

&doublequote,

&escapechar,

&lineterminator,

&quotechar,

&quoting,

&skipinitialspace,

&strict))

return NULL;

_csvstate *module_state = _csv_state_from_type(type, "dialect_new");

if (module_state == NULL) {

return NULL;

}

if (dialect != NULL) {

if (PyUnicode_Check(dialect)) {

dialect = get_dialect_from_registry(dialect, module_state);

if (dialect == NULL)

return NULL;

}

else

Py_INCREF(dialect);

/* Can we reuse this instance? */

if (PyObject_TypeCheck(dialect, module_state->dialect_type) &&

delimiter == NULL &&

doublequote == NULL &&

escapechar == NULL &&

lineterminator == NULL &&

quotechar == NULL &&

quoting == NULL &&

skipinitialspace == NULL &&

strict == NULL)

return dialect;

}

self = (DialectObj *)type->tp_alloc(type, 0);

if (self == NULL) {

Py_CLEAR(dialect);

return NULL;

}

self->lineterminator = NULL;

Py_XINCREF(delimiter);

Py_XINCREF(doublequote);

Py_XINCREF(escapechar);

Py_XINCREF(lineterminator);

Py_XINCREF(quotechar);

Py_XINCREF(quoting);

Py_XINCREF(skipinitialspace);

Py_XINCREF(strict);

if (dialect != NULL) {

#define DIALECT_GETATTR(v, n) \

do { \

if (v == NULL) { \

if (PyObject_GetOptionalAttrString(dialect, n, &v) < 0) { \

goto err; \

} \

} \

} while (0)

DIALECT_GETATTR(delimiter, "delimiter");

DIALECT_GETATTR(doublequote, "doublequote");

DIALECT_GETATTR(escapechar, "escapechar");

DIALECT_GETATTR(lineterminator, "lineterminator");

DIALECT_GETATTR(quotechar, "quotechar");

DIALECT_GETATTR(quoting, "quoting");

DIALECT_GETATTR(skipinitialspace, "skipinitialspace");

DIALECT_GETATTR(strict, "strict");

}

#undef DIALECT_GETATTR

/* check types and convert to C values */

#define DIASET(meth, name, target, src, dflt) \

if (meth(name, target, src, dflt)) \

goto err

DIASET(_set_char, "delimiter", &self->delimiter, delimiter, ',');

DIASET(_set_bool, "doublequote", &self->doublequote, doublequote, true);

DIASET(_set_char_or_none, "escapechar", &self->escapechar, escapechar, NOT_SET);

DIASET(_set_str, "lineterminator", &self->lineterminator, lineterminator, "\r\n");

DIASET(_set_char_or_none, "quotechar", &self->quotechar, quotechar, '"');

DIASET(_set_int, "quoting", &self->quoting, quoting, QUOTE_MINIMAL);

DIASET(_set_bool, "skipinitialspace", &self->skipinitialspace, skipinitialspace, false);

DIASET(_set_bool, "strict", &self->strict, strict, false);

#undef DIASET

/* validate options */

if (dialect_check_quoting(self->quoting))

goto err;

if (quotechar == Py_None && quoting == NULL)

self->quoting = QUOTE_NONE;

if (self->quoting != QUOTE_NONE && self->quotechar == NOT_SET) {

PyErr_SetString(PyExc_TypeError,

"quotechar must be set if quoting enabled");

goto err;

}

if (dialect_check_char("delimiter", self->delimiter, self, true) ||

dialect_check_char("escapechar", self->escapechar, self,

!self->skipinitialspace) ||

dialect_check_char("quotechar", self->quotechar, self,

!self->skipinitialspace) ||

dialect_check_chars("delimiter", "escapechar",

self->delimiter, self->escapechar) ||

dialect_check_chars("delimiter", "quotechar",

self->delimiter, self->quotechar) ||

dialect_check_chars("escapechar", "quotechar",

self->escapechar, self->quotechar))

{

goto err;

}

ret = Py_NewRef(self);

err:

Py_CLEAR(self);

Py_CLEAR(dialect);

Py_CLEAR(delimiter);

Py_CLEAR(doublequote);

Py_CLEAR(escapechar);

Py_CLEAR(lineterminator);

Py_CLEAR(quotechar);

Py_CLEAR(quoting);

Py_CLEAR(skipinitialspace);

Py_CLEAR(strict);

return ret;

}

/* Since dialect is now a heap type, it inherits pickling method for

* protocol 0 and 1 from object, therefore it needs to be overridden */

PyDoc_STRVAR(dialect_reduce_doc, "raises an exception to avoid pickling");

static PyObject *

Dialect_reduce(PyObject *self, PyObject *args) {

PyErr_Format(PyExc_TypeError,

"cannot pickle '%.100s' instances", _PyType_Name(Py_TYPE(self)));

return NULL;

}

static struct PyMethodDef dialect_methods[] = {

{"__reduce__", Dialect_reduce, METH_VARARGS, dialect_reduce_doc},

{"__reduce_ex__", Dialect_reduce, METH_VARARGS, dialect_reduce_doc},

{NULL, NULL}

};

PyDoc_STRVAR(Dialect_Type_doc,

"CSV dialect\n"

"\n"

"The Dialect type records CSV parsing and generation options.\n");

static int

Dialect_clear(PyObject *op)

{

DialectObj *self = _DialectObj_CAST(op);

Py_CLEAR(self->lineterminator);

return 0;

}

static int

Dialect_traverse(PyObject *op, visitproc visit, void *arg)

{

DialectObj *self = _DialectObj_CAST(op);

Py_VISIT(self->lineterminator);

Py_VISIT(Py_TYPE(self));

return 0;

}

static PyType_Slot Dialect_Type_slots[] = {

{Py_tp_doc, (char*)Dialect_Type_doc},

{Py_tp_members, Dialect_memberlist},

{Py_tp_getset, Dialect_getsetlist},

{Py_tp_new, dialect_new},

{Py_tp_methods, dialect_methods},

{Py_tp_dealloc, Dialect_dealloc},

{Py_tp_clear, Dialect_clear},

{Py_tp_traverse, Dialect_traverse},

{0, NULL}

};

PyType_Spec Dialect_Type_spec = {

.name = "_csv.Dialect",

.basicsize = sizeof(DialectObj),

.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC |

Py_TPFLAGS_IMMUTABLETYPE),

.slots = Dialect_Type_slots,

};

/*

* Return an instance of the dialect type, given a Python instance or kwarg

* description of the dialect

*/

static PyObject *

_call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs)

{

PyObject *type = (PyObject *)module_state->dialect_type;

if (dialect_inst) {

return PyObject_VectorcallDict(type, &dialect_inst, 1, kwargs);

}

else {

return PyObject_VectorcallDict(type, NULL, 0, kwargs);

}

}

/*

* READER

*/

static int

parse_save_field(ReaderObj *self)

{

int quoting = self->dialect->quoting;

PyObject *field;

if (self->unquoted_field &&

self->field_len == 0 &&

(quoting == QUOTE_NOTNULL || quoting == QUOTE_STRINGS))

{

field = Py_NewRef(Py_None);

}

else {

field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,

(void *) self->field, self->field_len);

if (field == NULL) {

return -1;

}

if (self->unquoted_field &&

self->field_len != 0 &&

(quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS))

{

PyObject *tmp = PyNumber_Float(field);

Py_DECREF(field);

if (tmp == NULL) {

return -1;

}

field = tmp;

}

self->field_len = 0;

}

if (PyList_Append(self->fields, field) < 0) {

Py_DECREF(field);

return -1;

}

Py_DECREF(field);

return 0;

}

static int

parse_grow_buff(ReaderObj *self)

{

assert((size_t)self->field_size <= PY_SSIZE_T_MAX / sizeof(Py_UCS4));

Py_ssize_t field_size_new = self->field_size ? 2 * self->field_size : 4096;

Py_UCS4 *field_new = self->field;

PyMem_Resize(field_new, Py_UCS4, field_size_new);

if (field_new == NULL) {

PyErr_NoMemory();

return 0;

}

self->field = field_new;

self->field_size = field_size_new;

return 1;

}

static int

parse_add_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)

{

Py_ssize_t field_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit);

if (self->field_len >= field_limit) {

PyErr_Format(module_state->error_obj,

"field larger than field limit (%zd)",

field_limit);

return -1;

}

if (self->field_len == self->field_size && !parse_grow_buff(self))

return -1;

self->field[self->field_len++] = c;

return 0;

}

static int

parse_process_char(ReaderObj *self, _csvstate *module_state, Py_UCS4 c)

{

DialectObj *dialect = self->dialect;

switch (self->state) {

case START_RECORD:

/* start of record */

if (c == EOL)

/* empty line - return [] */

break;

else if (c == '\n' || c == '\r') {

self->state = EAT_CRNL;

break;

}

/* normal character - handle as START_FIELD */

self->state = START_FIELD;

_Py_FALLTHROUGH;

case START_FIELD:

/* expecting field */

self->unquoted_field = true;

if (c == '\n' || c == '\r' || c == EOL) {

/* save empty field - return [fields] */

if (parse_save_field(self) < 0)

return -1;

self->state = (c == EOL ? START_RECORD : EAT_CRNL);

}

else if (c == dialect->quotechar &&

dialect->quoting != QUOTE_NONE) {

/* start quoted field */

self->unquoted_field = false;

self->state = IN_QUOTED_FIELD;

}

else if (c == dialect->escapechar) {

/* possible escaped character */

self->state = ESCAPED_CHAR;

}

else if (c == ' ' && dialect->skipinitialspace)

/* ignore spaces at start of field */

;

else if (c == dialect->delimiter) {

/* save empty field */

if (parse_save_field(self) < 0)

return -1;

}

else {

/* begin new unquoted field */

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = IN_FIELD;

}

break;

case ESCAPED_CHAR:

if (c == '\n' || c=='\r') {

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = AFTER_ESCAPED_CRNL;

break;

}

if (c == EOL)

c = '\n';

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = IN_FIELD;

break;

case AFTER_ESCAPED_CRNL:

if (c == EOL)

break;

_Py_FALLTHROUGH;

case IN_FIELD:

/* in unquoted field */

if (c == '\n' || c == '\r' || c == EOL) {

/* end of line - return [fields] */

if (parse_save_field(self) < 0)

return -1;

self->state = (c == EOL ? START_RECORD : EAT_CRNL);

}

else if (c == dialect->escapechar) {

/* possible escaped character */

self->state = ESCAPED_CHAR;

}

else if (c == dialect->delimiter) {

/* save field - wait for new field */

if (parse_save_field(self) < 0)

return -1;

self->state = START_FIELD;

}

else {

/* normal character - save in field */

if (parse_add_char(self, module_state, c) < 0)

return -1;

}

break;

case IN_QUOTED_FIELD:

/* in quoted field */

if (c == EOL)

;

else if (c == dialect->escapechar) {

/* Possible escape character */

self->state = ESCAPE_IN_QUOTED_FIELD;

}

else if (c == dialect->quotechar &&

dialect->quoting != QUOTE_NONE) {

if (dialect->doublequote) {

/* doublequote; " represented by "" */

self->state = QUOTE_IN_QUOTED_FIELD;

}

else {

/* end of quote part of field */

self->state = IN_FIELD;

}

}

else {

/* normal character - save in field */

if (parse_add_char(self, module_state, c) < 0)

return -1;

}

break;

case ESCAPE_IN_QUOTED_FIELD:

if (c == EOL)

c = '\n';

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = IN_QUOTED_FIELD;

break;

case QUOTE_IN_QUOTED_FIELD:

/* doublequote - seen a quote in a quoted field */

if (dialect->quoting != QUOTE_NONE &&

c == dialect->quotechar) {

/* save "" as " */

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = IN_QUOTED_FIELD;

}

else if (c == dialect->delimiter) {

/* save field - wait for new field */

if (parse_save_field(self) < 0)

return -1;

self->state = START_FIELD;

}

else if (c == '\n' || c == '\r' || c == EOL) {

/* end of line - return [fields] */

if (parse_save_field(self) < 0)

return -1;

self->state = (c == EOL ? START_RECORD : EAT_CRNL);

}

else if (!dialect->strict) {

if (parse_add_char(self, module_state, c) < 0)

return -1;

self->state = IN_FIELD;

}

else {

/* illegal */

PyErr_Format(module_state->error_obj, "'%c' expected after '%c'",

dialect->delimiter,

dialect->quotechar);

return -1;

}

break;

case EAT_CRNL:

if (c == '\n' || c == '\r')

;

else if (c == EOL)

self->state = START_RECORD;

else {

PyErr_Format(module_state->error_obj,

"new-line character seen in unquoted field - "

"do you need to open the file with newline=''?");

return -1;

}

break;

}

return 0;

}

static int

parse_reset(ReaderObj *self)

{

Py_XSETREF(self->fields, PyList_New(0));

if (self->fields == NULL)

return -1;

self->field_len = 0;

self->state = START_RECORD;

self->unquoted_field = false;

return 0;

}

static PyObject *

Reader_iternext_lock_held(PyObject *op)

{

ReaderObj *self = _ReaderObj_CAST(op);

PyObject *fields = NULL;

Py_UCS4 c;

Py_ssize_t pos, linelen;

int kind;

const void *data;

PyObject *lineobj;

_csvstate *module_state = _csv_state_from_type(Py_TYPE(self),

"Reader.__next__");

if (module_state == NULL) {

return NULL;

}

if (parse_reset(self) < 0)

return NULL;

do {

lineobj = PyIter_Next(self->input_iter);

if (lineobj == NULL) {

/* End of input OR exception */

if (!PyErr_Occurred() && (self->field_len != 0 ||

self->state == IN_QUOTED_FIELD)) {

if (self->dialect->strict)

PyErr_SetString(module_state->error_obj,

"unexpected end of data");

else if (parse_save_field(self) >= 0)

break;

}

return NULL;

}

if (!PyUnicode_Check(lineobj)) {

PyErr_Format(module_state->error_obj,

"iterator should return strings, "

"not %.200s "

"(the file should be opened in text mode)",

Py_TYPE(lineobj)->tp_name

);

Py_DECREF(lineobj);

return NULL;

}

if (self->fields == NULL) {

PyErr_SetString(module_state->error_obj,

"iterator has already advanced the reader");

Py_DECREF(lineobj);

return NULL;

}

++self->line_num;

kind = PyUnicode_KIND(lineobj);

data = PyUnicode_DATA(lineobj);

pos = 0;

linelen = PyUnicode_GET_LENGTH(lineobj);

while (linelen--) {

c = PyUnicode_READ(kind, data, pos);

if (parse_process_char(self, module_state, c) < 0) {

Py_DECREF(lineobj);

goto err;

}

pos++;

}

Py_DECREF(lineobj);

if (parse_process_char(self, module_state, EOL) < 0)

goto err;

} while (self->state != START_RECORD);

fields = self->fields;

self->fields = NULL;

err:

return fields;

}

static PyObject *

Reader_iternext(PyObject *op)

{