Issue #12797: Added custom opener parameter to builtin open() and FileIO.open().

rosslagerwall · rosslagerwall · commit 59142db6d35f · 2011-10-31T20:34:46.000+02:00
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
@@ -776,7 +776,7 @@ are always available.  They are listed here in alphabetical order.
    :meth:`__index__` method that returns an integer.
 
 
-.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True)
+.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None)
 
    Open *file* and return a corresponding stream.  If the file cannot be opened,
    an :exc:`OSError` is raised.
@@ -883,6 +883,15 @@ are always available.  They are listed here in alphabetical order.
    closed.  If a filename is given *closefd* has no effect and must be ``True``
    (the default).
 
+   A custom opener can be used by passing a callable as *opener*. The underlying
+   file descriptor for the file object is then obtained by calling *opener* with
+   (*file*, *flags*). *opener* must return an open file descriptor (passing
+   :mod:`os.open` as *opener* results in functionality similar to passing
+   ``None``).
+
+   .. versionchanged:: 3.3
+      The *opener* parameter was added.
+
    The type of file object returned by the :func:`open` function depends on the
    mode.  When :func:`open` is used to open a file in a text mode (``'w'``,
    ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
@@ -458,7 +458,7 @@ I/O Base Classes
 Raw File I/O
 ^^^^^^^^^^^^
 
-.. class:: FileIO(name, mode='r', closefd=True)
+.. class:: FileIO(name, mode='r', closefd=True, opener=None)
 
    :class:`FileIO` represents an OS-level file containing bytes data.
    It implements the :class:`RawIOBase` interface (and therefore the
@@ -479,6 +479,15 @@ Raw File I/O
    The :meth:`read` (when called with a positive argument), :meth:`readinto`
    and :meth:`write` methods on this class will only make one system call.
 
+   A custom opener can be used by passing a callable as *opener*. The underlying
+   file descriptor for the file object is then obtained by calling *opener* with
+   (*name*, *flags*). *opener* must return an open file descriptor (passing
+   :mod:`os.open` as *opener* results in functionality similar to passing
+   ``None``).
+
+   .. versionchanged:: 3.3
+      The *opener* parameter was added.
+
    In addition to the attributes and methods from :class:`IOBase` and
    :class:`RawIOBase`, :class:`FileIO` provides the following data
    attributes and methods:
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
@@ -27,7 +27,7 @@
 
 
 def open(file, mode="r", buffering=-1, encoding=None, errors=None,
-         newline=None, closefd=True):
+         newline=None, closefd=True, opener=None):
 
     r"""Open file and return a stream.  Raise IOError upon failure.
 
@@ -122,6 +122,12 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
     be kept open when the file is closed. This does not work when a file name is
     given and must be True in that case.
 
+    A custom opener can be used by passing a callable as *opener*. The
+    underlying file descriptor for the file object is then obtained by calling
+    *opener* with (*file*, *flags*). *opener* must return an open file
+    descriptor (passing os.open as *opener* results in functionality similar to
+    passing None).
+
     open() returns a file object whose type depends on the mode, and
     through which the standard file operations such as reading and writing
     are performed. When open() is used to open a file in a text mode ('w',
@@ -176,7 +182,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
                  (writing and "w" or "") +
                  (appending and "a" or "") +
                  (updating and "+" or ""),
-                 closefd)
+                 closefd, opener=opener)
     line_buffering = False
     if buffering == 1 or buffering < 0 and raw.isatty():
         buffering = -1
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
@@ -621,6 +621,15 @@ def test_types_have_dict(self):
         for obj in test:
             self.assertTrue(hasattr(obj, "__dict__"))
 
+    def test_opener(self):
+        with self.open(support.TESTFN, "w") as f:
+            f.write("egg\n")
+        fd = os.open(support.TESTFN, os.O_RDONLY)
+        def opener(path, flags):
+            return fd
+        with self.open("non-existent", "r", opener=opener) as f:
+            self.assertEqual(f.read(), "egg\n")
+
 class CIOTest(IOTest):
 
     def test_IOBase_finalize(self):
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #12797: Added custom opener parameter to builtin open() and
+  FileIO.open().
+
 - Issue #10519: Avoid unnecessary recursive function calls in
   setobject.c.
 
diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c
@@ -95,7 +95,7 @@ PyDoc_STRVAR(module_doc,
  */
 PyDoc_STRVAR(open_doc,
 "open(file, mode='r', buffering=-1, encoding=None,\n"
-"     errors=None, newline=None, closefd=True) -> file object\n"
+"     errors=None, newline=None, closefd=True, opener=None) -> file object\n"
 "\n"
 "Open file and return a stream.  Raise IOError upon failure.\n"
 "\n"
@@ -190,6 +190,12 @@ PyDoc_STRVAR(open_doc,
 "when the file is closed. This does not work when a file name is given\n"
 "and must be True in that case.\n"
 "\n"
+"A custom opener can be used by passing a callable as *opener*. The\n"
+"underlying file descriptor for the file object is then obtained by\n"
+"calling *opener* with (*file*, *flags*). *opener* must return an open\n"
+"file descriptor (passing os.open as *opener* results in functionality\n"
+"similar to passing None).\n"
+"\n"
 "open() returns a file object whose type depends on the mode, and\n"
 "through which the standard file operations such as reading and writing\n"
 "are performed. When open() is used to open a file in a text mode ('w',\n"
@@ -210,8 +216,8 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds)
 {
     char *kwlist[] = {"file", "mode", "buffering",
                       "encoding", "errors", "newline",
-                      "closefd", NULL};
-    PyObject *file;
+                      "closefd", "opener", NULL};
+    PyObject *file, *opener = Py_None;
     char *mode = "r";
     int buffering = -1, closefd = 1;
     char *encoding = NULL, *errors = NULL, *newline = NULL;
@@ -228,10 +234,10 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds)
     _Py_IDENTIFIER(isatty);
     _Py_IDENTIFIER(fileno);
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist,
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzziO:open", kwlist,
                                      &file, &mode, &buffering,
                                      &encoding, &errors, &newline,
-                                     &closefd)) {
+                                     &closefd, &opener)) {
         return NULL;
     }
 
@@ -331,7 +337,7 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds)
 
     /* Create the Raw file stream */
     raw = PyObject_CallFunction((PyObject *)&PyFileIO_Type,
-				"Osi", file, rawmode, closefd);
+                                "OsiO", file, rawmode, closefd, opener);
     if (raw == NULL)
         return NULL;
 
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
@@ -212,9 +212,9 @@ static int
 fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
 {
     fileio *self = (fileio *) oself;
-    static char *kwlist[] = {"file", "mode", "closefd", NULL};
+    static char *kwlist[] = {"file", "mode", "closefd", "opener", NULL};
     const char *name = NULL;
-    PyObject *nameobj, *stringobj = NULL;
+    PyObject *nameobj, *stringobj = NULL, *opener = Py_None;
     char *mode = "r";
     char *s;
 #ifdef MS_WINDOWS
@@ -233,8 +233,9 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
             return -1;
     }
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|si:fileio",
-                                     kwlist, &nameobj, &mode, &closefd))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|siO:fileio",
+                                     kwlist, &nameobj, &mode, &closefd,
+                                     &opener))
         return -1;
 
     if (PyFloat_Check(nameobj)) {
@@ -363,15 +364,35 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
             goto error;
         }
 
-        Py_BEGIN_ALLOW_THREADS
         errno = 0;
+        if (opener == Py_None) {
+            Py_BEGIN_ALLOW_THREADS
 #ifdef MS_WINDOWS
-        if (widename != NULL)
-            self->fd = _wopen(widename, flags, 0666);
-        else
+            if (widename != NULL)
+                self->fd = _wopen(widename, flags, 0666);
+            else
 #endif
-            self->fd = open(name, flags, 0666);
-        Py_END_ALLOW_THREADS
+                self->fd = open(name, flags, 0666);
+            Py_END_ALLOW_THREADS
+        } else {
+            PyObject *fdobj = PyObject_CallFunction(
+                                  opener, "Oi", nameobj, flags);
+            if (fdobj == NULL)
+                goto error;
+            if (!PyLong_Check(fdobj)) {
+                Py_DECREF(fdobj);
+                PyErr_SetString(PyExc_TypeError,
+                        "expected integer from opener");
+                goto error;
+            }
+
+            self->fd = PyLong_AsLong(fdobj);
+            Py_DECREF(fdobj);
+            if (self->fd == -1) {
+                goto error;
+            }
+        }
+
         if (self->fd < 0) {
 #ifdef MS_WINDOWS
             if (widename != NULL)
@@ -1017,13 +1038,17 @@ fileio_getstate(fileio *self)
 
 
 PyDoc_STRVAR(fileio_doc,
-"file(name: str[, mode: str]) -> file IO object\n"
+"file(name: str[, mode: str][, opener: None]) -> file IO object\n"
 "\n"
 "Open a file.  The mode can be 'r', 'w' or 'a' for reading (default),\n"
 "writing or appending.  The file will be created if it doesn't exist\n"
 "when opened for writing or appending; it will be truncated when\n"
 "opened for writing.  Add a '+' to the mode to allow simultaneous\n"
-"reading and writing.");
+"reading and writing. A custom opener can be used by passing a\n"
+"callable as *opener*. The underlying file descriptor for the file\n"
+"object is then obtained by calling opener with (*name*, *flags*).\n"
+"*opener* must return an open file descriptor (passing os.open as\n"
+"*opener* results in functionality similar to passing None).");
 
 PyDoc_STRVAR(read_doc,
 "read(size: int) -> bytes.  read at most size bytes, returned as bytes.\n"
