Issue3243 - Support iterable bodies in httplib. Patch contributions by Xuanji Li and Chris AtLee.

orsenthil · orsenthil · commit 7bc0d872ddb0 · 2010-12-19T10:49:52.000Z
diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
@@ -393,14 +393,18 @@ HTTPConnection Objects
    string.
 
    The *body* may also be an open :term:`file object`, in which case the
-   contents of the file is sent; this file object should support
-   ``fileno()`` and ``read()`` methods. The header Content-Length is
-   automatically set to the length of the file as reported by
-   stat.
+   contents of the file is sent; this file object should support ``fileno()``
+   and ``read()`` methods. The header Content-Length is automatically set to
+   the length of the file as reported by stat. The *body* argument may also be
+   an iterable and Contet-Length header should be explicitly provided when the
+   body is an iterable.
 
    The *headers* argument should be a mapping of extra HTTP
    headers to send with the request.
 
+   .. versionadded:: 3.2
+      *body* can be an iterable
+
 .. method:: HTTPConnection.getresponse()
 
    Should be called after a request is sent to get the response from the server.
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
@@ -21,13 +21,14 @@ The :mod:`urllib.request` module defines the following functions:
    :class:`Request` object.
 
    *data* may be a string specifying additional data to send to the
-   server, or ``None`` if no such data is needed.  Currently HTTP
-   requests are the only ones that use *data*; the HTTP request will
-   be a POST instead of a GET when the *data* parameter is provided.
-   *data* should be a buffer in the standard
+   server, or ``None`` if no such data is needed. *data* may also be an
+   iterable object and in that case Content-Length value must be specified in
+   the headers. Currently HTTP requests are the only ones that use *data*; the
+   HTTP request will be a POST instead of a GET when the *data* parameter is
+   provided.  *data* should be a buffer in the standard
    :mimetype:`application/x-www-form-urlencoded` format.  The
-   :func:`urllib.parse.urlencode` function takes a mapping or sequence
-   of 2-tuples and returns a string in this format. urllib.request module uses
+   :func:`urllib.parse.urlencode` function takes a mapping or sequence of
+   2-tuples and returns a string in this format. urllib.request module uses
    HTTP/1.1 and includes ``Connection:close`` header in its HTTP requests.
 
    The optional *timeout* parameter specifies a timeout in seconds for
@@ -76,6 +77,9 @@ The :mod:`urllib.request` module defines the following functions:
       HTTPS virtual hosts are now supported if possible (that is, if
       :data:`ssl.HAS_SNI` is true).
 
+   .. versionadded:: 3.2
+      *data* can be an iterable object.
+
 .. function:: install_opener(opener)
 
    Install an :class:`OpenerDirector` instance as the default global opener.
diff --git a/Lib/http/client.py b/Lib/http/client.py
@@ -71,6 +71,7 @@
 import io
 import os
 import socket
+import collections
 from urllib.parse import urlsplit
 import warnings
 
@@ -730,7 +731,11 @@ def close(self):
         self.__state = _CS_IDLE
 
     def send(self, data):
-        """Send `data' to the server."""
+        """Send `data' to the server.
+        ``data`` can be a string object, a bytes object, an array object, a
+        file-like object that supports a .read() method, or an iterable object.
+        """
+
         if self.sock is None:
             if self.auto_open:
                 self.connect()
@@ -762,8 +767,16 @@ def send(self, data):
                 if encode:
                     datablock = datablock.encode("iso-8859-1")
                 self.sock.sendall(datablock)
-        else:
+
+        try:
             self.sock.sendall(data)
+        except TypeError:
+            if isinstance(data, collections.Iterable):
+                for d in data:
+                    self.sock.sendall(d)
+            else:
+                raise TypeError("data should be byte-like object\
+                        or an iterable, got %r " % type(it))
 
     def _output(self, s):
         """Add a line of output to the current request buffer.
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
@@ -230,6 +230,22 @@ def test_send(self):
         conn.send(io.BytesIO(expected))
         self.assertEqual(expected, sock.data)
 
+    def test_send_iter(self):
+        expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \
+                   b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \
+                   b'\r\nonetwothree'
+
+        def body():
+            yield b"one"
+            yield b"two"
+            yield b"three"
+
+        conn = client.HTTPConnection('example.com')
+        sock = FakeSocket("")
+        conn.sock = sock
+        conn.request('GET', '/foo', body(), {'Content-Length': '11'})
+        self.assertEquals(sock.data, expected)
+
     def test_chunked(self):
         chunked_start = (
             'HTTP/1.1 200 OK\r\n'
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
@@ -4,6 +4,7 @@
 import os
 import io
 import socket
+import array
 
 import urllib.request
 from urllib.request import Request, OpenerDirector
@@ -765,7 +766,7 @@ def test_http(self):
         o = h.parent = MockOpener()
 
         url = "http://example.com/"
-        for method, data in [("GET", None), ("POST", "blah")]:
+        for method, data in [("GET", None), ("POST", b"blah")]:
             req = Request(url, data, {"Foo": "bar"})
             req.timeout = None
             req.add_unredirected_header("Spam", "eggs")
@@ -795,7 +796,7 @@ def test_http(self):
 
         # check adding of standard headers
         o.addheaders = [("Spam", "eggs")]
-        for data in "", None:  # POST, GET
+        for data in b"", None:  # POST, GET
             req = Request("http://example.com/", data)
             r = MockResponse(200, "OK", {}, "")
             newreq = h.do_request_(req)
@@ -821,14 +822,58 @@ def test_http(self):
             self.assertEqual(req.unredirected_hdrs["Host"], "baz")
             self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
 
+        # Check iterable body support
+        def iterable_body():
+            yield b"one"
+            yield b"two"
+            yield b"three"
+
+        for headers in {}, {"Content-Length": 11}:
+            req = Request("http://example.com/", iterable_body(), headers)
+            if not headers:
+                # Having an iterable body without a Content-Length should
+                # raise an exception
+                self.assertRaises(ValueError, h.do_request_, req)
+            else:
+                newreq = h.do_request_(req)
+
+        # A file object
+
+        """
+        file_obj = io.StringIO()
+        file_obj.write("Something\nSomething\nSomething\n")
+
+        for headers in {}, {"Content-Length": 30}:
+            req = Request("http://example.com/", file_obj, headers)
+            if not headers:
+                # Having an iterable body without a Content-Length should
+                # raise an exception
+                self.assertRaises(ValueError, h.do_request_, req)
+            else:
+                newreq = h.do_request_(req)
+                self.assertEqual(int(newreq.get_header('Content-length')),30)
+
+        file_obj.close()
+
+        # array.array Iterable - Content Length is calculated
+
+        iterable_array = array.array("I",[1,2,3,4])
+
+        for headers in {}, {"Content-Length": 16}:
+            req = Request("http://example.com/", iterable_array, headers)
+            newreq = h.do_request_(req)
+            self.assertEqual(int(newreq.get_header('Content-length')),16)
+        """
+
+
     def test_http_doubleslash(self):
         # Checks the presence of any unnecessary double slash in url does not
         # break anything. Previously, a double slash directly after the host
         # could could cause incorrect parsing.
         h = urllib.request.AbstractHTTPHandler()
         o = h.parent = MockOpener()
 
-        data = ""
+        data = b""
         ds_urls = [
             "http://example.com/foo/bar/baz.html",
             "http://example.com//foo/bar/baz.html",
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
@@ -94,6 +94,7 @@
 import socket
 import sys
 import time
+import collections
 
 from urllib.error import URLError, HTTPError, ContentTooShortError
 from urllib.parse import (
@@ -1053,8 +1054,17 @@ def do_request_(self, request):
                     'Content-type',
                     'application/x-www-form-urlencoded')
             if not request.has_header('Content-length'):
-                request.add_unredirected_header(
-                    'Content-length', '%d' % len(data))
+                try:
+                    mv = memoryview(data)
+                except TypeError:
+                    print(data)
+                    if isinstance(data, collections.Iterable):
+                        raise ValueError("Content-Length should be specified \
+                                for iterable data of type %r %r" % (type(data),
+                                data))
+                else:
+                    request.add_unredirected_header(
+                            'Content-length', '%d' % len(mv) * mv.itemsize)
 
         sel_host = host
         if request.has_proxy():
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -23,6 +23,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #3243:  Support iterable bodies in httplib. Patch Contributions by
+  Xuanji Li and Chris AtLee.
+
 - Issue #10611: SystemExit exception will no longer kill a unittest run.
 
 - Issue #9857: It is now possible to skip a test in a setUp, tearDown or clean
