Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 8e5d0ca

Browse files
kristjanvalurkristjanvalur
committed
Issue #19009
Enhance HTTPResponse.readline() performance
1 parent 4278b2d commit 8e5d0ca

File tree

2 files changed

+390
-91
lines changed

2 files changed

+390
-91
lines changed

‎Lib/http/client.py

Lines changed: 121 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ def parse_headers(fp, _class=HTTPMessage):
271271
return email.parser.Parser(_class=_class).parsestr(hstring)
272272

273273

274-
class HTTPResponse(io.RawIOBase):
274+
class HTTPResponse(io.BufferedIOBase):
275275

276276
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
277277

@@ -496,9 +496,10 @@ def read(self, amt=None):
496496
return b""
497497

498498
if amt is not None:
499-
# Amount is given, so call base class version
500-
# (which is implemented in terms of self.readinto)
501-
return super(HTTPResponse, self).read(amt)
499+
# Amount is given, implement using readinto
500+
b = bytearray(amt)
501+
n = self.readinto(b)
502+
return memoryview(b)[:n].tobytes()
502503
else:
503504
# Amount is not given (unbounded read) so we must check self.length
504505
# and self.chunked
@@ -578,71 +579,67 @@ def _read_and_discard_trailer(self):
578579
if line in (b'\r\n', b'\n', b''):
579580
break
580581

582+
def _get_chunk_left(self):
583+
# return self.chunk_left, reading a new chunk if necessary.
584+
# chunk_left == 0: at the end of the current chunk, need to close it
585+
# chunk_left == None: No current chunk, should read next.
586+
# This function returns non-zero or None if the last chunk has
587+
# been read.
588+
chunk_left = self.chunk_left
589+
if not chunk_left: # Can be 0 or None
590+
if chunk_left is not None:
591+
# We are at the end of chunk. dicard chunk end
592+
self._safe_read(2) # toss the CRLF at the end of the chunk
593+
try:
594+
chunk_left = self._read_next_chunk_size()
595+
except ValueError:
596+
raise IncompleteRead(b'')
597+
if chunk_left == 0:
598+
# last chunk: 1*("0") [ chunk-extension ] CRLF
599+
self._read_and_discard_trailer()
600+
# we read everything; close the "file"
601+
self._close_conn()
602+
chunk_left = None
603+
self.chunk_left = chunk_left
604+
return chunk_left
605+
581606
def _readall_chunked(self):
582607
assert self.chunked != _UNKNOWN
583-
chunk_left = self.chunk_left
584608
value = []
585-
while True:
586-
if chunk_left is None:
587-
try:
588-
chunk_left = self._read_next_chunk_size()
589-
if chunk_left == 0:
590-
break
591-
except ValueError:
592-
raise IncompleteRead(b''.join(value))
593-
value.append(self._safe_read(chunk_left))
594-
595-
# we read the whole chunk, get another
596-
self._safe_read(2) # toss the CRLF at the end of the chunk
597-
chunk_left = None
598-
599-
self._read_and_discard_trailer()
600-
601-
# we read everything; close the "file"
602-
self._close_conn()
603-
604-
return b''.join(value)
609+
try:
610+
while True:
611+
chunk_left = self._get_chunk_left()
612+
if chunk_left is None:
613+
break
614+
value.append(self._safe_read(chunk_left))
615+
self.chunk_left = 0
616+
return b''.join(value)
617+
except IncompleteRead:
618+
raise IncompleteRead(b''.join(value))
605619

606620
def _readinto_chunked(self, b):
607621
assert self.chunked != _UNKNOWN
608-
chunk_left = self.chunk_left
609-
610622
total_bytes = 0
611623
mvb = memoryview(b)
612-
while True:
613-
if chunk_left is None:
614-
try:
615-
chunk_left = self._read_next_chunk_size()
616-
if chunk_left == 0:
617-
break
618-
except ValueError:
619-
raise IncompleteRead(bytes(b[0:total_bytes]))
620-
621-
if len(mvb) < chunk_left:
622-
n = self._safe_readinto(mvb)
623-
self.chunk_left = chunk_left - n
624-
return total_bytes + n
625-
elif len(mvb) == chunk_left:
626-
n = self._safe_readinto(mvb)
627-
self._safe_read(2) # toss the CRLF at the end of the chunk
628-
self.chunk_left = None
629-
return total_bytes + n
630-
else:
631-
temp_mvb = mvb[0:chunk_left]
624+
try:
625+
while True:
626+
chunk_left = self._get_chunk_left()
627+
if chunk_left is None:
628+
return total_bytes
629+
630+
if len(mvb) <= chunk_left:
631+
n = self._safe_readinto(mvb)
632+
self.chunk_left = chunk_left - n
633+
return total_bytes + n
634+
635+
temp_mvb = mvb[:chunk_left]
632636
n = self._safe_readinto(temp_mvb)
633637
mvb = mvb[n:]
634638
total_bytes += n
639+
self.chunk_left = 0
635640

636-
# we read the whole chunk, get another
637-
self._safe_read(2) # toss the CRLF at the end of the chunk
638-
chunk_left = None
639-
640-
self._read_and_discard_trailer()
641-
642-
# we read everything; close the "file"
643-
self._close_conn()
644-
645-
return total_bytes
641+
except IncompleteRead:
642+
raise IncompleteRead(bytes(b[0:total_bytes]))
646643

647644
def _safe_read(self, amt):
648645
"""Read the number of bytes requested, compensating for partial reads.
@@ -683,6 +680,73 @@ def _safe_readinto(self, b):
683680
total_bytes += n
684681
return total_bytes
685682

683+
def read1(self, n=-1):
684+
"""Read with at most one underlying system call. If at least one
685+
byte is buffered, return that instead.
686+
"""
687+
if self.fp is None or self._method == "HEAD":
688+
return b""
689+
if self.chunked:
690+
return self._read1_chunked(n)
691+
try:
692+
result = self.fp.read1(n)
693+
except ValueError:
694+
if n >= 0:
695+
raise
696+
# some implementations, like BufferedReader, don't support -1
697+
# Read an arbitrarily selected largeish chunk.
698+
result = self.fp.read1(16*1024)
699+
if not result and n:
700+
self._close_conn()
701+
return result
702+
703+
def peek(self, n=-1):
704+
# Having this enables IOBase.readline() to read more than one
705+
# byte at a time
706+
if self.fp is None or self._method == "HEAD":
707+
return b""
708+
if self.chunked:
709+
return self._peek_chunked(n)
710+
return self.fp.peek(n)
711+
712+
def readline(self, limit=-1):
713+
if self.fp is None or self._method == "HEAD":
714+
return b""
715+
if self.chunked:
716+
# Fallback to IOBase readline which uses peek() and read()
717+
return super().readline(limit)
718+
result = self.fp.readline(limit)
719+
if not result and limit:
720+
self._close_conn()
721+
return result
722+
723+
def _read1_chunked(self, n):
724+
# Strictly speaking, _get_chunk_left() may cause more than one read,
725+
# but that is ok, since that is to satisfy the chunked protocol.
726+
chunk_left = self._get_chunk_left()
727+
if chunk_left is None or n == 0:
728+
return b''
729+
if not (0 <= n <= chunk_left):
730+
n = chunk_left # if n is negative or larger than chunk_left
731+
read = self.fp.read1(n)
732+
self.chunk_left -= len(read)
733+
if not read:
734+
raise IncompleteRead(b"")
735+
return read
736+
737+
def _peek_chunked(self, n):
738+
# Strictly speaking, _get_chunk_left() may cause more than one read,
739+
# but that is ok, since that is to satisfy the chunked protocol.
740+
try:
741+
chunk_left = self._get_chunk_left()
742+
except IncompleteRead:
743+
return b'' # peek doesn't worry about protocol
744+
if chunk_left is None:
745+
return b'' # eof
746+
# peek is allowed to return more than requested. Just request the
747+
# entire chunk, and truncate what we get.
748+
return self.fp.peek(chunk_left)[:chunk_left]
749+
686750
def fileno(self):
687751
return self.fp.fileno()
688752

0 commit comments

Comments
 (0)