@@ -211,6 +211,7 @@ def getPage(**kwargs):
211211 refreshing = kwargs .get ("refreshing" , False )
212212 retrying = kwargs .get ("retrying" , False )
213213 crawling = kwargs .get ("crawling" , False )
214+ skipRead = kwargs .get ("skipRead" , False )
214215
215216 if not urlparse .urlsplit (url ).netloc :
216217 url = urlparse .urljoin (conf .url , url )
@@ -266,7 +267,7 @@ def getPage(**kwargs):
266267
267268 multipartOpener = urllib2 .build_opener (proxyHandler , multipartpost .MultipartPostHandler )
268269 conn = multipartOpener .open (unicodeencode (url ), multipart )
269- page = Connect ._connReadProxy (conn )
270+ page = Connect ._connReadProxy (conn ) if not skipRead else None
270271 responseHeaders = conn .info ()
271272 responseHeaders [URI_HTTP_HEADER ] = conn .geturl ()
272273 page = decodePage (page , responseHeaders .get (HTTP_HEADER .CONTENT_ENCODING ), responseHeaders .get (HTTP_HEADER .CONTENT_TYPE ))
@@ -380,12 +381,12 @@ def getPage(**kwargs):
380381
381382 # Get HTTP response
382383 if hasattr (conn , 'redurl' ):
383- page = threadData .lastRedirectMsg [1 ] if kb .redirectChoice == REDIRECTION .NO \
384- else Connect ._connReadProxy (conn )
384+ page = ( threadData .lastRedirectMsg [1 ] if kb .redirectChoice == REDIRECTION .NO \
385+ else Connect ._connReadProxy (conn )) if not skipRead else None
385386 skipLogTraffic = kb .redirectChoice == REDIRECTION .NO
386387 code = conn .redcode
387388 else :
388- page = Connect ._connReadProxy (conn )
389+ page = Connect ._connReadProxy (conn ) if not skipRead else None
389390
390391 code = code or conn .code
391392 responseHeaders = conn .info ()
@@ -439,7 +440,7 @@ def getPage(**kwargs):
439440 responseHeaders = None
440441
441442 try :
442- page = e .read ()
443+ page = e .read () if not skipRead else None
443444 responseHeaders = e .info ()
444445 responseHeaders [URI_HTTP_HEADER ] = e .geturl ()
445446 page = decodePage (page , responseHeaders .get (HTTP_HEADER .CONTENT_ENCODING ), responseHeaders .get (HTTP_HEADER .CONTENT_TYPE ))
@@ -820,10 +821,10 @@ def _randomizeParameter(paramString, randomParameter):
820821
821822 auxHeaders [HTTP_HEADER .RANGE ] = "bytes=-1"
822823
823- _ , headers , code = Connect .getPage (url = uri , get = get , post = post , cookie = cookie , ua = ua , referer = referer , host = host , silent = silent , method = method , auxHeaders = auxHeaders , raise404 = raise404 )
824+ _ , headers , code = Connect .getPage (url = uri , get = get , post = post , cookie = cookie , ua = ua , referer = referer , host = host , silent = silent , method = method , auxHeaders = auxHeaders , raise404 = raise404 , skipRead = ( kb . nullConnection == NULLCONNECTION . SKIP_READ ) )
824825
825826 if headers :
826- if kb .nullConnection == NULLCONNECTION .HEAD and HTTP_HEADER .CONTENT_LENGTH in headers :
827+ if kb .nullConnection in ( NULLCONNECTION .HEAD , NULLCONNECTION . SKIP_READ ) and HTTP_HEADER .CONTENT_LENGTH in headers :
827828 pageLength = int (headers [HTTP_HEADER .CONTENT_LENGTH ])
828829 elif kb .nullConnection == NULLCONNECTION .RANGE and HTTP_HEADER .CONTENT_RANGE in headers :
829830 pageLength = int (headers [HTTP_HEADER .CONTENT_RANGE ][headers [HTTP_HEADER .CONTENT_RANGE ].find ('/' ) + 1 :])
0 commit comments