Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3e86595

Browse files
loewisloewis
committed
Patch #1349118: urllib2 now supports user:pass@ style proxy
specifications, raises IOErrors when proxies for unsupported protocols are defined, and uses the https proxy on https redirections.
1 parent 29602d2 commit 3e86595

3 files changed

Lines changed: 98 additions & 6 deletions

File tree

‎Lib/urllib.py‎

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
"splitnport", "splitquery", "splitattr", "splitvalue",
3838
"splitgophertype", "getproxies"]
3939

40-
__version__ = '1.16' # XXX This version is not always updated :-(
40+
__version__ = '1.17' # XXX This version is not always updated :-(
4141

4242
MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
4343

@@ -271,6 +271,7 @@ def open_http(self, url, data=None):
271271
"""Use HTTP protocol."""
272272
import httplib
273273
user_passwd = None
274+
proxy_passwd= None
274275
if isinstance(url, str):
275276
host, selector = splithost(url)
276277
if host:
@@ -279,6 +280,9 @@ def open_http(self, url, data=None):
279280
realhost = host
280281
else:
281282
host, selector = url
283+
# check whether the proxy contains authorization information
284+
proxy_passwd, host = splituser(host)
285+
# now we proceed with the url we want to obtain
282286
urltype, rest = splittype(selector)
283287
url = rest
284288
user_passwd = None
@@ -295,6 +299,13 @@ def open_http(self, url, data=None):
295299

296300
#print "proxy via http:", host, selector
297301
if not host: raise IOError, ('http error', 'no host given')
302+
303+
if proxy_passwd:
304+
import base64
305+
proxy_auth = base64.encodestring(proxy_passwd).strip()
306+
else:
307+
proxy_auth = None
308+
298309
if user_passwd:
299310
import base64
300311
auth = base64.encodestring(user_passwd).strip()
@@ -307,6 +318,7 @@ def open_http(self, url, data=None):
307318
h.putheader('Content-length', '%d' % len(data))
308319
else:
309320
h.putrequest('GET', selector)
321+
if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
310322
if auth: h.putheader('Authorization', 'Basic %s' % auth)
311323
if realhost: h.putheader('Host', realhost)
312324
for args in self.addheaders: h.putheader(*args)
@@ -349,6 +361,7 @@ def open_https(self, url, data=None):
349361
"""Use HTTPS protocol."""
350362
import httplib
351363
user_passwd = None
364+
proxy_passwd = None
352365
if isinstance(url, str):
353366
host, selector = splithost(url)
354367
if host:
@@ -357,6 +370,8 @@ def open_https(self, url, data=None):
357370
realhost = host
358371
else:
359372
host, selector = url
373+
# here, we determine, whether the proxy contains authorization information
374+
proxy_passwd, host = splituser(host)
360375
urltype, rest = splittype(selector)
361376
url = rest
362377
user_passwd = None
@@ -370,6 +385,11 @@ def open_https(self, url, data=None):
370385
selector = "%s://%s%s" % (urltype, realhost, rest)
371386
#print "proxy via https:", host, selector
372387
if not host: raise IOError, ('https error', 'no host given')
388+
if proxy_passwd:
389+
import base64
390+
proxy_auth = base64.encodestring(proxy_passwd).strip()
391+
else:
392+
proxy_auth = None
373393
if user_passwd:
374394
import base64
375395
auth = base64.encodestring(user_passwd).strip()
@@ -385,7 +405,8 @@ def open_https(self, url, data=None):
385405
h.putheader('Content-length', '%d' % len(data))
386406
else:
387407
h.putrequest('GET', selector)
388-
if auth: h.putheader('Authorization', 'Basic %s' % auth)
408+
if proxy_auth: h.putheader('Proxy-Authorization: Basic %s' % proxy_auth)
409+
if auth: h.putheader('Authorization: Basic %s' % auth)
389410
if realhost: h.putheader('Host', realhost)
390411
for args in self.addheaders: h.putheader(*args)
391412
h.endheaders()
@@ -404,6 +425,8 @@ def open_https(self, url, data=None):
404425

405426
def open_gopher(self, url):
406427
"""Use Gopher protocol."""
428+
if not isinstance(url, str):
429+
raise IOError, ('gopher error', 'proxy support for gopher protocol currently not implemented')
407430
import gopherlib
408431
host, selector = splithost(url)
409432
if not host: raise IOError, ('gopher error', 'no host given')
@@ -419,6 +442,8 @@ def open_gopher(self, url):
419442
return addinfourl(fp, noheaders(), "gopher:" + url)
420443

421444
def open_file(self, url):
445+
if not isinstance(url, str):
446+
raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
422447
"""Use local file or FTP depending on form of URL."""
423448
if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
424449
return self.open_ftp(url)
@@ -462,6 +487,8 @@ def open_local_file(self, url):
462487

463488
def open_ftp(self, url):
464489
"""Use FTP protocol."""
490+
if not isinstance(url, str):
491+
raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
465492
import mimetypes, mimetools
466493
try:
467494
from cStringIO import StringIO
@@ -522,6 +549,8 @@ def open_ftp(self, url):
522549

523550
def open_data(self, url, data=None):
524551
"""Use "data" URL."""
552+
if not isinstance(url, str):
553+
raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
525554
# ignore POSTed data
526555
#
527556
# syntax of data URLs:
@@ -624,8 +653,7 @@ def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
624653

625654
def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
626655
"""Error 401 -- authentication required.
627-
See this URL for a description of the basic authentication scheme:
628-
http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt"""
656+
This function supports Basic authentication only."""
629657
if not 'www-authenticate' in headers:
630658
URLopener.http_error_default(self, url, fp,
631659
errcode, errmsg, headers)
@@ -644,7 +672,63 @@ def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
644672
return getattr(self,name)(url, realm)
645673
else:
646674
return getattr(self,name)(url, realm, data)
675+
676+
def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
677+
"""Error 407 -- proxy authentication required.
678+
This function supports Basic authentication only."""
679+
if not 'proxy-authenticate' in headers:
680+
URLopener.http_error_default(self, url, fp,
681+
errcode, errmsg, headers)
682+
stuff = headers['proxy-authenticate']
683+
import re
684+
match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
685+
if not match:
686+
URLopener.http_error_default(self, url, fp,
687+
errcode, errmsg, headers)
688+
scheme, realm = match.groups()
689+
if scheme.lower() != 'basic':
690+
URLopener.http_error_default(self, url, fp,
691+
errcode, errmsg, headers)
692+
name = 'retry_proxy_' + self.type + '_basic_auth'
693+
if data is None:
694+
return getattr(self,name)(url, realm)
695+
else:
696+
return getattr(self,name)(url, realm, data)
697+
698+
def retry_proxy_http_basic_auth(self, url, realm, data=None):
699+
host, selector = splithost(url)
700+
newurl = 'http://' + host + selector
701+
proxy = self.proxies['http']
702+
urltype, proxyhost = splittype(proxy)
703+
proxyhost, proxyselector = splithost(proxyhost)
704+
i = proxyhost.find('@') + 1
705+
proxyhost = proxyhost[i:]
706+
user, passwd = self.get_user_passwd(proxyhost, realm, i)
707+
if not (user or passwd): return None
708+
proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
709+
self.proxies['http'] = 'http://' + proxyhost + proxyselector
710+
if data is None:
711+
return self.open(newurl)
712+
else:
713+
return self.open(newurl, data)
647714

715+
def retry_proxy_https_basic_auth(self, url, realm, data=None):
716+
host, selector = splithost(url)
717+
newurl = 'https://' + host + selector
718+
proxy = self.proxies['https']
719+
urltype, proxyhost = splittype(proxy)
720+
proxyhost, proxyselector = splithost(proxyhost)
721+
i = proxyhost.find('@') + 1
722+
proxyhost = proxyhost[i:]
723+
user, passwd = self.get_user_passwd(proxyhost, realm, i)
724+
if not (user or passwd): return None
725+
proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
726+
self.proxies['https'] = 'https://' + proxyhost + proxyselector
727+
if data is None:
728+
return self.open(newurl)
729+
else:
730+
return self.open(newurl, data)
731+
648732
def retry_http_basic_auth(self, url, realm, data=None):
649733
host, selector = splithost(url)
650734
i = host.find('@') + 1
@@ -665,8 +749,11 @@ def retry_https_basic_auth(self, url, realm, data=None):
665749
user, passwd = self.get_user_passwd(host, realm, i)
666750
if not (user or passwd): return None
667751
host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
668-
newurl = '//' + host + selector
669-
return self.open_https(newurl, data)
752+
newurl = 'https://' + host + selector
753+
if data is None:
754+
return self.open(newurl)
755+
else:
756+
return self.open(newurl, data)
670757

671758
def get_user_passwd(self, host, realm, clear_cache = 0):
672759
key = realm + '@' + host.lower()

‎Misc/ACKS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ Fredrik Nehr
427427
Chad Netzer
428428
Max Neunh�ffer
429429
George Neville-Neil
430+
Johannes Nicolai
430431
Samuel Nicolary
431432
Gustavo Niemeyer
432433
Oscar Nierstrasz

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,10 @@ Extension Modules
337337
Library
338338
-------
339339

340+
- Patch #1349118: urllib2 now supports user:pass@ style proxy
341+
specifications, raises IOErrors when proxies for unsupported protocols
342+
are defined, and uses the https proxy on https redirections.
343+
340344
- Bug #902075: urllib2 now supports 'host:port' style proxy specifications.
341345

342346
- Bug #1407902: Add support for sftp:// URIs to urlparse.

0 commit comments

Comments
 (0)