Bug 3347: robotparser failed because it didn't convert bytes to string.

jeremyhylton · jeremyhylton · commit 73fd46d24e45 · 2008-07-18T20:59:44.000Z
The solution is to convert bytes to text via utf-8.  I'm not entirely
sure if this is safe, but it looks like robots.txt is expected to be
ascii.
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
@@ -136,8 +136,9 @@ def RobotTest(index, robots_txt, good_urls, bad_urls,
 
 RobotTest(7, doc, good, bad)
 
-class TestCase(unittest.TestCase):
-    def runTest(self):
+class NetworkTestCase(unittest.TestCase):
+
+    def testPasswordProtectedSite(self):
         support.requires('network')
         # whole site is password-protected.
         url = 'http://mueblesmoraleda.com'
@@ -146,9 +147,17 @@ def runTest(self):
         parser.read()
         self.assertEqual(parser.can_fetch("*", url+"/robots.txt"), False)
 
+    def testPythonOrg(self):
+        support.requires('network')
+        parser = urllib.robotparser.RobotFileParser(
+            "http://www.python.org/robots.txt")
+        parser.read()
+        self.assertTrue(parser.can_fetch("*",
+                                         "http://www.python.org/robots.txt"))
+
 def test_main():
+    support.run_unittest(NetworkTestCase)
     support.run_unittest(tests)
-    TestCase().run()
 
 if __name__=='__main__':
     support.Verbose = 1
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
@@ -60,7 +60,8 @@ def read(self):
             elif err.code >= 400:
                 self.allow_all = True
         else:
-            self.parse(f.read().splitlines())
+            raw = f.read()
+            self.parse(raw.decode("utf-8").splitlines())
 
     def _add_entry(self, entry):
         if "*" in entry.useragents:
@@ -123,7 +124,10 @@ def can_fetch(self, useragent, url):
             return True
         # search for given user agent matches
         # the first match counts
-        url = urllib.parse.quote(urllib.parse.urlparse(urllib.parse.unquote(url))[2]) or "/"
+        url = urllib.parse.quote(
+            urllib.parse.urlparse(urllib.parse.unquote(url))[2])
+        if not url:
+            url = "/"
         for entry in self.entries:
             if entry.applies_to(useragent):
                 return entry.allowance(url)
