@@ -59,25 +59,25 @@ def read(self):
5959 elif status >= 400 :
6060 self .allow_all = 1
6161 else :
62- # status < 400
62+ # status < 400
6363 self .parse (connection .getfile ().readlines ())
6464
6565 def parse (self , lines ):
6666 """parse the input lines from a robot.txt file.
67- We allow that a user-agent: line is not preceded by
68- one or more blank lines."""
67+ We allow that a user-agent: line is not preceded by
68+ one or more blank lines."""
6969 state = 0
7070 linenumber = 0
7171 entry = Entry ()
72-
72+
7373 for line in lines :
7474 line = string .strip (line )
7575 linenumber = linenumber + 1
7676 if not line :
7777 if state == 1 :
7878 _debug ("line %d: warning: you should insert"
79- " allow: or disallow: directives below any"
80- " user-agent: line" % linenumber )
79+ " allow: or disallow: directives below any"
80+ " user-agent: line" % linenumber )
8181 entry = Entry ()
8282 state = 0
8383 elif state == 2 :
@@ -98,7 +98,7 @@ def parse(self, lines):
9898 if line [0 ] == "user-agent" :
9999 if state == 2 :
100100 _debug ("line %d: warning: you should insert a blank"
101- " line before any user-agent"
101+ " line before any user-agent"
102102 " directive" % linenumber )
103103 self .entries .append (entry )
104104 entry = Entry ()
@@ -107,14 +107,14 @@ def parse(self, lines):
107107 elif line [0 ] == "disallow" :
108108 if state == 0 :
109109 _debug ("line %d: error: you must insert a user-agent:"
110- " directive before this line" % linenumber )
110+ " directive before this line" % linenumber )
111111 else :
112112 entry .rulelines .append (RuleLine (line [1 ], 0 ))
113113 state = 2
114114 elif line [0 ] == "allow" :
115115 if state == 0 :
116116 _debug ("line %d: error: you must insert a user-agent:"
117- " directive before this line" % linenumber )
117+ " directive before this line" % linenumber )
118118 else :
119119 entry .rulelines .append (RuleLine (line [1 ], 1 ))
120120 else :
@@ -212,7 +212,7 @@ def _test():
212212 print rp .can_fetch ('*' , 'http://www.musi-cal.com/' )
213213 print rp .can_fetch ('Musi-Cal-Robot/1.0' ,
214214 'http://www.musi-cal.com/cgi-bin/event-search'
215- '?city=San+Francisco' )
215+ '?city=San+Francisco' )
216216
217217if __name__ == '__main__' :
218218 _test ()
0 commit comments