File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -83,7 +83,7 @@ def _add_entry(self, entry):
8383 self .entries .append (entry )
8484
8585 def parse (self , lines ):
86- """parse the input lines from a robot .txt file.
86+ """parse the input lines from a robots .txt file.
8787 We allow that a user-agent: line is not preceded by
8888 one or more blank lines."""
8989 state = 0
@@ -148,7 +148,7 @@ def parse(self, lines):
148148
149149 def can_fetch (self , useragent , url ):
150150 """using the parsed robots.txt decide if useragent can fetch url"""
151- _debug ("Checking robot .txt allowance for:\n user agent: %s\n url: %s" %
151+ _debug ("Checking robots .txt allowance for:\n user agent: %s\n url: %s" %
152152 (useragent , url ))
153153 if self .disallow_all :
154154 return False
Original file line number Diff line number Diff line change @@ -1962,7 +1962,7 @@ repr Redo repr() but with limits on most sizes.
19621962rexec Restricted execution facilities ("safe" exec, eval, etc).
19631963rfc822 RFC-822 message manipulation class.
19641964rlcompleter Word completion for GNU readline 2.0.
1965- robotparser Parse robot .txt files, useful for web spiders.
1965+ robotparser Parse robots .txt files, useful for web spiders.
19661966sched A generally useful event scheduler class.
19671967sets Module for a set datatype.
19681968sgmllib A parser for SGML.
You can’t perform that action at this time.
0 commit comments