check project name length, url length

binux · binux · commit 9707d6829cd4 · 2015-06-14T21:10:45.000+08:00
diff --git a/pyspider/database/base/projectdb.py b/pyspider/database/base/projectdb.py
@@ -54,3 +54,10 @@ def check_update(self, timestamp, fields=None):
 
     def split_group(self, group, lower=True):
         return re.split("\W+", (group or '').lower())
+
+    def verify_project_name(self, name):
+        if len(name) > 64:
+            return False
+        if re.search(r"[^\w]", name):
+            return False
+        return True
diff --git a/pyspider/libs/base_handler.py b/pyspider/libs/base_handler.py
@@ -216,6 +216,8 @@ def _crawl(self, url, **kwargs):
         """
         task = {}
 
+        assert len(url) < 1024, "Maximum URL length error: len(url) > 1024"
+
         if kwargs.get('callback'):
             callback = kwargs['callback']
             if isinstance(callback, six.string_types) and hasattr(self, callback):
diff --git a/pyspider/webui/debug.py b/pyspider/webui/debug.py
@@ -32,17 +32,11 @@
 default_script = inspect.getsource(sample_handler)
 
 
-def verify_project_name(project):
-    if re.search(r"[^\w]", project):
-        return False
-    return True
-
-
 @app.route('/debug/<project>', methods=['GET', 'POST'])
 def debug(project):
-    if not verify_project_name(project):
-        return 'project name is not allowed!', 400
     projectdb = app.config['projectdb']
+    if not projectdb.verify_project_name(project):
+        return 'project name is not allowed!', 400
     info = projectdb.get(project, fields=['name', 'script'])
     if info:
         script = info['script']
@@ -169,9 +163,9 @@ def run(project):
 
 @app.route('/debug/<project>/save', methods=['POST', ])
 def save(project):
-    if not verify_project_name(project):
-        return 'project name is not allowed!', 400
     projectdb = app.config['projectdb']
+    if not projectdb.verify_project_name(project):
+        return 'project name is not allowed!', 400
     script = request.form['script']
     project_info = projectdb.get(project, fields=['name', 'status', 'group'])
     if project_info and 'lock' in projectdb.split_group(project_info.get('group')) \
@@ -208,9 +202,9 @@ def save(project):
 
 @app.route('/debug/<project>/get')
 def get_script(project):
-    if not verify_project_name(project):
-        return 'project name is not allowed!', 400
     projectdb = app.config['projectdb']
+    if not projectdb.verify_project_name(project):
+        return 'project name is not allowed!', 400
     info = projectdb.get(project, fields=['name', 'script'])
     return json.dumps(utils.unicode_obj(info)), \
            200, {'Content-Type': 'application/json'}
diff --git a/pyspider/webui/webdav.py b/pyspider/webui/webdav.py
@@ -18,11 +18,6 @@
 from .app import app
 
 
-def verify_project_name(project):
-    if re.search(r"[^\w]", project):
-        return False
-    return True
-
 class ContentIO(BytesIO):
     def close(self):
         self.content = self.getvalue()
@@ -49,7 +44,7 @@ def project(self):
         if projectdb:
             self._project = projectdb.get(self.project_name)
         if not self._project:
-            if verify_project_name(self.project_name) and self.name.endswith('.py'):
+            if projectdb.verify_project_name(self.project_name) and self.name.endswith('.py'):
                 self.new_project = True
                 self._project = {
                     'name': self.project_name,
