Fix for an Issue #268

stamparm · stamparm · commit 79fca8e9d56f · 2012-12-03T12:13:59.000+01:00
diff --git a/lib/core/settings.py b/lib/core/settings.py
@@ -222,10 +222,10 @@
                         )
 
 # Regular expression used for parsing charset info from meta html headers
-META_CHARSET_REGEX = r'<meta http-equiv="?content-type"?[^>]+charset=(?P<result>[^">]+)'
+META_CHARSET_REGEX = r'(?si)<head>.*<meta http-equiv="?content-type"?[^>]+charset=(?P<result>[^">]+).*</head>'
 
 # Regular expression used for parsing refresh info from meta html headers
-META_REFRESH_REGEX = r'<meta http-equiv="?refresh"?[^>]+content="?[^">]+url=(?P<result>[^">]+)'
+META_REFRESH_REGEX = r'(?si)<head>.*<meta http-equiv="?refresh"?[^>]+content="?[^">]+url=(?P<result>[^">]+).*</head>'
 
 # Regular expression used for parsing empty fields in tested form data
 EMPTY_FORM_FIELDS_REGEX = r'(&|\A)(?P<result>[^=]+=(&|\Z))'
diff --git a/lib/request/basic.py b/lib/request/basic.py
@@ -220,7 +220,7 @@ def decodePage(page, contentEncoding, contentType):
         if contentType and (contentType.find("charset=") != -1):
             httpCharset = checkCharEncoding(contentType.split("charset=")[-1])
 
-        metaCharset = checkCharEncoding(extractRegexResult(META_CHARSET_REGEX, page, re.DOTALL | re.IGNORECASE))
+        metaCharset = checkCharEncoding(extractRegexResult(META_CHARSET_REGEX, page))
 
         if ((httpCharset or metaCharset) and not all([httpCharset, metaCharset]))\
             or (httpCharset == metaCharset and all([httpCharset, metaCharset])):
diff --git a/lib/request/connect.py b/lib/request/connect.py
@@ -372,8 +372,8 @@ def getPage(**kwargs):
             page = decodePage(page, responseHeaders.get(HTTPHEADER.CONTENT_ENCODING), responseHeaders.get(HTTPHEADER.CONTENT_TYPE))
             status = getUnicode(conn.msg)
 
-            if extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE) and not refreshing:
-                url = extractRegexResult(META_REFRESH_REGEX, page, re.DOTALL | re.IGNORECASE)
+            if extractRegexResult(META_REFRESH_REGEX, page) and not refreshing:
+                url = extractRegexResult(META_REFRESH_REGEX, page)
 
                 debugMsg = "got HTML meta refresh header"
                 logger.debug(debugMsg)
