rename ReDoSUtil to NfaUtils, and rename the "performance" folder to "regexp"

erik-krogh · erik-krogh · commit 13482fc97bd7 · 2022-06-23T14:36:25.000+02:00
diff --git a/config/identical-files.json b/config/identical-files.json
@@ -484,27 +484,27 @@
     "ruby/ql/lib/codeql/ruby/security/internal/SensitiveDataHeuristics.qll"
   ],
   "ReDoS Util Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
-    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll",
-    "java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll"
+    "javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll",
+    "python/ql/lib/semmle/python/security/regexp/NfaUtils.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll"
   ],
   "ReDoS Exponential Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll"
+    "javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll"
   ],
   "ReDoS Polynomial Python/JS/Ruby/Java": [
-    "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll",
-    "java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll"
+    "javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll",
+    "java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll"
   ],
   "RegexpMatching Python/JS/Ruby": [
-    "javascript/ql/lib/semmle/javascript/security/performance/RegexpMatching.qll",
-    "python/ql/lib/semmle/python/security/performance/RegexpMatching.qll",
-    "ruby/ql/lib/codeql/ruby/security/performance/RegexpMatching.qll"
+    "javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll",
+    "python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll",
+    "ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll"
   ],
   "BadTagFilterQuery Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll",
diff --git a/java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll b/java/ql/lib/semmle/code/java/security/regexp/ExponentialBackTracking.qll
@@ -62,7 +62,7 @@
  *     a suffix `x` (possible empty) that is most likely __not__ accepted.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /**
  * Holds if state `s` might be inside a backtracking repetition.
diff --git a/java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll b/java/ql/lib/semmle/code/java/security/regexp/NfaUtils.qll
@@ -1,18 +1,13 @@
 /**
- * Provides classes for working with regular expressions that can
- * perform backtracking in superlinear/exponential time.
+ * Provides classes and predicates for constructing an NFA from
+ * a regular expression, and various utilities for reasoning about
+ * the resulting NFA.
  *
- * This module contains a number of utility predicates for compiling a regular expression into a NFA and reasoning about this NFA.
- *
- * The `ReDoSConfiguration` contains a `isReDoSCandidate` predicate that is used to
- * to determine which states the prefix/suffix search should happen on.
- * There is only meant to exist one `ReDoSConfiguration` at a time.
- *
- * The predicate `hasReDoSResult` outputs a de-duplicated set of
- * states that will cause backtracking (a rejecting suffix exists).
+ * These utilities are used both by the ReDoS queries and by
+ * other queries that benefit from reasoning about NFAs.
  */
 
-import ReDoSUtilSpecific
+import NfaUtilsSpecific
 
 /**
  * Gets the char after `c` (from a simplified ASCII table).
diff --git a/java/ql/lib/semmle/code/java/security/regexp/NfaUtilsSpecific.qll b/java/ql/lib/semmle/code/java/security/regexp/NfaUtilsSpecific.qll
diff --git a/java/ql/lib/semmle/code/java/security/regexp/PolynomialReDoSQuery.qll b/java/ql/lib/semmle/code/java/security/regexp/PolynomialReDoSQuery.qll
@@ -1,6 +1,6 @@
 /** Definitions and configurations for the Polynomial ReDoS query */
 
-import semmle.code.java.security.performance.SuperlinearBackTracking
+import semmle.code.java.security.regexp.SuperlinearBackTracking
 import semmle.code.java.dataflow.DataFlow
 import semmle.code.java.regex.RegexTreeView
 import semmle.code.java.regex.RegexFlowConfigs
diff --git a/java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll b/java/ql/lib/semmle/code/java/security/regexp/SuperlinearBackTracking.qll
@@ -3,7 +3,7 @@
  * perform backtracking in superlinear time.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /*
  * This module implements the analysis described in the paper:
diff --git a/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql b/java/ql/src/Security/CWE/CWE-730/PolynomialReDoS.ql
@@ -13,7 +13,7 @@
  */
 
 import java
-import semmle.code.java.security.performance.PolynomialReDoSQuery
+import semmle.code.java.security.regexp.PolynomialReDoSQuery
 import DataFlow::PathGraph
 
 from DataFlow::PathNode source, DataFlow::PathNode sink, PolynomialBackTrackingTerm regexp
diff --git a/java/ql/src/Security/CWE/CWE-730/ReDoS.ql b/java/ql/src/Security/CWE/CWE-730/ReDoS.ql
@@ -14,7 +14,7 @@
  */
 
 import java
-import semmle.code.java.security.performance.ExponentialBackTracking
+import semmle.code.java.security.regexp.ExponentialBackTracking
 
 from RegExpTerm t, string pump, State s, string prefixMsg
 where
diff --git a/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql b/java/ql/test/query-tests/security/CWE-730/PolynomialReDoS.ql
@@ -1,6 +1,6 @@
 import java
 import TestUtilities.InlineExpectationsTest
-import semmle.code.java.security.performance.PolynomialReDoSQuery
+import semmle.code.java.security.regexp.PolynomialReDoSQuery
 
 class HasPolyRedos extends InlineExpectationsTest {
   HasPolyRedos() { this = "HasPolyRedos" }
diff --git a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql
@@ -1,6 +1,6 @@
 import java
 import TestUtilities.InlineExpectationsTest
-import semmle.code.java.security.performance.ExponentialBackTracking
+import semmle.code.java.security.regexp.ExponentialBackTracking
 import semmle.code.java.regex.regex
 
 class HasExpRedos extends InlineExpectationsTest {
diff --git a/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll b/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll
@@ -2,7 +2,7 @@
  * Provides precicates for reasoning about bad tag filter vulnerabilities.
  */
 
-import performance.RegexpMatching
+import regexp.RegexpMatching
 
 /**
  * Holds if the regexp `root` should be tested against `str`.
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll b/javascript/ql/lib/semmle/javascript/security/regexp/ExponentialBackTracking.qll
@@ -62,7 +62,7 @@
  *     a suffix `x` (possible empty) that is most likely __not__ accepted.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /**
  * Holds if state `s` might be inside a backtracking repetition.
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll b/javascript/ql/lib/semmle/javascript/security/regexp/NfaUtils.qll
@@ -1,18 +1,13 @@
 /**
- * Provides classes for working with regular expressions that can
- * perform backtracking in superlinear/exponential time.
+ * Provides classes and predicates for constructing an NFA from
+ * a regular expression, and various utilities for reasoning about
+ * the resulting NFA.
  *
- * This module contains a number of utility predicates for compiling a regular expression into a NFA and reasoning about this NFA.
- *
- * The `ReDoSConfiguration` contains a `isReDoSCandidate` predicate that is used to
- * to determine which states the prefix/suffix search should happen on.
- * There is only meant to exist one `ReDoSConfiguration` at a time.
- *
- * The predicate `hasReDoSResult` outputs a de-duplicated set of
- * states that will cause backtracking (a rejecting suffix exists).
+ * These utilities are used both by the ReDoS queries and by
+ * other queries that benefit from reasoning about NFAs.
  */
 
-import ReDoSUtilSpecific
+import NfaUtilsSpecific
 
 /**
  * Gets the char after `c` (from a simplified ASCII table).
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/NfaUtilsSpecific.qll b/javascript/ql/lib/semmle/javascript/security/regexp/NfaUtilsSpecific.qll
@@ -1,5 +1,5 @@
 /**
- * Provides JavaScript-specific definitions for use in the ReDoSUtil module.
+ * Provides JavaScript-specific definitions for use in the NfaUtils module.
  */
 
 import javascript
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/PolynomialReDoS.qll b/javascript/ql/lib/semmle/javascript/security/regexp/PolynomialReDoS.qll
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/PolynomialReDoSCustomizations.qll b/javascript/ql/lib/semmle/javascript/security/regexp/PolynomialReDoSCustomizations.qll
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll b/javascript/ql/lib/semmle/javascript/security/regexp/RegexpMatching.qll
@@ -3,7 +3,7 @@
  * and for testing which capture groups are filled when a particular regexp matches a string.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /** A root term */
 class RootTerm extends RegExpTerm {
diff --git a/javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll b/javascript/ql/lib/semmle/javascript/security/regexp/SuperlinearBackTracking.qll
@@ -3,7 +3,7 @@
  * perform backtracking in superlinear time.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /*
  * This module implements the analysis described in the paper:
diff --git a/javascript/ql/src/Performance/PolynomialReDoS.ql b/javascript/ql/src/Performance/PolynomialReDoS.ql
@@ -14,8 +14,8 @@
  */
 
 import javascript
-import semmle.javascript.security.performance.PolynomialReDoS::PolynomialReDoS
-import semmle.javascript.security.performance.SuperlinearBackTracking
+import semmle.javascript.security.regexp.PolynomialReDoS::PolynomialReDoS
+import semmle.javascript.security.regexp.SuperlinearBackTracking
 import DataFlow::PathGraph
 
 from
diff --git a/javascript/ql/src/Performance/ReDoS.ql b/javascript/ql/src/Performance/ReDoS.ql
@@ -15,8 +15,8 @@
  */
 
 import javascript
-import semmle.javascript.security.performance.ReDoSUtil
-import semmle.javascript.security.performance.ExponentialBackTracking
+import semmle.javascript.security.regexp.NfaUtils
+import semmle.javascript.security.regexp.ExponentialBackTracking
 
 from RegExpTerm t, string pump, State s, string prefixMsg
 where hasReDoSResult(t, pump, s, prefixMsg)
diff --git a/javascript/ql/src/Security/CWE-116/IncompleteMultiCharacterSanitization.ql b/javascript/ql/src/Security/CWE-116/IncompleteMultiCharacterSanitization.ql
@@ -57,7 +57,7 @@ DangerousPrefix getADangerousMatchedPrefix(EmptyReplaceRegExpTerm t) {
   not exists(EmptyReplaceRegExpTerm pred | pred = t.getPredecessor+() and not pred.isNullable())
 }
 
-private import semmle.javascript.security.performance.ReDoSUtil as ReDoSUtil
+private import semmle.javascript.security.regexp.NfaUtils as NfaUtils
 
 /**
  * Gets a char from a dangerous prefix that is matched by `t`.
@@ -69,8 +69,8 @@ DangerousPrefixSubstring getADangerousMatchedChar(EmptyReplaceRegExpTerm t) {
   t.getAMatchedString() = result
   or
   // A substring matched by some character class. This is only used to match the "word" part of a HTML tag (e.g. "iframe" in "<iframe").
-  exists(ReDoSUtil::CharacterClass cc |
-    cc = ReDoSUtil::getCanonicalCharClass(t) and
+  exists(NfaUtils::CharacterClass cc |
+    cc = NfaUtils::getCanonicalCharClass(t) and
     cc.matches(result) and
     result.regexpMatch("\\w") and
     // excluding character classes that match ">" (e.g. /<[^<]*>/), as these might consume nested HTML tags, and thus prevent the dangerous pattern this query is looking for.
diff --git a/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.ql b/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.ql
@@ -1,4 +1,4 @@
-import semmle.javascript.security.performance.SuperlinearBackTracking
+import semmle.javascript.security.regexp.SuperlinearBackTracking
 
 from PolynomialBackTrackingTerm t
 select t, t.getReason()
diff --git a/python/ql/lib/semmle/python/security/BadTagFilterQuery.qll b/python/ql/lib/semmle/python/security/BadTagFilterQuery.qll
@@ -2,7 +2,7 @@
  * Provides precicates for reasoning about bad tag filter vulnerabilities.
  */
 
-import performance.RegexpMatching
+import regexp.RegexpMatching
 
 /**
  * Holds if the regexp `root` should be tested against `str`.
diff --git a/python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll b/python/ql/lib/semmle/python/security/regexp/ExponentialBackTracking.qll
@@ -62,7 +62,7 @@
  *     a suffix `x` (possible empty) that is most likely __not__ accepted.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /**
  * Holds if state `s` might be inside a backtracking repetition.
diff --git a/python/ql/lib/semmle/python/security/regexp/NfaUtils.qll b/python/ql/lib/semmle/python/security/regexp/NfaUtils.qll
@@ -1,18 +1,13 @@
 /**
- * Provides classes for working with regular expressions that can
- * perform backtracking in superlinear/exponential time.
+ * Provides classes and predicates for constructing an NFA from
+ * a regular expression, and various utilities for reasoning about
+ * the resulting NFA.
  *
- * This module contains a number of utility predicates for compiling a regular expression into a NFA and reasoning about this NFA.
- *
- * The `ReDoSConfiguration` contains a `isReDoSCandidate` predicate that is used to
- * to determine which states the prefix/suffix search should happen on.
- * There is only meant to exist one `ReDoSConfiguration` at a time.
- *
- * The predicate `hasReDoSResult` outputs a de-duplicated set of
- * states that will cause backtracking (a rejecting suffix exists).
+ * These utilities are used both by the ReDoS queries and by
+ * other queries that benefit from reasoning about NFAs.
  */
 
-import ReDoSUtilSpecific
+import NfaUtilsSpecific
 
 /**
  * Gets the char after `c` (from a simplified ASCII table).
diff --git a/python/ql/lib/semmle/python/security/regexp/NfaUtilsSpecific.qll b/python/ql/lib/semmle/python/security/regexp/NfaUtilsSpecific.qll
@@ -1,5 +1,5 @@
 /**
- * Provides Python-specific definitions for use in the ReDoSUtil module.
+ * Provides Python-specific definitions for use in the NfaUtils module.
  */
 
 import python
diff --git a/python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll b/python/ql/lib/semmle/python/security/regexp/RegexpMatching.qll
@@ -3,7 +3,7 @@
  * and for testing which capture groups are filled when a particular regexp matches a string.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /** A root term */
 class RootTerm extends RegExpTerm {
diff --git a/python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll b/python/ql/lib/semmle/python/security/regexp/SuperlinearBackTracking.qll
@@ -3,7 +3,7 @@
  * perform backtracking in superlinear time.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /*
  * This module implements the analysis described in the paper:
diff --git a/python/ql/src/Security/CWE-730/PolynomialReDoS.ql b/python/ql/src/Security/CWE-730/PolynomialReDoS.ql
@@ -13,7 +13,7 @@
  */
 
 import python
-import semmle.python.security.performance.SuperlinearBackTracking
+import semmle.python.security.regexp.SuperlinearBackTracking
 import semmle.python.security.dataflow.PolynomialReDoSQuery
 import DataFlow::PathGraph
 
diff --git a/python/ql/src/Security/CWE-730/ReDoS.ql b/python/ql/src/Security/CWE-730/ReDoS.ql
@@ -14,7 +14,7 @@
  */
 
 import python
-import semmle.python.security.performance.ExponentialBackTracking
+import semmle.python.security.regexp.ExponentialBackTracking
 
 from RegExpTerm t, string pump, State s, string prefixMsg
 where
diff --git a/python/ql/test/query-tests/Security/CWE-730-PolynomialReDoS/PolynomialBackTracking.ql b/python/ql/test/query-tests/Security/CWE-730-PolynomialReDoS/PolynomialBackTracking.ql
@@ -1,5 +1,5 @@
 import python
-import semmle.python.security.performance.SuperlinearBackTracking
+import semmle.python.security.regexp.SuperlinearBackTracking
 
 from PolynomialBackTrackingTerm t
 select t.getRegex(), t, t.getReason()
diff --git a/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll b/ruby/ql/lib/codeql/ruby/security/BadTagFilterQuery.qll
@@ -2,7 +2,7 @@
  * Provides precicates for reasoning about bad tag filter vulnerabilities.
  */
 
-import performance.RegexpMatching
+import regexp.RegexpMatching
 
 /**
  * Holds if the regexp `root` should be tested against `str`.
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll b/ruby/ql/lib/codeql/ruby/security/regexp/ExponentialBackTracking.qll
@@ -62,7 +62,7 @@
  *     a suffix `x` (possible empty) that is most likely __not__ accepted.
  */
 
-import ReDoSUtil
+import NfaUtils
 
 /**
  * Holds if state `s` might be inside a backtracking repetition.
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll b/ruby/ql/lib/codeql/ruby/security/regexp/NfaUtils.qll
@@ -1,18 +1,13 @@
 /**
- * Provides classes for working with regular expressions that can
- * perform backtracking in superlinear/exponential time.
+ * Provides classes and predicates for constructing an NFA from
+ * a regular expression, and various utilities for reasoning about
+ * the resulting NFA.
  *
- * This module contains a number of utility predicates for compiling a regular expression into a NFA and reasoning about this NFA.
- *
- * The `ReDoSConfiguration` contains a `isReDoSCandidate` predicate that is used to
- * to determine which states the prefix/suffix search should happen on.
- * There is only meant to exist one `ReDoSConfiguration` at a time.
- *
- * The predicate `hasReDoSResult` outputs a de-duplicated set of
- * states that will cause backtracking (a rejecting suffix exists).
+ * These utilities are used both by the ReDoS queries and by
+ * other queries that benefit from reasoning about NFAs.
  */
 
-import ReDoSUtilSpecific
+import NfaUtilsSpecific
 
 /**
  * Gets the char after `c` (from a simplified ASCII table).
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/NfaUtilsSpecific.qll b/ruby/ql/lib/codeql/ruby/security/regexp/NfaUtilsSpecific.qll
@@ -1,5 +1,5 @@
 /**
- * Provides Ruby-specific definitions for use in the ReDoSUtil module.
+ * Provides Ruby-specific definitions for use in the NfaUtils module.
  */
 
 import codeql.ruby.Regexp
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/PolynomialReDoSCustomizations.qll b/ruby/ql/lib/codeql/ruby/security/regexp/PolynomialReDoSCustomizations.qll
@@ -9,7 +9,7 @@ private import codeql.ruby.CFG
 private import codeql.ruby.DataFlow
 private import codeql.ruby.dataflow.RemoteFlowSources
 private import codeql.ruby.Regexp
-private import codeql.ruby.security.performance.SuperlinearBackTracking
+private import codeql.ruby.security.regexp.SuperlinearBackTracking
 
 module PolynomialReDoS {
   /**
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/PolynomialReDoSQuery.qll b/ruby/ql/lib/codeql/ruby/security/regexp/PolynomialReDoSQuery.qll
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/RegExpInjectionCustomizations.qll b/ruby/ql/lib/codeql/ruby/security/regexp/RegExpInjectionCustomizations.qll
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/RegExpInjectionQuery.qll b/ruby/ql/lib/codeql/ruby/security/regexp/RegExpInjectionQuery.qll
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll b/ruby/ql/lib/codeql/ruby/security/regexp/RegexpMatching.qll
diff --git a/ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll b/ruby/ql/lib/codeql/ruby/security/regexp/SuperlinearBackTracking.qll
diff --git a/ruby/ql/src/queries/security/cwe-1333/PolynomialReDoS.ql b/ruby/ql/src/queries/security/cwe-1333/PolynomialReDoS.ql
diff --git a/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql b/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql
diff --git a/ruby/ql/src/queries/security/cwe-1333/RegExpInjection.ql b/ruby/ql/src/queries/security/cwe-1333/RegExpInjection.ql
