Merge pull request #4310 from asgerf/js/extract-xml-with-codeql

codeql-ci · web-flow · commit 19316930cd2c · 2020-09-24T10:14:46.000-07:00
Approved by aibaars, esbena
diff --git a/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java b/javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java
@@ -60,6 +60,7 @@
 import com.semmle.util.io.csv.CSVReader;
 import com.semmle.util.language.LegacyLanguage;
 import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.OS;
 import com.semmle.util.projectstructure.ProjectLayout;
 import com.semmle.util.trap.TrapWriter;
 
@@ -1239,11 +1240,29 @@ public Set<String> getXmlExtensions() {
   protected void extractXml() throws IOException {
     if (xmlExtensions.isEmpty()) return;
     List<String> cmd = new ArrayList<>();
-    cmd.add("odasa");
-    cmd.add("index");
-    cmd.add("--xml");
-    cmd.add("--extensions");
-    cmd.addAll(xmlExtensions);
+    if (EnvironmentVariables.getCodeQLDist() == null) {
+      // Use the legacy odasa XML extractor
+      cmd.add("odasa");
+      cmd.add("index");
+      cmd.add("--xml");
+      cmd.add("--extensions");
+      cmd.addAll(xmlExtensions);
+    } else {
+      String command = Env.getOS() == OS.WINDOWS ? "codeql.exe" : "codeql";
+      cmd.add(Paths.get(EnvironmentVariables.getCodeQLDist(), command).toString());
+      cmd.add("database");
+      cmd.add("index-files");
+      cmd.add("--language");
+      cmd.add("xml");
+      cmd.add("--size-limit");
+      cmd.add("10m");
+      for (String extension : xmlExtensions) {
+        cmd.add("--include-extension");
+        cmd.add(extension);
+      }
+      cmd.add("--");
+      cmd.add(EnvironmentVariables.getWipDatabase());
+    }
     ProcessBuilder pb = new ProcessBuilder(cmd);
     try {
       pb.redirectError(Redirect.INHERIT);
diff --git a/javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java b/javascript/extractor/src/com/semmle/js/extractor/EnvironmentVariables.java
@@ -14,6 +14,11 @@ public class EnvironmentVariables {
   public static final String LGTM_WORKSPACE_ENV_VAR =
       "LGTM_WORKSPACE";
 
+  public static final String CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE_ENV_VAR =
+      "CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE";
+
+  public static final String CODEQL_DIST_ENV_VAR = "CODEQL_DIST";
+
   /**
    * Gets the extractor root based on the <code>CODEQL_EXTRACTOR_JAVASCRIPT_ROOT</code> or <code>
    * SEMMLE_DIST</code> or environment variable, or <code>null</code> if neither is set.
@@ -49,4 +54,13 @@ public static String getScratchDir() {
 
     throw new UserError(CODEQL_EXTRACTOR_JAVASCRIPT_SCRATCH_DIR_ENV_VAR + " or " + LGTM_WORKSPACE_ENV_VAR + " must be set");
   }
+
+  public static String getCodeQLDist() {
+    return Env.systemEnv().getNonEmpty(CODEQL_DIST_ENV_VAR);
+  }
+
+  /** Gets the output database directory. */
+  public static String getWipDatabase() {
+    return Env.systemEnv().getNonEmpty(CODEQL_EXTRACTOR_JAVASCRIPT_WIP_DATABASE_ENV_VAR);
+  }
 }
