atom is no longer afraid of minified and transpiled js code (#217)

prabhu · web-flow · commit fbfc6ce915aa · 2025-09-19T18:15:21.000+01:00
* Tweaks

Signed-off-by: Prabhu Subramanian &lt;prabhu@appthreat.com&gt;

---------

Signed-off-by: Prabhu Subramanian &lt;prabhu@appthreat.com&gt;
diff --git a/.github/workflows/nodejstests.yml b/.github/workflows/nodejstests.yml
@@ -198,28 +198,3 @@ jobs:
         shell: cmd
         run: |
           atom -l js --remove-atom -o %GITHUB_WORKSPACE%\\repotests\\shiftleft-ts-example\\app.atom %GITHUB_WORKSPACE%\\repotests\\shiftleft-ts-example
-
-  devenv:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-15]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v26
-      - uses: cachix/cachix-action@v14
-        with:
-          name: devenv
-      - name: Install devenv.sh
-        run: nix profile install nixpkgs#devenv
-      - name: Build the devenv shell
-        run: devenv test
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run atom --help
-        run: |
-          devenv shell -- atom --help
-          devenv shell -- cdxgen --help
-          devenv shell -- atom-tools --help
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -105,7 +105,6 @@ jobs:
       - run: |
           bash ci/native-image.sh
           ./target/graalvm-native-image/atom --remove-atom -o /tmp/java.atom -l java $GITHUB_WORKSPACE/repotests/shiftleft-java-example -Dlog4j.configurationFile=log4j2.xml
-          ./target/graalvm-native-image/atom --remove-atom -o /tmp/juice.atom -l js $GITHUB_WORKSPACE/repotests/juice-shop -Dlog4j.configurationFile=log4j2.xml
         env:
           JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -1,8 +1,9 @@
 name: repotests
 on:
+  pull_request:
   push:
     branches:
-      - feature/*
+      - main
   workflow_dispatch:
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
diff --git a/build.sbt b/build.sbt
@@ -1,9 +1,9 @@
 name                     := "atom"
 ThisBuild / organization := "io.appthreat"
-ThisBuild / version      := "2.4.0"
+ThisBuild / version      := "2.4.1"
 ThisBuild / scalaVersion := "3.7.3"
 
-val chenVersion = "2.5.1"
+val chenVersion = "2.5.3"
 
 lazy val atom = Projects.atom
 
diff --git a/codemeta.json b/codemeta.json
@@ -7,7 +7,7 @@
   "downloadUrl": "https://github.com/AppThreat/atom",
   "issueTracker": "https://github.com/AppThreat/atom/issues",
   "name": "atom",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "description": "Atom is a novel intermediate representation for next-generation code analysis.",
   "applicationCategory": "code-analysis",
   "keywords": [
diff --git a/wrapper/nodejs/package-lock.json b/wrapper/nodejs/package-lock.json
diff --git a/wrapper/nodejs/package.json b/wrapper/nodejs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "description": "Create atom (⚛) representation for your application, packages and libraries",
   "exports": "./index.js",
   "type": "module",
diff --git a/wrapper/nodejs/packages/atom-common/package.json b/wrapper/nodejs/packages/atom-common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@appthreat/atom-common",
-  "version": "1.0.9",
+  "version": "1.0.10",
   "description": "Common library for the @appthreat/atom project.",
   "main": "index.js",
   "type": "module",
diff --git a/wrapper/nodejs/packages/atom-common/utils.mjs b/wrapper/nodejs/packages/atom-common/utils.mjs
@@ -5,11 +5,8 @@ import { spawnSync } from "node:child_process";
 const IGNORE_DIRS = process.env.ASTGEN_IGNORE_DIRS
   ? process.env.ASTGEN_IGNORE_DIRS.split(",")
   : [
-      "node_modules",
       "venv",
       "docs",
-      "test",
-      "tests",
       "e2e",
       "e2e-beta",
       "examples",
@@ -19,30 +16,24 @@ const IGNORE_DIRS = process.env.ASTGEN_IGNORE_DIRS
       "codemods",
       "flow-typed",
       "i18n",
-      "vendor",
-      "www",
-      "dist",
-      "build",
-      "__tests__"
     ];
 
 const IGNORE_FILE_PATTERN = new RegExp(
   process.env.ASTGEN_IGNORE_FILE_PATTERN ||
-    "(test|spec|min|three|\\.d)\\.(js|ts|jsx|tsx)$",
+    "(three|\\.d)\\.(js|ts|jsx|tsx)$",
   "i"
 );
 
-export const getAllFiles = (dir, extn, files, result, regex) => {
+export const getAllFiles = (dir, extn, files, result, regex, ignore_node_modules=true) => {
   files = files || readdirSync(dir);
   result = result || [];
   regex = regex || new RegExp(`\\${extn}$`);
 
   for (let i = 0; i < files.length; i++) {
     const file = files[i];
     if (
-      file.startsWith(".") ||
-      file.startsWith("__") ||
-      IGNORE_FILE_PATTERN.test(file)
+          file.startsWith(".") ||
+          IGNORE_FILE_PATTERN.test(file)
     ) {
       continue;
     }
@@ -52,8 +43,8 @@ export const getAllFiles = (dir, extn, files, result, regex) => {
       const dirName = basename(fileWithDir);
       if (
         dirName.startsWith(".") ||
-        dirName.startsWith("__") ||
-        IGNORE_DIRS.includes(dirName.toLowerCase())
+        IGNORE_DIRS.includes(dirName.toLowerCase()) ||
+        (ignore_node_modules && dirName.toLowerCase() === "node_modules")
       ) {
         continue;
       }
@@ -63,7 +54,8 @@ export const getAllFiles = (dir, extn, files, result, regex) => {
           extn,
           readdirSync(fileWithDir),
           result,
-          regex
+          regex,
+          ignore_node_modules
         );
       } catch (error) {
         // ignore
diff --git a/wrapper/nodejs/packages/atom-parsetools/.eslintrc.cjs b/wrapper/nodejs/packages/atom-parsetools/.eslintrc.cjs
@@ -1,15 +1,13 @@
 module.exports = {
-    "env": {
-        "node": true,
-        "es2021": true
-    },
-    "extends": "eslint:recommended",
-    "overrides": [
-    ],
-    "parserOptions": {
-        "ecmaVersion": "latest",
-        "sourceType": "module"
-    },
-    "rules": {
-    }
-}
+  env: {
+    node: true,
+    es2021: true
+  },
+  extends: "eslint:recommended",
+  overrides: [],
+  parserOptions: {
+    ecmaVersion: "latest",
+    sourceType: "module"
+  },
+  rules: {}
+};
diff --git a/wrapper/nodejs/packages/atom-parsetools/astgen.js b/wrapper/nodejs/packages/atom-parsetools/astgen.js
@@ -59,17 +59,53 @@ const babelSafeParserOptions = {
 
 /**
  * Return paths to all (j|tsx?) files.
+ * Optionally includes specific bundled files from node_modules if:
+ * 1. ASTGEN_INCLUDE_NODE_MODULES_BUNDLES is "true", OR
+ * 2. ASTGEN_IGNORE_DIRS is non-empty and doesn't include "node_modules"
  */
-const getAllSrcJSAndTSFiles = (src) =>
-  Promise.all([
+const getAllSrcJSAndTSFiles = (src) => {
+  const filePattern = "\\.(js|jsx|cjs|mjs|ts|tsx|vue|svelte|xsjs|xsjslib|ejs)$";
+  const bundledNodeModulesPattern =
+    "node_modules/.*\\.(bundle|dist|index|min|app)\\.(js|cjs|mjs)$";
+
+  // Step 1: Collect all JS/TS files EXCLUDING node_modules
+  const allFilesPromise = Promise.resolve(
     getAllFiles(
       src,
       undefined,
       undefined,
       undefined,
-      new RegExp("\\.(js|jsx|cjs|mjs|ts|tsx|vue|svelte)$")
+      new RegExp(filePattern),
+      true // ignore node_modules
     )
-  ]);
+  );
+
+  // Step 2: Check if we should include node_modules bundles
+  const shouldIncludeNodeModulesBundles =
+    process.env?.ASTGEN_INCLUDE_NODE_MODULES_BUNDLES === "true" ||
+    (process.env?.ASTGEN_IGNORE_DIRS &&
+      process.env.ASTGEN_IGNORE_DIRS.length > 0 &&
+      !process.env.ASTGEN_IGNORE_DIRS.toLowerCase().includes("node_modules"));
+
+  let bundledFilesPromise = Promise.resolve([]);
+  if (shouldIncludeNodeModulesBundles) {
+    bundledFilesPromise = Promise.resolve(
+      getAllFiles(
+        src,
+        undefined,
+        undefined,
+        undefined,
+        new RegExp(bundledNodeModulesPattern),
+        false // DO NOT ignore node_modules
+      )
+    );
+  }
+
+  // Step 3: Combine both lists
+  return Promise.all([allFilesPromise, bundledFilesPromise]).then(
+    ([allFiles, bundledFiles]) => [...allFiles, ...bundledFiles]
+  );
+};
 
 /**
  * Convert a single JS/TS file to AST
@@ -242,30 +278,44 @@ const createJSAst = async (options) => {
   try {
     const promiseMap = await getAllSrcJSAndTSFiles(options.src);
     const srcFiles = promiseMap.flatMap((d) => d);
+    const CONCURRENCY_LIMIT = 10;
+    const chunks = [];
+    for (let i = 0; i < srcFiles.length; i += CONCURRENCY_LIMIT) {
+      chunks.push(srcFiles.slice(i, i + CONCURRENCY_LIMIT));
+    }
     let ts;
     if (options.tsTypes) {
-      ts = createTsc(srcFiles);
+      const projectFiles = srcFiles.filter(
+        (file) => !file.includes("node_modules")
+      );
+      ts = createTsc(projectFiles);
     }
-    for (const file of srcFiles) {
+    for (const chunk of chunks) {
+      await Promise.all(chunk.map((file) => processFile(file, options, ts)));
+    }
+  } catch (err) {
+    console.error(err);
+  }
+};
+
+const processFile = (file, options, ts) => {
+  try {
+    const ast = fileToJsAst(file);
+    writeAstFile(file, ast, options);
+    if (ts) {
       try {
-        const ast = fileToJsAst(file);
-        writeAstFile(file, ast, options);
-      } catch (err) {
-        console.error(file, err.message);
-      }
-      if (ts) {
-        try {
-          const tsAst = ts.program.getSourceFile(file);
+        const tsAst = ts.program.getSourceFile(file);
+        if (tsAst) {
           tsc.forEachChild(tsAst, ts.addType);
           writeTypesFile(file, ts.seenTypes, options);
           ts.seenTypes.clear();
-        } catch (err) {
-          console.warn("Retrieving types", file, ":", err.message);
         }
+      } catch (err) {
+        console.warn("Retrieving types", file, ":", err.message);
       }
     }
   } catch (err) {
-    console.error(err);
+    console.error(file, err.message);
   }
 };
 
diff --git a/wrapper/nodejs/packages/atom-parsetools/package.json b/wrapper/nodejs/packages/atom-parsetools/package.json
@@ -1,15 +1,15 @@
 {
   "name": "@appthreat/atom-parsetools",
-  "version": "1.0.9",
+  "version": "1.0.10",
   "description": "Parsing tools that complement the @appthreat/atom project.",
   "main": "./index.js",
   "type": "module",
   "scripts": {
-    "pretty": "prettier --write *.mjs *.js --trailing-comma=none",
+    "pretty": "prettier --write *.cjs *.js --trailing-comma=none",
     "lint": "eslint *.mjs *.js"
   },
   "dependencies": {
-    "@appthreat/atom-common": "^1.0.9",
+    "@appthreat/atom-common": "^1.0.10",
     "@babel/parser": "^7.28.4",
     "typescript": "^5.9.2",
     "yargs": "^17.7.2"

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@appthreat/atom",`
`3`		`- "version": "2.4.0",`
	`3`	`+ "version": "2.4.1",`
`4`	`4`	`"description": "Create atom (⚛) representation for your application, packages and libraries",`
`5`	`5`	`"exports": "./index.js",`
`6`	`6`	`"type": "module",`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@appthreat/atom-common",`
`3`		`- "version": "1.0.9",`
	`3`	`+ "version": "1.0.10",`
`4`	`4`	`"description": "Common library for the @appthreat/atom project.",`
`5`	`5`	`"main": "index.js",`
`6`	`6`	`"type": "module",`