-
Notifications
You must be signed in to change notification settings - Fork 49
[WIP] Parallelize env var reads #163
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,7 +2,12 @@ import dotenv from "dotenv"; | |
| import * as core from "@actions/core"; | ||
| import { validateCli } from "@1password/op-js"; | ||
| import { installCliOnGithubActionRunner } from "./op-cli-installer"; | ||
| import { loadSecrets, unsetPrevious, validateAuth } from "./utils"; | ||
| import { | ||
| loadSecrets, | ||
| loadSecretsFromEnvFileBatched, | ||
| unsetPrevious, | ||
| validateAuth, | ||
| } from "./utils"; | ||
| import { envFilePath } from "./constants"; | ||
|
|
||
| const loadSecretsAction = async () => { | ||
|
|
@@ -30,7 +35,11 @@ const loadSecretsAction = async () => { | |
| await installCLI(); | ||
|
|
||
| // Load secrets | ||
| await loadSecrets(shouldExportEnv); | ||
| if (file) { | ||
| await loadSecretsFromEnvFileBatched(file, shouldExportEnv); | ||
| } else { | ||
| await loadSecrets(shouldExportEnv); | ||
| } | ||
| } catch (error) { | ||
| // It's possible for the Error constructor to be modified to be anything | ||
| // in JavaScript, so the following code accounts for this possibility. | ||
|
|
@@ -39,7 +48,7 @@ const loadSecretsAction = async () => { | |
| if (error instanceof Error) { | ||
| message = error.message; | ||
| } else { | ||
| String(error); | ||
| message = String(error); | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unrelated, but useful change. I can open a separate PR for it. |
||
| } | ||
| core.setFailed(message); | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,11 @@ | ||
| import fs from "node:fs"; | ||
| import * as core from "@actions/core"; | ||
| import * as exec from "@actions/exec"; | ||
| import { read, setClientInfo } from "@1password/op-js"; | ||
| import { | ||
| extractSecret, | ||
| loadSecrets, | ||
| loadSecretsFromEnvFileBatched, | ||
| unsetPrevious, | ||
| validateAuth, | ||
| } from "./utils"; | ||
|
|
@@ -16,6 +18,13 @@ import { | |
| } from "./constants"; | ||
|
|
||
| jest.mock("@1password/op-js"); | ||
| jest.mock("node:fs", () => ({ | ||
| // eslint-disable-next-line @typescript-eslint/naming-convention | ||
| __esModule: true, | ||
| default: { | ||
| readFileSync: jest.fn(), | ||
| }, | ||
| })); | ||
|
|
||
| beforeEach(() => { | ||
| jest.clearAllMocks(); | ||
|
|
@@ -141,11 +150,13 @@ describe("loadSecrets", () => { | |
| it("sets the client info and gets the executed output", async () => { | ||
| await loadSecrets(true); | ||
|
|
||
| expect(setClientInfo).toHaveBeenCalledWith({ | ||
| name: "1Password GitHub Action", | ||
| id: "GHA", | ||
| }); | ||
| expect(exec.getExecOutput).toHaveBeenCalledWith('sh -c "op env ls"'); | ||
| expect(setClientInfo).toHaveBeenCalledWith( | ||
| expect.objectContaining({ | ||
| name: "1Password GitHub Action", | ||
| id: "GHA", | ||
| }), | ||
| ); | ||
|
Comment on lines
+153
to
+158
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unrelated change. I can revert that :) |
||
| expect(exec.getExecOutput).toHaveBeenCalledWith("op", ["env", "ls"]); | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also unrelated, see my comment at the implementation |
||
| expect(core.exportVariable).toHaveBeenCalledWith( | ||
| "OP_MANAGED_VARIABLES", | ||
| "MOCK_SECRET", | ||
|
|
@@ -156,7 +167,7 @@ describe("loadSecrets", () => { | |
| (exec.getExecOutput as jest.Mock).mockReturnValueOnce({ stdout: "" }); | ||
| await loadSecrets(true); | ||
|
|
||
| expect(exec.getExecOutput).toHaveBeenCalledWith('sh -c "op env ls"'); | ||
| expect(exec.getExecOutput).toHaveBeenCalledWith("op", ["env", "ls"]); | ||
| expect(core.exportVariable).not.toHaveBeenCalled(); | ||
| }); | ||
|
|
||
|
|
@@ -175,6 +186,61 @@ describe("loadSecrets", () => { | |
| }); | ||
| }); | ||
|
|
||
| describe("loadSecretsFromEnvFileBatched", () => { | ||
| const envFilePath = "/tmp/test.env"; | ||
|
|
||
| beforeEach(() => { | ||
| jest.clearAllMocks(); | ||
| }); | ||
|
|
||
| it("loads secrets from env file and sets them as outputs", async () => { | ||
| (fs.readFileSync as unknown as jest.Mock).mockReturnValue( | ||
| Buffer.from("FOO=op://vault/item/foo\nBAR=op://vault/item/bar\n"), | ||
| ); | ||
| (exec.getExecOutput as jest.Mock).mockReturnValueOnce({ | ||
| stdout: JSON.stringify({ FOO: "foo-value", BAR: "bar-value" }), | ||
| }); | ||
|
|
||
| await loadSecretsFromEnvFileBatched(envFilePath, false); | ||
|
|
||
| expect(exec.getExecOutput).toHaveBeenCalledWith( | ||
| "op", | ||
| expect.arrayContaining([ | ||
| "run", | ||
| `--env-file=${envFilePath}`, | ||
| "--no-masking", | ||
| ]), | ||
| expect.anything(), | ||
| ); | ||
| expect(core.setOutput).toHaveBeenCalledWith("FOO", "foo-value"); | ||
| expect(core.setOutput).toHaveBeenCalledWith("BAR", "bar-value"); | ||
| expect(core.exportVariable).not.toHaveBeenCalledWith( | ||
| "OP_MANAGED_VARIABLES", | ||
| expect.anything(), | ||
| ); | ||
| expect(core.setSecret).toHaveBeenCalledWith("foo-value"); | ||
| expect(core.setSecret).toHaveBeenCalledWith("bar-value"); | ||
| }); | ||
|
|
||
| it("loads secrets from env file and exports them as env vars (including managed list)", async () => { | ||
| (fs.readFileSync as unknown as jest.Mock).mockReturnValue( | ||
| Buffer.from("FOO=op://vault/item/foo\nBAR=op://vault/item/bar\n"), | ||
| ); | ||
| (exec.getExecOutput as jest.Mock).mockReturnValueOnce({ | ||
| stdout: JSON.stringify({ FOO: "foo-value", BAR: "bar-value" }), | ||
| }); | ||
|
|
||
| await loadSecretsFromEnvFileBatched(envFilePath, true); | ||
|
|
||
| expect(core.exportVariable).toHaveBeenCalledWith("FOO", "foo-value"); | ||
| expect(core.exportVariable).toHaveBeenCalledWith("BAR", "bar-value"); | ||
| expect(core.exportVariable).toHaveBeenCalledWith( | ||
| envManagedVariables, | ||
| "FOO,BAR", | ||
| ); | ||
| }); | ||
| }); | ||
|
|
||
| describe("unsetPrevious", () => { | ||
| const testManagedEnv = "TEST_SECRET"; | ||
| const testSecretValue = "MyS3cr#T"; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,5 @@ | ||
| import fs from "node:fs"; | ||
| import dotenv from "dotenv"; | ||
| import * as core from "@actions/core"; | ||
| import * as exec from "@actions/exec"; | ||
| import { read, setClientInfo, semverToInt } from "@1password/op-js"; | ||
|
|
@@ -10,6 +12,8 @@ import { | |
| envManagedVariables, | ||
| } from "./constants"; | ||
|
|
||
| const envFileKeysEnvVar = "OP_KEYS_JSON"; | ||
|
|
||
| export const validateAuth = (): void => { | ||
| const isConnect = process.env[envConnectHost] && process.env[envConnectToken]; | ||
| const isServiceAccount = process.env[envServiceAccountToken]; | ||
|
|
@@ -57,6 +61,84 @@ export const extractSecret = ( | |
| } | ||
| }; | ||
|
|
||
| const exportResolvedSecret = ( | ||
| envName: string, | ||
| secretValue: string, | ||
| shouldExportEnv: boolean, | ||
| ): void => { | ||
| core.info(`Populating variable: ${envName}`); | ||
| if (shouldExportEnv) { | ||
| core.exportVariable(envName, secretValue); | ||
| } else { | ||
| core.setOutput(envName, secretValue); | ||
| } | ||
| // Skip setSecret for empty strings to avoid the warning: | ||
| // "Can't add secret mask for empty string in ##[add-mask] command." | ||
| if (secretValue) { | ||
| core.setSecret(secretValue); | ||
| } | ||
| }; | ||
|
|
||
| export const loadSecretsFromEnvFileBatched = async ( | ||
| envFile: string, | ||
| shouldExportEnv: boolean, | ||
| ): Promise<void> => { | ||
| const envFileBuf = fs.readFileSync(envFile); | ||
| const envFileVars = dotenv.parse(envFileBuf); | ||
| const envNames = Object.keys(envFileVars); | ||
| if (envNames.length === 0) { | ||
| return; | ||
| } | ||
|
|
||
| // Pass User-Agent Information to the 1Password CLI | ||
| setClientInfo({ | ||
| name: "1Password GitHub Action", | ||
| id: "GHA", | ||
| build: semverToInt(version), | ||
| }); | ||
|
|
||
| // Resolve all secrets in a single `op run --env-file` call, then emit just the | ||
| // keys we care about as JSON. `silent: true` ensures values never hit logs. | ||
| const nodeScript = | ||
| 'const keys=JSON.parse(process.env.OP_KEYS_JSON||"[]");const out={};for(const k of keys){out[k]=process.env[k]??"";}process.stdout.write(JSON.stringify(out));'; | ||
|
|
||
| const res = await exec.getExecOutput( | ||
| "op", | ||
| [ | ||
| "run", | ||
| `--env-file=${envFile}`, | ||
| "--no-masking", | ||
| "--", | ||
| "node", | ||
| "-e", | ||
| nodeScript, | ||
| ], | ||
| { | ||
| silent: true, | ||
| env: { | ||
| ...process.env, | ||
| [envFileKeysEnvVar]: JSON.stringify(envNames), | ||
| }, | ||
| }, | ||
| ); | ||
|
Comment on lines
+102
to
+123
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I can put |
||
|
|
||
| let resolved: Record<string, string> = {}; | ||
| try { | ||
| resolved = JSON.parse(res.stdout) as Record<string, string>; | ||
| } catch { | ||
| throw new Error("Failed to parse secrets resolved from 1Password CLI."); | ||
| } | ||
|
|
||
| for (const envName of envNames) { | ||
| const secretValue = resolved[envName] ?? ""; | ||
| exportResolvedSecret(envName, secretValue, shouldExportEnv); | ||
| } | ||
|
|
||
| if (shouldExportEnv) { | ||
| core.exportVariable(envManagedVariables, envNames.join()); | ||
| } | ||
| }; | ||
|
|
||
| export const loadSecrets = async (shouldExportEnv: boolean): Promise<void> => { | ||
| // Pass User-Agent Information to the 1Password CLI | ||
| setClientInfo({ | ||
|
|
@@ -68,7 +150,7 @@ export const loadSecrets = async (shouldExportEnv: boolean): Promise<void> => { | |
| // Load secrets from environment variables using 1Password CLI. | ||
| // Iterate over them to find 1Password references, extract the secret values, | ||
| // and make them available in the next steps either as step outputs or as environment variables. | ||
| const res = await exec.getExecOutput(`sh -c "op env ls"`); | ||
| const res = await exec.getExecOutput("op", ["env", "ls"]); | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This change is also unrelated. Personally, I think the change makes sense, but maybe there's a specific reason for |
||
|
|
||
| if (res.stdout === "") { | ||
| return; | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you want the
distto be included in the PR?