From 538cced1fc37efc6342efacbd0a25ba8739da18e Mon Sep 17 00:00:00 2001 From: nams1570 Date: Fri, 16 Jan 2026 13:11:14 -0800 Subject: [PATCH 1/5] fix: Vercel Sandbox runner error handling On Sentry, we saw an error with vercel sandbox. This was thrown because the runner script itself had no safeguards. Also, the error raised was uninformative. So we improve the error handling in the file. --- apps/backend/src/lib/email-rendering.tsx | 2 +- apps/backend/src/lib/js-execution.tsx | 31 ++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/apps/backend/src/lib/email-rendering.tsx b/apps/backend/src/lib/email-rendering.tsx index 801858ec2f..8e992a8808 100644 --- a/apps/backend/src/lib/email-rendering.tsx +++ b/apps/backend/src/lib/email-rendering.tsx @@ -69,7 +69,7 @@ const entryJs = deindent` type EmailRenderResult = { html: string, text: string, subject?: string, notificationCategory?: string }; type ExecuteResult = | { status: "ok", data: unknown } - | { status: "error", error: string }; + | { status: "error", error: unknown }; async function bundleAndExecute( files: Record & { '/entry.js': string } diff --git a/apps/backend/src/lib/js-execution.tsx b/apps/backend/src/lib/js-execution.tsx index 30dc0bf770..1aa17d2b17 100644 --- a/apps/backend/src/lib/js-execution.tsx +++ b/apps/backend/src/lib/js-execution.tsx @@ -83,9 +83,24 @@ function createVercelSandboxEngine(): JsEngine { const runnerScript = ` import { writeFileSync } from 'fs'; - import fn from './code.mjs'; - const result = await fn(); - writeFileSync('${resultPath}', JSON.stringify(result)); + let result; + let resultAsJson; + try { + const {default: fn} = await import('./code.mjs'); + result = await fn(); + if (result === undefined) { + result = { status: 'error', error: { message: 'Execution of innerCode returned undefined' } }; + } + resultAsJson = JSON.stringify(result); + } catch (e) { + if(e instanceof Error) { + result = { status: 'error', error: { message: e.message, stack: e.stack, cause: e.cause} }; + } else { + result = { status: 'error', error: { message: String(e), stack: undefined, cause: undefined} }; + } + resultAsJson = JSON.stringify(result); + } + writeFileSync('${resultPath}', resultAsJson); `; await sandbox.writeFiles([ @@ -96,7 +111,9 @@ function createVercelSandboxEngine(): JsEngine { const runResult = await sandbox.runCommand('node', ['/vercel/sandbox/runner.mjs']); if (runResult.exitCode !== 0) { - throw new StackAssertionError("Vercel Sandbox execution failed", { exitCode: runResult.exitCode }); + // This shouldn't happen since the runner has a try-catch wrapper. + // If we get here, something unexpected failed (OOM, timeout, disk error, etc.) + throw new StackAssertionError("Vercel Sandbox runner failed unexpectedly outside of runner safeguards. This should never happen.", { innerCode:code, innerOptions:options, exitCode: runResult.exitCode }); } // Read the result file by catting it to stdout @@ -112,13 +129,13 @@ function createVercelSandboxEngine(): JsEngine { const catResult = await sandbox.runCommand({ cmd: 'cat', args: [resultPath], stdout: stdoutStream }); if (catResult.exitCode !== 0) { - throw new StackAssertionError("Failed to read result file from Vercel Sandbox", { exitCode: catResult.exitCode }); + throw new StackAssertionError("Failed to read result file from Vercel Sandbox", { exitCode: catResult.exitCode, innerCode:code, innerOptions:options }); } try { return JSON.parse(resultJson); - } catch (e) { - throw new StackAssertionError("Failed to parse result from Vercel Sandbox", { resultJson, cause: e }); + } catch (e: any) { + throw new StackAssertionError("Failed to parse result from Vercel Sandbox", { resultJson, cause: e, innerCode:code, innerOptions:options }); } } finally { await sandbox.stop(); From 24816c1f4632cf1c8901ac8fe78c56f8a7669f02 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Fri, 16 Jan 2026 18:13:36 -0800 Subject: [PATCH 2/5] fix: better error handling for email-rendering We had a small bug where when the vercel fallback failed, we still returned the freestyle failure. We also make errors more informative. --- apps/backend/src/lib/email-rendering.tsx | 23 +++++++++++++++++++---- apps/backend/src/lib/js-execution.tsx | 16 ++++++++-------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/apps/backend/src/lib/email-rendering.tsx b/apps/backend/src/lib/email-rendering.tsx index 8e992a8808..0fd2529b6f 100644 --- a/apps/backend/src/lib/email-rendering.tsx +++ b/apps/backend/src/lib/email-rendering.tsx @@ -61,7 +61,10 @@ const entryJs = deindent` const result = await renderAll(); return { status: "ok", data: result }; } catch (e) { - return { status: "error", error: String(e) }; + if (e instanceof Error) { + return { status: "error", error: { message: e.message, stack: e.stack, cause: e.cause } }; + } + return { status: "error", error: { message: String(e), stack: undefined, cause: undefined } }; } }; `; @@ -87,7 +90,7 @@ async function bundleAndExecute( if (["development", "test"].includes(getNodeEnvironment())) { const executeResult = await executeJavascript(bundle.data, { nodeModules, engine: 'freestyle' }) as ExecuteResult; if (executeResult.status === "error") { - return Result.error(executeResult.error); + return Result.error(JSON.stringify(executeResult.error)); } return Result.ok(executeResult.data as T); } @@ -97,11 +100,23 @@ async function bundleAndExecute( if (executeResult.status === "error") { const vercelResult = await executeJavascript(bundle.data, { nodeModules, engine: 'vercel-sandbox' }) as ExecuteResult; if (vercelResult.status === "error") { - return Result.error(executeResult.error); + captureError("email-rendering-freestyle-and-vercel-runtime-error", new StackAssertionError( + "Email rendering failed with both freestyle and vercel-sandbox engines", + { + freestyleError: executeResult.error, + vercelError: vercelResult.error, + innerCode: bundle.data, + innerOptions: [ + { nodeModules, engine: 'freestyle' }, + { nodeModules, engine: 'vercel-sandbox' }, + ], + }, + )); + return Result.error(JSON.stringify(vercelResult.error)); } captureError("email-rendering-freestyle-runtime-error", new StackAssertionError( "Email rendering failed with freestyle but succeeded with vercel-sandbox", - { freestyleError: executeResult.error } + { freestyleError: executeResult.error, innerCode: bundle.data, innerOptions: { nodeModules, engine: 'freestyle' } } )); return Result.ok(vercelResult.data as T); } diff --git a/apps/backend/src/lib/js-execution.tsx b/apps/backend/src/lib/js-execution.tsx index 1aa17d2b17..e3531f3a31 100644 --- a/apps/backend/src/lib/js-execution.tsx +++ b/apps/backend/src/lib/js-execution.tsx @@ -42,7 +42,7 @@ function createFreestyleEngine(): JsEngine { }); if (response.result === undefined) { - throw new StackAssertionError("Freestyle execution returned undefined result", { response }); + throw new StackAssertionError("Freestyle execution returned undefined result", { response, innerCode: code, innerOptions: options }); } return response.result; @@ -75,7 +75,7 @@ function createVercelSandboxEngine(): JsEngine { const installResult = await sandbox.runCommand('npm', ['install', '--no-save', ...packages]); if (installResult.exitCode !== 0) { - throw new StackAssertionError("Failed to install packages in Vercel Sandbox", { exitCode: installResult.exitCode }); + throw new StackAssertionError("Failed to install packages in Vercel Sandbox", { exitCode: installResult.exitCode, innerCode: code, innerOptions: options }); } } @@ -113,7 +113,7 @@ function createVercelSandboxEngine(): JsEngine { if (runResult.exitCode !== 0) { // This shouldn't happen since the runner has a try-catch wrapper. // If we get here, something unexpected failed (OOM, timeout, disk error, etc.) - throw new StackAssertionError("Vercel Sandbox runner failed unexpectedly outside of runner safeguards. This should never happen.", { innerCode:code, innerOptions:options, exitCode: runResult.exitCode }); + throw new StackAssertionError("Vercel Sandbox runner failed unexpectedly outside of runner safeguards. This should never happen.", { innerCode: code, innerOptions: options, exitCode: runResult.exitCode }); } // Read the result file by catting it to stdout @@ -129,13 +129,13 @@ function createVercelSandboxEngine(): JsEngine { const catResult = await sandbox.runCommand({ cmd: 'cat', args: [resultPath], stdout: stdoutStream }); if (catResult.exitCode !== 0) { - throw new StackAssertionError("Failed to read result file from Vercel Sandbox", { exitCode: catResult.exitCode, innerCode:code, innerOptions:options }); + throw new StackAssertionError("Failed to read result file from Vercel Sandbox", { exitCode: catResult.exitCode, innerCode: code, innerOptions: options }); } try { return JSON.parse(resultJson); } catch (e: any) { - throw new StackAssertionError("Failed to parse result from Vercel Sandbox", { resultJson, cause: e, innerCode:code, innerOptions:options }); + throw new StackAssertionError("Failed to parse result from Vercel Sandbox", { resultJson, cause: e, innerCode: code, innerOptions: options }); } } finally { await sandbox.stop(); @@ -195,7 +195,7 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) { if (failures.length > 0) { captureError("js-execution-sanity-test-failures", new StackAssertionError( `JS execution sanity test: ${failures.length} engine(s) failed`, - { failures, successfulEngines: results.map(r => r.engine) } + { failures, successfulEngines: results.map(r => r.engine), innerCode: code, innerOptions: options } )); } @@ -208,7 +208,7 @@ async function runSanityTest(code: string, options: ExecuteJavascriptOptions) { if (!allEqual) { captureError("js-execution-sanity-test-mismatch", new StackAssertionError( "JS execution sanity test: engines returned different results", - { results } + { results, innerCode: code, innerOptions: options } )); } } @@ -245,7 +245,7 @@ async function runWithFallback(code: string, options: ExecuteJavascriptOptions): if (i < engines.length - 1) { captureError(`js-execution-${engine.name}-failed`, new StackAssertionError( `JS execution engine '${engine.name}' failed, falling back to next engine`, - { error: engineError, attempts: retryResult.attempts } + { error: engineError, attempts: retryResult.attempts, innerCode: code, innerOptions: options } )); } } From 9231e2f5350d245d05ef09c61d29d1d7204dce28 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Mon, 19 Jan 2026 22:42:59 -0800 Subject: [PATCH 3/5] chore: add test coverage for error paths in email-rendering --- apps/backend/src/lib/email-rendering.test.tsx | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/apps/backend/src/lib/email-rendering.test.tsx b/apps/backend/src/lib/email-rendering.test.tsx index 66d44bed78..ce28c54928 100644 --- a/apps/backend/src/lib/email-rendering.test.tsx +++ b/apps/backend/src/lib/email-rendering.test.tsx @@ -313,7 +313,7 @@ describe('renderEmailsForTenancyBatched', () => { }); describe('error handling', () => { - it('should return error for invalid template syntax', async () => { + it('bundling error: invalid syntax', async () => { const request = createMockRequest(1, { templateSource: 'invalid syntax {{{ not jsx', }); @@ -326,9 +326,13 @@ describe('renderEmailsForTenancyBatched', () => { } }); - it('should return error for invalid theme syntax', async () => { + it('bundling error: missing required export', async () => { const request = createMockRequest(1, { - themeSource: 'export function EmailTheme( { unclosed bracket', + templateSource: ` + export function WrongName() { + return
Wrong function name
; + } + `, }); const result = await renderEmailsForTenancyBatched([request]); @@ -338,12 +342,12 @@ describe('renderEmailsForTenancyBatched', () => { } }); - it('should return error when template does not export EmailTemplate', async () => { + it('runtime error: component throws (returns JSON with message and stack)', async () => { const request = createMockRequest(1, { templateSource: ` export const variablesSchema = (v: any) => v; - export function WrongName() { - return
Wrong function name
; + export function EmailTemplate() { + throw new Error('Template render failed'); } `, }); @@ -351,23 +355,55 @@ describe('renderEmailsForTenancyBatched', () => { expect(result.status).toBe('error'); if (result.status === 'error') { - expect(result.error).toBeDefined(); + expect(result.error).toContain('Template render failed'); + // Verify error is JSON with stack trace + const parsed = JSON.parse(result.error); + expect(parsed.message).toContain('Template render failed'); + expect(parsed.stack).toBeDefined(); } }); - it('should return error when theme does not export EmailTheme', async () => { + it('runtime error: arktype validation fails', async () => { const request = createMockRequest(1, { - themeSource: ` - export function WrongThemeName({ children }: any) { - return
{children}
; + templateSource: ` + import { type } from "arktype"; + export const variablesSchema = type({ requiredField: "string" }); + export function EmailTemplate({ variables }: any) { + return
{variables.requiredField}
; } `, + input: { + user: { displayName: 'User 1' }, + project: { displayName: 'Project 1' }, + variables: { wrongField: 'value' }, + }, }); const result = await renderEmailsForTenancyBatched([request]); expect(result.status).toBe('error'); if (result.status === 'error') { - expect(result.error).toBeDefined(); + expect(result.error).toContain('requiredField'); + } + }); + + it('batch behavior: single failure fails entire batch', async () => { + const requests = [ + createMockRequest(1), + createMockRequest(2, { + templateSource: ` + export const variablesSchema = (v: any) => v; + export function EmailTemplate() { + throw new Error('Second template error'); + } + `, + }), + createMockRequest(3), + ]; + const result = await renderEmailsForTenancyBatched(requests); + + expect(result.status).toBe('error'); + if (result.status === 'error') { + expect(result.error).toContain('Second template error'); } }); }); From cc82e5079c62a856568107805326e688fb3a8d77 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Mon, 19 Jan 2026 23:00:54 -0800 Subject: [PATCH 4/5] chore: add preview mode tests for renderEmailWithTemplate There is sufficient nuance/ complexity to dissuade merging the two renderEmailxxx functions. While the render-email.test.tsx covers the main path through renderEmailWithTemplate, preview mode is never explicitly tested. --- apps/backend/src/lib/email-rendering.test.tsx | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/apps/backend/src/lib/email-rendering.test.tsx b/apps/backend/src/lib/email-rendering.test.tsx index ce28c54928..b7e055c413 100644 --- a/apps/backend/src/lib/email-rendering.test.tsx +++ b/apps/backend/src/lib/email-rendering.test.tsx @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { renderEmailsForTenancyBatched, type RenderEmailRequestForTenancy } from './email-rendering'; +import { renderEmailsForTenancyBatched, renderEmailWithTemplate, type RenderEmailRequestForTenancy } from './email-rendering'; describe('renderEmailsForTenancyBatched', () => { const createSimpleTemplateSource = (content: string) => ` @@ -500,3 +500,55 @@ describe('renderEmailsForTenancyBatched', () => { }, 30000); // Extended timeout for large batch }); }); + +describe('renderEmailWithTemplate', () => { + const simpleTemplate = ` + export const variablesSchema = (v: any) => v; + export function EmailTemplate({ user, project }: any) { + return ( +
+ {user.displayName} + {project.displayName} +
+ ); + } + `; + + const simpleTheme = ` + export function EmailTheme({ children }: any) { + return
{children}
; + } + `; + + it('preview mode: uses default user and project when not provided', async () => { + const result = await renderEmailWithTemplate(simpleTemplate, simpleTheme, { + previewMode: true, + }); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data.html).toContain('John Doe'); + expect(result.data.html).toContain('My Project'); + } + }); + + it('preview mode: merges PreviewVariables from template', async () => { + const templateWithPreviewVars = ` + import { type } from "arktype"; + export const variablesSchema = type({ greeting: "string" }); + export function EmailTemplate({ variables }: any) { + return
{variables.greeting}
; + } + EmailTemplate.PreviewVariables = { greeting: "Hello from preview!" }; + `; + + const result = await renderEmailWithTemplate(templateWithPreviewVars, simpleTheme, { + previewMode: true, + }); + + expect(result.status).toBe('ok'); + if (result.status === 'ok') { + expect(result.data.html).toContain('Hello from preview!'); + } + }); +}); From 96f77621211167388eb4077b8206109a3066fc42 Mon Sep 17 00:00:00 2001 From: nams1570 Date: Tue, 20 Jan 2026 10:56:59 -0800 Subject: [PATCH 5/5] fix: simplify runner script and parsing We now read from a file buffer rather than cat. Cat returns the same result for an empty file and a nonexistent file. readFile makes it more explicit when the file wasn't created. We simplify runner to better match freestyle behavior --- apps/backend/package.json | 2 +- apps/backend/src/lib/js-execution.tsx | 48 ++++++++------------------- pnpm-lock.yaml | 10 +++--- 3 files changed, 19 insertions(+), 41 deletions(-) diff --git a/apps/backend/package.json b/apps/backend/package.json index 99ca7228db..d6f1f55ebf 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -77,7 +77,7 @@ "@upstash/qstash": "^2.8.2", "@vercel/functions": "^2.0.0", "@vercel/otel": "^1.10.4", - "@vercel/sandbox": "^1.1.2", + "@vercel/sandbox": "^1.2.0", "ai": "^4.3.17", "bcrypt": "^5.1.1", "chokidar-cli": "^3.0.0", diff --git a/apps/backend/src/lib/js-execution.tsx b/apps/backend/src/lib/js-execution.tsx index e3531f3a31..97cab57b44 100644 --- a/apps/backend/src/lib/js-execution.tsx +++ b/apps/backend/src/lib/js-execution.tsx @@ -83,24 +83,9 @@ function createVercelSandboxEngine(): JsEngine { const runnerScript = ` import { writeFileSync } from 'fs'; - let result; - let resultAsJson; - try { - const {default: fn} = await import('./code.mjs'); - result = await fn(); - if (result === undefined) { - result = { status: 'error', error: { message: 'Execution of innerCode returned undefined' } }; - } - resultAsJson = JSON.stringify(result); - } catch (e) { - if(e instanceof Error) { - result = { status: 'error', error: { message: e.message, stack: e.stack, cause: e.cause} }; - } else { - result = { status: 'error', error: { message: String(e), stack: undefined, cause: undefined} }; - } - resultAsJson = JSON.stringify(result); - } - writeFileSync('${resultPath}', resultAsJson); + import fn from './code.mjs'; + const result = await fn(); + writeFileSync(${JSON.stringify(resultPath)}, JSON.stringify(result)); `; await sandbox.writeFiles([ @@ -111,26 +96,14 @@ function createVercelSandboxEngine(): JsEngine { const runResult = await sandbox.runCommand('node', ['/vercel/sandbox/runner.mjs']); if (runResult.exitCode !== 0) { - // This shouldn't happen since the runner has a try-catch wrapper. - // If we get here, something unexpected failed (OOM, timeout, disk error, etc.) - throw new StackAssertionError("Vercel Sandbox runner failed unexpectedly outside of runner safeguards. This should never happen.", { innerCode: code, innerOptions: options, exitCode: runResult.exitCode }); + throw new StackAssertionError("Vercel Sandbox runner exited with non-zero code", { innerCode: code, innerOptions: options, exitCode: runResult.exitCode }); } - // Read the result file by catting it to stdout - let resultJson = ''; - const { Writable } = await import('stream'); - const stdoutStream = new Writable({ - write(chunk, _encoding, callback) { - resultJson += chunk.toString(); - callback(); - }, - }); - - const catResult = await sandbox.runCommand({ cmd: 'cat', args: [resultPath], stdout: stdoutStream }); - - if (catResult.exitCode !== 0) { - throw new StackAssertionError("Failed to read result file from Vercel Sandbox", { exitCode: catResult.exitCode, innerCode: code, innerOptions: options }); + const resultBuffer = await sandbox.readFileToBuffer({ path: resultPath }); + if (resultBuffer === null) { + throw new StackAssertionError("Result file not found in Vercel Sandbox", { resultPath, innerCode: code, innerOptions: options }); } + const resultJson = resultBuffer.toString(); try { return JSON.parse(resultJson); @@ -151,6 +124,11 @@ const engineMap = new Map([ const engines: JsEngine[] = Array.from(engineMap.values()); +/** + * Executes the given code with the given options. Returns the result of the code execution + * if it is JSON-serializable. Has undefined behavior if it is not JSON-serializable or if + * the code throws an error. + */ export async function executeJavascript(code: string, options: ExecuteJavascriptOptions = {}): Promise { return await traceSpan({ description: 'js-execution.executeJavascript', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b533bc9fb3..7ad38e6779 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -196,8 +196,8 @@ importers: specifier: ^1.10.4 version: 1.10.4(@opentelemetry/api-logs@0.53.0)(@opentelemetry/api@1.9.0)(@opentelemetry/instrumentation@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/resources@1.26.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-logs@0.53.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-metrics@2.2.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.26.0(@opentelemetry/api@1.9.0)) '@vercel/sandbox': - specifier: ^1.1.2 - version: 1.1.9 + specifier: ^1.2.0 + version: 1.2.0 ai: specifier: ^4.3.17 version: 4.3.17(react@19.2.3)(zod@3.25.76) @@ -9157,8 +9157,8 @@ packages: '@opentelemetry/sdk-metrics': ^1.19.0 '@opentelemetry/sdk-trace-base': ^1.19.0 - '@vercel/sandbox@1.1.9': - resolution: {integrity: sha512-DbCYoA8GVMRshFtXNpSPX+7xqgCoUgk0Zm+F0B22UgvtvmAvxpxtPp9t8pfW56AUez3UXhvdJNIaO16A9G6AkQ==} + '@vercel/sandbox@1.2.0': + resolution: {integrity: sha512-oq0d2xQuiDq8LyoZOZW+i+QmNkMQ0/ddEGHMukBsF+cxX7ohBEXOV/a0+SopBwEsKCjq9j55QFP56G3MU/iEjA==} '@vercel/speed-insights@1.0.12': resolution: {integrity: sha512-ZGQ+a7bcfWJD2VYEp2R1LHvRAMyyaFBYytZXsfnbOMkeOvzGNVxUL7aVUvisIrTZjXTSsxG45DKX7yiw6nq2Jw==} @@ -25418,7 +25418,7 @@ snapshots: '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.26.0(@opentelemetry/api@1.9.0) - '@vercel/sandbox@1.1.9': + '@vercel/sandbox@1.2.0': dependencies: '@vercel/oidc': 3.1.0 async-retry: 1.3.3